Lines Matching refs:hr_dev
48 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) in hns_roce_check_whether_mhop() argument
54 hop_num = hr_dev->caps.qpc_hop_num; in hns_roce_check_whether_mhop()
57 hop_num = hr_dev->caps.mpt_hop_num; in hns_roce_check_whether_mhop()
60 hop_num = hr_dev->caps.cqc_hop_num; in hns_roce_check_whether_mhop()
63 hop_num = hr_dev->caps.srqc_hop_num; in hns_roce_check_whether_mhop()
66 hop_num = hr_dev->caps.sccc_hop_num; in hns_roce_check_whether_mhop()
69 hop_num = hr_dev->caps.qpc_timer_hop_num; in hns_roce_check_whether_mhop()
72 hop_num = hr_dev->caps.cqc_timer_hop_num; in hns_roce_check_whether_mhop()
75 hop_num = hr_dev->caps.gmv_hop_num; in hns_roce_check_whether_mhop()
122 static int get_hem_table_config(struct hns_roce_dev *hr_dev, in get_hem_table_config() argument
126 struct device *dev = hr_dev->dev; in get_hem_table_config()
130 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz in get_hem_table_config()
132 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz in get_hem_table_config()
134 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; in get_hem_table_config()
135 mhop->hop_num = hr_dev->caps.qpc_hop_num; in get_hem_table_config()
138 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz in get_hem_table_config()
140 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz in get_hem_table_config()
142 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; in get_hem_table_config()
143 mhop->hop_num = hr_dev->caps.mpt_hop_num; in get_hem_table_config()
146 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz in get_hem_table_config()
148 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz in get_hem_table_config()
150 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; in get_hem_table_config()
151 mhop->hop_num = hr_dev->caps.cqc_hop_num; in get_hem_table_config()
154 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz in get_hem_table_config()
156 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz in get_hem_table_config()
158 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; in get_hem_table_config()
159 mhop->hop_num = hr_dev->caps.sccc_hop_num; in get_hem_table_config()
162 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz in get_hem_table_config()
164 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz in get_hem_table_config()
166 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; in get_hem_table_config()
167 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; in get_hem_table_config()
170 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz in get_hem_table_config()
172 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz in get_hem_table_config()
174 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; in get_hem_table_config()
175 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; in get_hem_table_config()
178 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz in get_hem_table_config()
180 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz in get_hem_table_config()
182 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; in get_hem_table_config()
183 mhop->hop_num = hr_dev->caps.srqc_hop_num; in get_hem_table_config()
186 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + in get_hem_table_config()
188 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz + in get_hem_table_config()
190 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num; in get_hem_table_config()
191 mhop->hop_num = hr_dev->caps.gmv_hop_num; in get_hem_table_config()
202 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, in hns_roce_calc_hem_mhop() argument
206 struct device *dev = hr_dev->dev; in hns_roce_calc_hem_mhop()
212 if (get_hem_table_config(hr_dev, mhop, table->type)) in hns_roce_calc_hem_mhop()
251 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, in hns_roce_alloc_hem() argument
295 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, in hns_roce_alloc_hem()
311 hns_roce_free_hem(hr_dev, hem); in hns_roce_alloc_hem()
315 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) in hns_roce_free_hem() argument
325 dma_free_coherent(hr_dev->dev, in hns_roce_free_hem()
335 static int calc_hem_config(struct hns_roce_dev *hr_dev, in calc_hem_config() argument
340 struct ib_device *ibdev = &hr_dev->ib_dev; in calc_hem_config()
347 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); in calc_hem_config()
385 static void free_mhop_hem(struct hns_roce_dev *hr_dev, in free_mhop_hem() argument
391 struct device *dev = hr_dev->dev; in free_mhop_hem()
394 hns_roce_free_hem(hr_dev, table->hem[index->buf]); in free_mhop_hem()
411 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, in alloc_mhop_hem() argument
417 struct device *dev = hr_dev->dev; in alloc_mhop_hem()
459 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, in alloc_mhop_hem()
480 free_mhop_hem(hr_dev, table, mhop, index); in alloc_mhop_hem()
485 static int set_mhop_hem(struct hns_roce_dev *hr_dev, in set_mhop_hem() argument
490 struct ib_device *ibdev = &hr_dev->ib_dev; in set_mhop_hem()
495 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); in set_mhop_hem()
503 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); in set_mhop_hem()
515 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); in set_mhop_hem()
523 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, in hns_roce_table_mhop_get() argument
527 struct ib_device *ibdev = &hr_dev->ib_dev; in hns_roce_table_mhop_get()
532 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
544 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
552 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
563 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
569 int hns_roce_table_get(struct hns_roce_dev *hr_dev, in hns_roce_table_get() argument
572 struct device *dev = hr_dev->dev; in hns_roce_table_get()
576 if (hns_roce_check_whether_mhop(hr_dev, table->type)) in hns_roce_table_get()
577 return hns_roce_table_mhop_get(hr_dev, table, obj); in hns_roce_table_get()
588 table->hem[i] = hns_roce_alloc_hem(hr_dev, in hns_roce_table_get()
598 if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) { in hns_roce_table_get()
599 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_get()
612 static void clear_mhop_hem(struct hns_roce_dev *hr_dev, in clear_mhop_hem() argument
617 struct ib_device *ibdev = &hr_dev->ib_dev; in clear_mhop_hem()
644 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) in clear_mhop_hem()
648 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) in clear_mhop_hem()
652 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) in clear_mhop_hem()
657 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, in hns_roce_table_mhop_put() argument
662 struct ib_device *ibdev = &hr_dev->ib_dev; in hns_roce_table_mhop_put()
667 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
679 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
680 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_put()
685 void hns_roce_table_put(struct hns_roce_dev *hr_dev, in hns_roce_table_put() argument
688 struct device *dev = hr_dev->dev; in hns_roce_table_put()
691 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_put()
692 hns_roce_table_mhop_put(hr_dev, table, obj, 1); in hns_roce_table_put()
702 if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) in hns_roce_table_put()
705 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_put()
711 void *hns_roce_table_find(struct hns_roce_dev *hr_dev, in hns_roce_table_find() argument
729 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_find()
737 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) in hns_roce_table_find()
780 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, in hns_roce_init_hem_table() argument
787 if (!hns_roce_check_whether_mhop(hr_dev, type)) { in hns_roce_init_hem_table()
788 table->table_chunk_size = hr_dev->caps.chunk_sz; in hns_roce_init_hem_table()
803 if (get_hem_table_config(hr_dev, &mhop, type)) in hns_roce_init_hem_table()
883 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, in hns_roce_cleanup_mhop_hem_table() argument
891 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) in hns_roce_cleanup_mhop_hem_table()
899 hns_roce_table_mhop_put(hr_dev, table, obj, 0); in hns_roce_cleanup_mhop_hem_table()
914 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, in hns_roce_cleanup_hem_table() argument
917 struct device *dev = hr_dev->dev; in hns_roce_cleanup_hem_table()
920 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_cleanup_hem_table()
921 hns_roce_cleanup_mhop_hem_table(hr_dev, table); in hns_roce_cleanup_hem_table()
927 if (hr_dev->hw->clear_hem(hr_dev, table, in hns_roce_cleanup_hem_table()
931 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_cleanup_hem_table()
937 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) in hns_roce_cleanup_hem() argument
939 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) in hns_roce_cleanup_hem()
940 hns_roce_cleanup_hem_table(hr_dev, in hns_roce_cleanup_hem()
941 &hr_dev->srq_table.table); in hns_roce_cleanup_hem()
942 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); in hns_roce_cleanup_hem()
943 if (hr_dev->caps.qpc_timer_entry_sz) in hns_roce_cleanup_hem()
944 hns_roce_cleanup_hem_table(hr_dev, in hns_roce_cleanup_hem()
945 &hr_dev->qpc_timer_table); in hns_roce_cleanup_hem()
946 if (hr_dev->caps.cqc_timer_entry_sz) in hns_roce_cleanup_hem()
947 hns_roce_cleanup_hem_table(hr_dev, in hns_roce_cleanup_hem()
948 &hr_dev->cqc_timer_table); in hns_roce_cleanup_hem()
949 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) in hns_roce_cleanup_hem()
950 hns_roce_cleanup_hem_table(hr_dev, in hns_roce_cleanup_hem()
951 &hr_dev->qp_table.sccc_table); in hns_roce_cleanup_hem()
952 if (hr_dev->caps.trrl_entry_sz) in hns_roce_cleanup_hem()
953 hns_roce_cleanup_hem_table(hr_dev, in hns_roce_cleanup_hem()
954 &hr_dev->qp_table.trrl_table); in hns_roce_cleanup_hem()
956 if (hr_dev->caps.gmv_entry_sz) in hns_roce_cleanup_hem()
957 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table); in hns_roce_cleanup_hem()
959 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); in hns_roce_cleanup_hem()
960 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); in hns_roce_cleanup_hem()
961 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); in hns_roce_cleanup_hem()
982 hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count, in hem_list_alloc_item() argument
992 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN, in hem_list_alloc_item()
1009 static void hem_list_free_item(struct hns_roce_dev *hr_dev, in hem_list_free_item() argument
1013 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, in hem_list_free_item()
1018 static void hem_list_free_all(struct hns_roce_dev *hr_dev, in hem_list_free_all() argument
1025 hem_list_free_item(hr_dev, hem, exist_bt); in hem_list_free_all()
1029 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, in hem_list_link_bt() argument
1036 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, in hem_list_assign_bt() argument
1139 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, in hem_list_alloc_mid_bt() argument
1160 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); in hem_list_alloc_mid_bt()
1165 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n", in hem_list_alloc_mid_bt()
1191 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, in hem_list_alloc_mid_bt()
1206 hem_list_link_bt(hr_dev, pre->addr + step, in hem_list_alloc_mid_bt()
1219 hem_list_free_all(hr_dev, &temp_list[level], true); in hem_list_alloc_mid_bt()
1225 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, in alloc_root_hem() argument
1243 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, in alloc_root_hem()
1253 static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, in alloc_fake_root_bt() argument
1260 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, in alloc_fake_root_bt()
1265 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); in alloc_fake_root_bt()
1272 static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base, in setup_middle_bt() argument
1288 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); in setup_middle_bt()
1296 setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, in setup_root_hem() argument
1326 ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r, in setup_root_hem()
1329 ret = setup_middle_bt(hr_dev, cpu_base, unit, r, in setup_root_hem()
1346 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, in hem_list_alloc_root_bt() argument
1362 root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions, in hem_list_alloc_root_bt()
1375 ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions, in hem_list_alloc_root_bt()
1379 hem_list_free_all(hr_dev, &head.branch[i], false); in hem_list_alloc_root_bt()
1381 hem_list_free_all(hr_dev, &head.root, true); in hem_list_alloc_root_bt()
1388 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, in hns_roce_hem_list_request() argument
1400 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", in hns_roce_hem_list_request()
1413 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, in hns_roce_hem_list_request()
1417 dev_err(hr_dev->dev, in hns_roce_hem_list_request()
1424 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, in hns_roce_hem_list_request()
1427 dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret); in hns_roce_hem_list_request()
1432 hns_roce_hem_list_release(hr_dev, hem_list); in hns_roce_hem_list_request()
1437 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, in hns_roce_hem_list_release() argument
1444 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], in hns_roce_hem_list_release()
1447 hem_list_free_all(hr_dev, &hem_list->root_bt, true); in hns_roce_hem_list_release()
1463 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, in hns_roce_hem_list_find_mtt() argument