Lines Matching refs:hem
84 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx, in hns_roce_check_hem_null() argument
92 if (i != hem_idx && hem[i]) in hns_roce_check_hem_null()
257 struct hns_roce_hem *hem; in hns_roce_alloc_hem() local
264 hem = kmalloc(sizeof(*hem), in hns_roce_alloc_hem()
266 if (!hem) in hns_roce_alloc_hem()
269 INIT_LIST_HEAD(&hem->chunk_list); in hns_roce_alloc_hem()
284 list_add_tail(&chunk->list, &hem->chunk_list); in hns_roce_alloc_hem()
308 return hem; in hns_roce_alloc_hem()
311 hns_roce_free_hem(hr_dev, hem); in hns_roce_alloc_hem()
315 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) in hns_roce_free_hem() argument
320 if (!hem) in hns_roce_free_hem()
323 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { in hns_roce_free_hem()
332 kfree(hem); in hns_roce_free_hem()
394 hns_roce_free_hem(hr_dev, table->hem[index->buf]); in free_mhop_hem()
395 table->hem[index->buf] = NULL; in free_mhop_hem()
459 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, in alloc_mhop_hem()
461 if (!table->hem[index->buf]) { in alloc_mhop_hem()
467 hns_roce_hem_first(table->hem[index->buf], &iter); in alloc_mhop_hem()
539 if (table->hem[index.buf]) { in hns_roce_table_mhop_get()
540 refcount_inc(&table->hem[index.buf]->refcount); in hns_roce_table_mhop_get()
559 refcount_set(&table->hem[index.buf]->refcount, 1); in hns_roce_table_mhop_get()
583 if (table->hem[i]) { in hns_roce_table_get()
584 refcount_inc(&table->hem[i]->refcount); in hns_roce_table_get()
588 table->hem[i] = hns_roce_alloc_hem(hr_dev, in hns_roce_table_get()
592 if (!table->hem[i]) { in hns_roce_table_get()
599 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_get()
600 table->hem[i] = NULL; in hns_roce_table_get()
606 refcount_set(&table->hem[i]->refcount, 1); in hns_roce_table_get()
625 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
629 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
675 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, in hns_roce_table_mhop_put()
698 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, in hns_roce_table_put()
705 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_put()
706 table->hem[i] = NULL; in hns_roce_table_put()
717 struct hns_roce_hem *hem; in hns_roce_table_find() local
731 hem = table->hem[obj / obj_per_chunk]; in hns_roce_table_find()
748 hem = table->hem[hem_idx]; in hns_roce_table_find()
754 if (!hem) in hns_roce_table_find()
757 list_for_each_entry(chunk, &hem->chunk_list, list) { in hns_roce_table_find()
792 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); in hns_roce_init_hem_table()
793 if (!table->hem) in hns_roce_init_hem_table()
818 table->hem = kcalloc(num_hem, sizeof(*table->hem), in hns_roce_init_hem_table()
820 if (!table->hem) in hns_roce_init_hem_table()
876 kfree(table->hem); in hns_roce_init_hem_table()
877 table->hem = NULL; in hns_roce_init_hem_table()
898 if (table->hem[i]) in hns_roce_cleanup_mhop_hem_table()
902 kfree(table->hem); in hns_roce_cleanup_mhop_hem_table()
903 table->hem = NULL; in hns_roce_cleanup_mhop_hem_table()
926 if (table->hem[i]) { in hns_roce_cleanup_hem_table()
931 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_cleanup_hem_table()
934 kfree(table->hem); in hns_roce_cleanup_hem_table()
985 struct hns_roce_hem_item *hem; in hem_list_alloc_item() local
987 hem = kzalloc(sizeof(*hem), GFP_KERNEL); in hem_list_alloc_item()
988 if (!hem) in hem_list_alloc_item()
992 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN, in hem_list_alloc_item()
993 &hem->dma_addr, GFP_KERNEL); in hem_list_alloc_item()
994 if (!hem->addr) { in hem_list_alloc_item()
995 kfree(hem); in hem_list_alloc_item()
1000 hem->count = count; in hem_list_alloc_item()
1001 hem->start = start; in hem_list_alloc_item()
1002 hem->end = end; in hem_list_alloc_item()
1003 INIT_LIST_HEAD(&hem->list); in hem_list_alloc_item()
1004 INIT_LIST_HEAD(&hem->sibling); in hem_list_alloc_item()
1006 return hem; in hem_list_alloc_item()
1010 struct hns_roce_hem_item *hem, bool exist_bt) in hem_list_free_item() argument
1013 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, in hem_list_free_item()
1014 hem->addr, hem->dma_addr); in hem_list_free_item()
1015 kfree(hem); in hem_list_free_item()
1021 struct hns_roce_hem_item *hem, *temp_hem; in hem_list_free_all() local
1023 list_for_each_entry_safe(hem, temp_hem, head, list) { in hem_list_free_all()
1024 list_del(&hem->list); in hem_list_free_all()
1025 hem_list_free_item(hr_dev, hem, exist_bt); in hem_list_free_all()
1037 struct hns_roce_hem_item *hem, void *cpu_addr, in hem_list_assign_bt() argument
1040 hem->addr = cpu_addr; in hem_list_assign_bt()
1041 hem->dma_addr = (dma_addr_t)phy_addr; in hem_list_assign_bt()
1044 static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem, in hem_list_page_is_in_range() argument
1047 return (hem->start <= offset && offset <= hem->end); in hem_list_page_is_in_range()
1053 struct hns_roce_hem_item *hem, *temp_hem; in hem_list_search_item() local
1056 list_for_each_entry_safe(hem, temp_hem, ba_list, list) { in hem_list_search_item()
1057 if (hem_list_page_is_in_range(hem, page_offset)) { in hem_list_search_item()
1058 found = hem; in hem_list_search_item()
1229 struct hns_roce_hem_item *hem; in alloc_root_hem() local
1243 hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, in alloc_root_hem()
1245 if (!hem) in alloc_root_hem()
1250 return hem; in alloc_root_hem()
1258 struct hns_roce_hem_item *hem; in alloc_fake_root_bt() local
1260 hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, in alloc_fake_root_bt()
1262 if (!hem) in alloc_fake_root_bt()
1265 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); in alloc_fake_root_bt()
1266 list_add(&hem->list, branch_head); in alloc_fake_root_bt()
1267 list_add(&hem->sibling, leaf_head); in alloc_fake_root_bt()
1276 struct hns_roce_hem_item *hem, *temp_hem; in setup_middle_bt() local
1286 list_for_each_entry_safe(hem, temp_hem, branch_head, list) { in setup_middle_bt()
1287 offset = (hem->start - r->offset) / step * BA_BYTE_LEN; in setup_middle_bt()
1288 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr); in setup_middle_bt()
1468 struct hns_roce_hem_item *hem, *temp_hem; in hns_roce_hem_list_find_mtt() local
1472 list_for_each_entry_safe(hem, temp_hem, head, sibling) { in hns_roce_hem_list_find_mtt()
1473 if (hem_list_page_is_in_range(hem, offset)) { in hns_roce_hem_list_find_mtt()
1474 nr = offset - hem->start; in hns_roce_hem_list_find_mtt()
1475 cpu_base = hem->addr + nr * BA_BYTE_LEN; in hns_roce_hem_list_find_mtt()
1476 nr = hem->end + 1 - offset; in hns_roce_hem_list_find_mtt()