Lines Matching refs:table
203 struct hns_roce_hem_table *table, unsigned long *obj, in hns_roce_calc_hem_mhop() argument
212 if (get_hem_table_config(hr_dev, mhop, table->type)) in hns_roce_calc_hem_mhop()
222 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
224 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : in hns_roce_calc_hem_mhop()
226 table_idx = *obj / (chunk_size / table->obj_size); in hns_roce_calc_hem_mhop()
242 table->type, mhop->hop_num); in hns_roce_calc_hem_mhop()
336 struct hns_roce_hem_table *table, unsigned long obj, in calc_hem_config() argument
347 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); in calc_hem_config()
355 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); in calc_hem_config()
372 table->type, mhop->hop_num); in calc_hem_config()
376 if (unlikely(index->buf >= table->num_hem)) { in calc_hem_config()
378 table->type, index->buf, table->num_hem); in calc_hem_config()
386 struct hns_roce_hem_table *table, in free_mhop_hem() argument
394 hns_roce_free_hem(hr_dev, table->hem[index->buf]); in free_mhop_hem()
395 table->hem[index->buf] = NULL; in free_mhop_hem()
399 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], in free_mhop_hem()
400 table->bt_l1_dma_addr[index->l1]); in free_mhop_hem()
401 table->bt_l1[index->l1] = NULL; in free_mhop_hem()
405 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], in free_mhop_hem()
406 table->bt_l0_dma_addr[index->l0]); in free_mhop_hem()
407 table->bt_l0[index->l0] = NULL; in free_mhop_hem()
412 struct hns_roce_hem_table *table, in alloc_mhop_hem() argument
425 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || in alloc_mhop_hem()
426 check_whether_bt_num_2(table->type, mhop->hop_num)) && in alloc_mhop_hem()
427 !table->bt_l0[index->l0]) { in alloc_mhop_hem()
428 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
429 &table->bt_l0_dma_addr[index->l0], in alloc_mhop_hem()
431 if (!table->bt_l0[index->l0]) { in alloc_mhop_hem()
439 if (check_whether_bt_num_3(table->type, mhop->hop_num) && in alloc_mhop_hem()
440 !table->bt_l1[index->l1]) { in alloc_mhop_hem()
441 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, in alloc_mhop_hem()
442 &table->bt_l1_dma_addr[index->l1], in alloc_mhop_hem()
444 if (!table->bt_l1[index->l1]) { in alloc_mhop_hem()
449 *(table->bt_l0[index->l0] + mhop->l1_idx) = in alloc_mhop_hem()
450 table->bt_l1_dma_addr[index->l1]; in alloc_mhop_hem()
457 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; in alloc_mhop_hem()
459 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, in alloc_mhop_hem()
461 if (!table->hem[index->buf]) { in alloc_mhop_hem()
467 hns_roce_hem_first(table->hem[index->buf], &iter); in alloc_mhop_hem()
469 if (table->type < HEM_TYPE_MTT) { in alloc_mhop_hem()
471 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; in alloc_mhop_hem()
473 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
475 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; in alloc_mhop_hem()
480 free_mhop_hem(hr_dev, table, mhop, index); in alloc_mhop_hem()
486 struct hns_roce_hem_table *table, unsigned long obj, in set_mhop_hem() argument
495 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); in set_mhop_hem()
503 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); in set_mhop_hem()
515 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); in set_mhop_hem()
524 struct hns_roce_hem_table *table, in hns_roce_table_mhop_get() argument
532 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
538 mutex_lock(&table->mutex); in hns_roce_table_mhop_get()
539 if (table->hem[index.buf]) { in hns_roce_table_mhop_get()
540 refcount_inc(&table->hem[index.buf]->refcount); in hns_roce_table_mhop_get()
544 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
551 if (table->type < HEM_TYPE_MTT) { in hns_roce_table_mhop_get()
552 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_get()
559 refcount_set(&table->hem[index.buf]->refcount, 1); in hns_roce_table_mhop_get()
563 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_get()
565 mutex_unlock(&table->mutex); in hns_roce_table_mhop_get()
570 struct hns_roce_hem_table *table, unsigned long obj) in hns_roce_table_get() argument
576 if (hns_roce_check_whether_mhop(hr_dev, table->type)) in hns_roce_table_get()
577 return hns_roce_table_mhop_get(hr_dev, table, obj); in hns_roce_table_get()
579 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_get()
581 mutex_lock(&table->mutex); in hns_roce_table_get()
583 if (table->hem[i]) { in hns_roce_table_get()
584 refcount_inc(&table->hem[i]->refcount); in hns_roce_table_get()
588 table->hem[i] = hns_roce_alloc_hem(hr_dev, in hns_roce_table_get()
589 table->table_chunk_size >> PAGE_SHIFT, in hns_roce_table_get()
590 table->table_chunk_size, in hns_roce_table_get()
592 if (!table->hem[i]) { in hns_roce_table_get()
598 if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) { in hns_roce_table_get()
599 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_get()
600 table->hem[i] = NULL; in hns_roce_table_get()
606 refcount_set(&table->hem[i]->refcount, 1); in hns_roce_table_get()
608 mutex_unlock(&table->mutex); in hns_roce_table_get()
613 struct hns_roce_hem_table *table, unsigned long obj, in clear_mhop_hem() argument
624 if (check_whether_bt_num_2(table->type, hop_num)) { in clear_mhop_hem()
625 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
626 chunk_ba_num, table->num_hem)) in clear_mhop_hem()
628 } else if (check_whether_bt_num_3(table->type, hop_num)) { in clear_mhop_hem()
629 if (hns_roce_check_hem_null(table->hem, index->buf, in clear_mhop_hem()
630 chunk_ba_num, table->num_hem)) { in clear_mhop_hem()
632 if (hns_roce_check_bt_null(table->bt_l1, index->l1, in clear_mhop_hem()
638 if (table->type < HEM_TYPE_MTT) { in clear_mhop_hem()
644 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) in clear_mhop_hem()
648 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) in clear_mhop_hem()
652 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) in clear_mhop_hem()
658 struct hns_roce_hem_table *table, in hns_roce_table_mhop_put() argument
667 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
674 mutex_lock(&table->mutex); in hns_roce_table_mhop_put()
675 else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount, in hns_roce_table_mhop_put()
676 &table->mutex)) in hns_roce_table_mhop_put()
679 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); in hns_roce_table_mhop_put()
680 free_mhop_hem(hr_dev, table, &mhop, &index); in hns_roce_table_mhop_put()
682 mutex_unlock(&table->mutex); in hns_roce_table_mhop_put()
686 struct hns_roce_hem_table *table, unsigned long obj) in hns_roce_table_put() argument
691 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_put()
692 hns_roce_table_mhop_put(hr_dev, table, obj, 1); in hns_roce_table_put()
696 i = obj / (table->table_chunk_size / table->obj_size); in hns_roce_table_put()
698 if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount, in hns_roce_table_put()
699 &table->mutex)) in hns_roce_table_put()
702 if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) in hns_roce_table_put()
705 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_table_put()
706 table->hem[i] = NULL; in hns_roce_table_put()
708 mutex_unlock(&table->mutex); in hns_roce_table_put()
712 struct hns_roce_hem_table *table, in hns_roce_table_find() argument
727 mutex_lock(&table->mutex); in hns_roce_table_find()
729 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_table_find()
730 obj_per_chunk = table->table_chunk_size / table->obj_size; in hns_roce_table_find()
731 hem = table->hem[obj / obj_per_chunk]; in hns_roce_table_find()
733 dma_offset = offset = idx_offset * table->obj_size; in hns_roce_table_find()
737 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) in hns_roce_table_find()
748 hem = table->hem[hem_idx]; in hns_roce_table_find()
776 mutex_unlock(&table->mutex); in hns_roce_table_find()
781 struct hns_roce_hem_table *table, u32 type, in hns_roce_init_hem_table() argument
788 table->table_chunk_size = hr_dev->caps.chunk_sz; in hns_roce_init_hem_table()
789 obj_per_chunk = table->table_chunk_size / obj_size; in hns_roce_init_hem_table()
792 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); in hns_roce_init_hem_table()
793 if (!table->hem) in hns_roce_init_hem_table()
818 table->hem = kcalloc(num_hem, sizeof(*table->hem), in hns_roce_init_hem_table()
820 if (!table->hem) in hns_roce_init_hem_table()
827 table->bt_l1 = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
828 sizeof(*table->bt_l1), in hns_roce_init_hem_table()
830 if (!table->bt_l1) in hns_roce_init_hem_table()
833 table->bt_l1_dma_addr = kcalloc(num_bt_l1, in hns_roce_init_hem_table()
834 sizeof(*table->bt_l1_dma_addr), in hns_roce_init_hem_table()
837 if (!table->bt_l1_dma_addr) in hns_roce_init_hem_table()
843 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), in hns_roce_init_hem_table()
845 if (!table->bt_l0) in hns_roce_init_hem_table()
848 table->bt_l0_dma_addr = kcalloc(num_bt_l0, in hns_roce_init_hem_table()
849 sizeof(*table->bt_l0_dma_addr), in hns_roce_init_hem_table()
851 if (!table->bt_l0_dma_addr) in hns_roce_init_hem_table()
856 table->type = type; in hns_roce_init_hem_table()
857 table->num_hem = num_hem; in hns_roce_init_hem_table()
858 table->obj_size = obj_size; in hns_roce_init_hem_table()
859 mutex_init(&table->mutex); in hns_roce_init_hem_table()
864 kfree(table->bt_l0); in hns_roce_init_hem_table()
865 table->bt_l0 = NULL; in hns_roce_init_hem_table()
868 kfree(table->bt_l1_dma_addr); in hns_roce_init_hem_table()
869 table->bt_l1_dma_addr = NULL; in hns_roce_init_hem_table()
872 kfree(table->bt_l1); in hns_roce_init_hem_table()
873 table->bt_l1 = NULL; in hns_roce_init_hem_table()
876 kfree(table->hem); in hns_roce_init_hem_table()
877 table->hem = NULL; in hns_roce_init_hem_table()
884 struct hns_roce_hem_table *table) in hns_roce_cleanup_mhop_hem_table() argument
891 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) in hns_roce_cleanup_mhop_hem_table()
893 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : in hns_roce_cleanup_mhop_hem_table()
896 for (i = 0; i < table->num_hem; ++i) { in hns_roce_cleanup_mhop_hem_table()
897 obj = i * buf_chunk_size / table->obj_size; in hns_roce_cleanup_mhop_hem_table()
898 if (table->hem[i]) in hns_roce_cleanup_mhop_hem_table()
899 hns_roce_table_mhop_put(hr_dev, table, obj, 0); in hns_roce_cleanup_mhop_hem_table()
902 kfree(table->hem); in hns_roce_cleanup_mhop_hem_table()
903 table->hem = NULL; in hns_roce_cleanup_mhop_hem_table()
904 kfree(table->bt_l1); in hns_roce_cleanup_mhop_hem_table()
905 table->bt_l1 = NULL; in hns_roce_cleanup_mhop_hem_table()
906 kfree(table->bt_l1_dma_addr); in hns_roce_cleanup_mhop_hem_table()
907 table->bt_l1_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
908 kfree(table->bt_l0); in hns_roce_cleanup_mhop_hem_table()
909 table->bt_l0 = NULL; in hns_roce_cleanup_mhop_hem_table()
910 kfree(table->bt_l0_dma_addr); in hns_roce_cleanup_mhop_hem_table()
911 table->bt_l0_dma_addr = NULL; in hns_roce_cleanup_mhop_hem_table()
915 struct hns_roce_hem_table *table) in hns_roce_cleanup_hem_table() argument
920 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { in hns_roce_cleanup_hem_table()
921 hns_roce_cleanup_mhop_hem_table(hr_dev, table); in hns_roce_cleanup_hem_table()
925 for (i = 0; i < table->num_hem; ++i) in hns_roce_cleanup_hem_table()
926 if (table->hem[i]) { in hns_roce_cleanup_hem_table()
927 if (hr_dev->hw->clear_hem(hr_dev, table, in hns_roce_cleanup_hem_table()
928 i * table->table_chunk_size / table->obj_size, 0)) in hns_roce_cleanup_hem_table()
931 hns_roce_free_hem(hr_dev, table->hem[i]); in hns_roce_cleanup_hem_table()
934 kfree(table->hem); in hns_roce_cleanup_hem_table()
941 &hr_dev->srq_table.table); in hns_roce_cleanup_hem()
942 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); in hns_roce_cleanup_hem()