Lines Matching refs:imr
100 struct mlx5_ib_mr *imr, int flags) in populate_klm() argument
107 pklm->key = mr_to_mdev(imr)->mkeys.null_mkey; in populate_klm()
130 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); in populate_klm()
133 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); in populate_klm()
140 pklm->key = mr_to_mdev(imr)->mkeys.null_mkey; in populate_klm()
194 struct mlx5_ib_mr *imr = mr->parent; in free_implicit_child_mr_work() local
195 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); in free_implicit_child_mr_work()
207 mlx5r_deref_odp_mkey(&imr->mmkey); in free_implicit_child_mr_work()
214 struct mlx5_ib_mr *imr = mr->parent; in destroy_unused_implicit_child_mr() local
216 if (!refcount_inc_not_zero(&imr->mmkey.usecount)) in destroy_unused_implicit_child_mr()
219 xa_erase(&imr->implicit_children, idx); in destroy_unused_implicit_child_mr()
405 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, in implicit_get_child_mr() argument
408 struct mlx5_ib_dev *dev = mr_to_mdev(imr); in implicit_get_child_mr()
414 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), in implicit_get_child_mr()
420 mr = mlx5_mr_cache_alloc(dev, imr->access_flags, in implicit_get_child_mr()
428 mr->access_flags = imr->access_flags; in implicit_get_child_mr()
429 mr->ibmr.pd = imr->ibmr.pd; in implicit_get_child_mr()
430 mr->ibmr.device = &mr_to_mdev(imr)->ib_dev; in implicit_get_child_mr()
435 mr->parent = imr; in implicit_get_child_mr()
454 xa_lock(&imr->implicit_children); in implicit_get_child_mr()
455 ret = __xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, in implicit_get_child_mr()
469 xa_unlock(&imr->implicit_children); in implicit_get_child_mr()
471 mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr->mmkey.key, mr); in implicit_get_child_mr()
475 xa_unlock(&imr->implicit_children); in implicit_get_child_mr()
486 struct mlx5_ib_mr *imr; in mlx5_ib_alloc_implicit_mr() local
496 imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM, in mlx5_ib_alloc_implicit_mr()
498 if (IS_ERR(imr)) { in mlx5_ib_alloc_implicit_mr()
500 return imr; in mlx5_ib_alloc_implicit_mr()
503 imr->access_flags = access_flags; in mlx5_ib_alloc_implicit_mr()
504 imr->ibmr.pd = &pd->ibpd; in mlx5_ib_alloc_implicit_mr()
505 imr->ibmr.iova = 0; in mlx5_ib_alloc_implicit_mr()
506 imr->umem = &umem_odp->umem; in mlx5_ib_alloc_implicit_mr()
507 imr->ibmr.lkey = imr->mmkey.key; in mlx5_ib_alloc_implicit_mr()
508 imr->ibmr.rkey = imr->mmkey.key; in mlx5_ib_alloc_implicit_mr()
509 imr->ibmr.device = &dev->ib_dev; in mlx5_ib_alloc_implicit_mr()
510 imr->is_odp_implicit = true; in mlx5_ib_alloc_implicit_mr()
511 xa_init(&imr->implicit_children); in mlx5_ib_alloc_implicit_mr()
513 err = mlx5r_umr_update_xlt(imr, 0, in mlx5_ib_alloc_implicit_mr()
522 err = mlx5r_store_odp_mkey(dev, &imr->mmkey); in mlx5_ib_alloc_implicit_mr()
526 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); in mlx5_ib_alloc_implicit_mr()
527 return imr; in mlx5_ib_alloc_implicit_mr()
530 mlx5_ib_dereg_mr(&imr->ibmr, NULL); in mlx5_ib_alloc_implicit_mr()
604 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, in pagefault_implicit_mr() argument
626 xa_lock(&imr->implicit_children); in pagefault_implicit_mr()
627 mtt = xa_load(&imr->implicit_children, idx); in pagefault_implicit_mr()
629 xa_unlock(&imr->implicit_children); in pagefault_implicit_mr()
630 mtt = implicit_get_child_mr(imr, idx); in pagefault_implicit_mr()
639 xa_unlock(&imr->implicit_children); in pagefault_implicit_mr()
679 err = mlx5r_umr_update_xlt(imr, upd_start_idx, upd_len, 0, in pagefault_implicit_mr()
684 mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n"); in pagefault_implicit_mr()