Lines Matching refs:iopt
31 struct io_pagetable *iopt, in iopt_area_contig_init() argument
35 lockdep_assert_held(&iopt->iova_rwsem); in iopt_area_contig_init()
39 iter->area = iopt_area_iter_first(iopt, iova, iova); in iopt_area_contig_init()
108 static int iopt_alloc_iova(struct io_pagetable *iopt, unsigned long *iova, in iopt_alloc_iova() argument
116 lockdep_assert_held(&iopt->iova_rwsem); in iopt_alloc_iova()
133 if (iova_alignment < iopt->iova_alignment) in iopt_alloc_iova()
136 interval_tree_for_each_span(&allowed_span, &iopt->allowed_itree, in iopt_alloc_iova()
138 if (RB_EMPTY_ROOT(&iopt->allowed_itree.rb_root)) { in iopt_alloc_iova()
149 &used_span, &iopt->reserved_itree, &iopt->area_itree, in iopt_alloc_iova()
163 static int iopt_check_iova(struct io_pagetable *iopt, unsigned long iova, in iopt_check_iova() argument
168 lockdep_assert_held(&iopt->iova_rwsem); in iopt_check_iova()
170 if ((iova & (iopt->iova_alignment - 1))) in iopt_check_iova()
177 if (iopt_reserved_iter_first(iopt, iova, last)) in iopt_check_iova()
181 if (iopt_area_iter_first(iopt, iova, last)) in iopt_check_iova()
189 static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, in iopt_insert_area() argument
194 lockdep_assert_held_write(&iopt->iova_rwsem); in iopt_insert_area()
201 if (area->page_offset & (iopt->iova_alignment - 1)) in iopt_insert_area()
219 area->iopt = iopt; in iopt_insert_area()
220 interval_tree_insert(&area->node, &iopt->area_itree); in iopt_insert_area()
224 static int iopt_alloc_area_pages(struct io_pagetable *iopt, in iopt_alloc_area_pages() argument
239 down_write(&iopt->iova_rwsem); in iopt_alloc_area_pages()
240 if ((length & (iopt->iova_alignment - 1)) || !length) { in iopt_alloc_area_pages()
250 iopt, dst_iova, in iopt_alloc_area_pages()
255 WARN_ON(iopt_check_iova(iopt, *dst_iova, length))) { in iopt_alloc_area_pages()
260 rc = iopt_check_iova(iopt, *dst_iova, length); in iopt_alloc_area_pages()
271 rc = iopt_insert_area(iopt, elm->area, elm->pages, iova, in iopt_alloc_area_pages()
279 up_write(&iopt->iova_rwsem); in iopt_alloc_area_pages()
287 if (area->iopt) { in iopt_abort_area()
288 down_write(&area->iopt->iova_rwsem); in iopt_abort_area()
289 interval_tree_remove(&area->node, &area->iopt->area_itree); in iopt_abort_area()
290 up_write(&area->iopt->iova_rwsem); in iopt_abort_area()
332 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list, in iopt_map_pages() argument
339 rc = iopt_alloc_area_pages(iopt, pages_list, length, dst_iova, in iopt_map_pages()
344 down_read(&iopt->domains_rwsem); in iopt_map_pages()
349 down_write(&iopt->iova_rwsem); in iopt_map_pages()
360 up_write(&iopt->iova_rwsem); in iopt_map_pages()
362 up_read(&iopt->domains_rwsem); in iopt_map_pages()
385 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt, in iopt_map_user_pages() argument
404 rc = iopt_map_pages(iopt, &pages_list, length, iova, iommu_prot, flags); in iopt_map_user_pages()
415 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova, in iopt_get_pages() argument
428 down_read(&iopt->iova_rwsem); in iopt_get_pages()
429 iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) { in iopt_get_pages()
448 up_read(&iopt->iova_rwsem); in iopt_get_pages()
451 up_read(&iopt->iova_rwsem); in iopt_get_pages()
456 static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start, in iopt_unmap_iova_range() argument
469 down_read(&iopt->domains_rwsem); in iopt_unmap_iova_range()
470 down_write(&iopt->iova_rwsem); in iopt_unmap_iova_range()
471 while ((area = iopt_area_iter_first(iopt, start, last))) { in iopt_unmap_iova_range()
495 up_write(&iopt->iova_rwsem); in iopt_unmap_iova_range()
496 up_read(&iopt->domains_rwsem); in iopt_unmap_iova_range()
497 iommufd_access_notify_unmap(iopt, area_first, in iopt_unmap_iova_range()
506 up_write(&iopt->iova_rwsem); in iopt_unmap_iova_range()
514 down_write(&iopt->iova_rwsem); in iopt_unmap_iova_range()
520 up_write(&iopt->iova_rwsem); in iopt_unmap_iova_range()
521 up_read(&iopt->domains_rwsem); in iopt_unmap_iova_range()
537 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova, in iopt_unmap_iova() argument
548 return iopt_unmap_iova_range(iopt, iova, iova_last, unmapped); in iopt_unmap_iova()
551 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped) in iopt_unmap_all() argument
555 rc = iopt_unmap_iova_range(iopt, 0, ULONG_MAX, unmapped); in iopt_unmap_all()
563 int iopt_set_allow_iova(struct io_pagetable *iopt, in iopt_set_allow_iova() argument
568 down_write(&iopt->iova_rwsem); in iopt_set_allow_iova()
569 swap(*allowed_iova, iopt->allowed_itree); in iopt_set_allow_iova()
571 for (allowed = iopt_allowed_iter_first(iopt, 0, ULONG_MAX); allowed; in iopt_set_allow_iova()
573 if (iopt_reserved_iter_first(iopt, allowed->node.start, in iopt_set_allow_iova()
575 swap(*allowed_iova, iopt->allowed_itree); in iopt_set_allow_iova()
576 up_write(&iopt->iova_rwsem); in iopt_set_allow_iova()
580 up_write(&iopt->iova_rwsem); in iopt_set_allow_iova()
584 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start, in iopt_reserve_iova() argument
589 lockdep_assert_held_write(&iopt->iova_rwsem); in iopt_reserve_iova()
591 if (iopt_area_iter_first(iopt, start, last) || in iopt_reserve_iova()
592 iopt_allowed_iter_first(iopt, start, last)) in iopt_reserve_iova()
601 interval_tree_insert(&reserved->node, &iopt->reserved_itree); in iopt_reserve_iova()
605 static void __iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner) in __iopt_remove_reserved_iova() argument
609 lockdep_assert_held_write(&iopt->iova_rwsem); in __iopt_remove_reserved_iova()
611 for (reserved = iopt_reserved_iter_first(iopt, 0, ULONG_MAX); reserved; in __iopt_remove_reserved_iova()
617 &iopt->reserved_itree); in __iopt_remove_reserved_iova()
623 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner) in iopt_remove_reserved_iova() argument
625 down_write(&iopt->iova_rwsem); in iopt_remove_reserved_iova()
626 __iopt_remove_reserved_iova(iopt, owner); in iopt_remove_reserved_iova()
627 up_write(&iopt->iova_rwsem); in iopt_remove_reserved_iova()
630 void iopt_init_table(struct io_pagetable *iopt) in iopt_init_table() argument
632 init_rwsem(&iopt->iova_rwsem); in iopt_init_table()
633 init_rwsem(&iopt->domains_rwsem); in iopt_init_table()
634 iopt->area_itree = RB_ROOT_CACHED; in iopt_init_table()
635 iopt->allowed_itree = RB_ROOT_CACHED; in iopt_init_table()
636 iopt->reserved_itree = RB_ROOT_CACHED; in iopt_init_table()
637 xa_init_flags(&iopt->domains, XA_FLAGS_ACCOUNT); in iopt_init_table()
638 xa_init_flags(&iopt->access_list, XA_FLAGS_ALLOC); in iopt_init_table()
645 iopt->iova_alignment = 1; in iopt_init_table()
648 void iopt_destroy_table(struct io_pagetable *iopt) in iopt_destroy_table() argument
653 iopt_remove_reserved_iova(iopt, NULL); in iopt_destroy_table()
655 while ((node = interval_tree_iter_first(&iopt->allowed_itree, 0, in iopt_destroy_table()
657 interval_tree_remove(node, &iopt->allowed_itree); in iopt_destroy_table()
661 WARN_ON(!RB_EMPTY_ROOT(&iopt->reserved_itree.rb_root)); in iopt_destroy_table()
662 WARN_ON(!xa_empty(&iopt->domains)); in iopt_destroy_table()
663 WARN_ON(!xa_empty(&iopt->access_list)); in iopt_destroy_table()
664 WARN_ON(!RB_EMPTY_ROOT(&iopt->area_itree.rb_root)); in iopt_destroy_table()
676 static void iopt_unfill_domain(struct io_pagetable *iopt, in iopt_unfill_domain() argument
681 lockdep_assert_held(&iopt->iova_rwsem); in iopt_unfill_domain()
682 lockdep_assert_held_write(&iopt->domains_rwsem); in iopt_unfill_domain()
688 if (iopt->next_domain_id != 0) { in iopt_unfill_domain()
691 xa_load(&iopt->domains, 0); in iopt_unfill_domain()
693 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_unfill_domain()
712 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_unfill_domain()
736 static int iopt_fill_domain(struct io_pagetable *iopt, in iopt_fill_domain() argument
743 lockdep_assert_held(&iopt->iova_rwsem); in iopt_fill_domain()
744 lockdep_assert_held_write(&iopt->domains_rwsem); in iopt_fill_domain()
746 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_fill_domain()
760 WARN_ON(iopt->next_domain_id != 0); in iopt_fill_domain()
771 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_fill_domain()
780 if (iopt->next_domain_id == 0) { in iopt_fill_domain()
792 static int iopt_check_iova_alignment(struct io_pagetable *iopt, in iopt_check_iova_alignment() argument
798 lockdep_assert_held(&iopt->iova_rwsem); in iopt_check_iova_alignment()
799 lockdep_assert_held(&iopt->domains_rwsem); in iopt_check_iova_alignment()
801 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_check_iova_alignment()
812 xa_for_each(&iopt->access_list, index, access) in iopt_check_iova_alignment()
820 int iopt_table_add_domain(struct io_pagetable *iopt, in iopt_table_add_domain() argument
829 down_write(&iopt->domains_rwsem); in iopt_table_add_domain()
830 down_write(&iopt->iova_rwsem); in iopt_table_add_domain()
832 xa_for_each(&iopt->domains, index, iter_domain) { in iopt_table_add_domain()
849 iopt->iova_alignment); in iopt_table_add_domain()
854 if (new_iova_alignment != iopt->iova_alignment) { in iopt_table_add_domain()
855 rc = iopt_check_iova_alignment(iopt, new_iova_alignment); in iopt_table_add_domain()
862 rc = iopt_reserve_iova(iopt, 0, geometry->aperture_start - 1, in iopt_table_add_domain()
868 rc = iopt_reserve_iova(iopt, geometry->aperture_end + 1, in iopt_table_add_domain()
874 rc = xa_reserve(&iopt->domains, iopt->next_domain_id, GFP_KERNEL); in iopt_table_add_domain()
878 rc = iopt_fill_domain(iopt, domain); in iopt_table_add_domain()
882 iopt->iova_alignment = new_iova_alignment; in iopt_table_add_domain()
883 xa_store(&iopt->domains, iopt->next_domain_id, domain, GFP_KERNEL); in iopt_table_add_domain()
884 iopt->next_domain_id++; in iopt_table_add_domain()
885 up_write(&iopt->iova_rwsem); in iopt_table_add_domain()
886 up_write(&iopt->domains_rwsem); in iopt_table_add_domain()
889 xa_release(&iopt->domains, iopt->next_domain_id); in iopt_table_add_domain()
891 __iopt_remove_reserved_iova(iopt, domain); in iopt_table_add_domain()
893 up_write(&iopt->iova_rwsem); in iopt_table_add_domain()
894 up_write(&iopt->domains_rwsem); in iopt_table_add_domain()
898 static int iopt_calculate_iova_alignment(struct io_pagetable *iopt) in iopt_calculate_iova_alignment() argument
905 lockdep_assert_held_write(&iopt->iova_rwsem); in iopt_calculate_iova_alignment()
906 lockdep_assert_held(&iopt->domains_rwsem); in iopt_calculate_iova_alignment()
909 if (iopt->disable_large_pages) in iopt_calculate_iova_alignment()
914 xa_for_each(&iopt->domains, index, domain) in iopt_calculate_iova_alignment()
918 xa_for_each(&iopt->access_list, index, access) in iopt_calculate_iova_alignment()
923 if (new_iova_alignment > iopt->iova_alignment) { in iopt_calculate_iova_alignment()
926 rc = iopt_check_iova_alignment(iopt, new_iova_alignment); in iopt_calculate_iova_alignment()
930 iopt->iova_alignment = new_iova_alignment; in iopt_calculate_iova_alignment()
934 void iopt_table_remove_domain(struct io_pagetable *iopt, in iopt_table_remove_domain() argument
940 down_write(&iopt->domains_rwsem); in iopt_table_remove_domain()
941 down_write(&iopt->iova_rwsem); in iopt_table_remove_domain()
943 xa_for_each(&iopt->domains, index, iter_domain) in iopt_table_remove_domain()
946 if (WARN_ON(iter_domain != domain) || index >= iopt->next_domain_id) in iopt_table_remove_domain()
953 iopt->next_domain_id--; in iopt_table_remove_domain()
954 iter_domain = xa_erase(&iopt->domains, iopt->next_domain_id); in iopt_table_remove_domain()
955 if (index != iopt->next_domain_id) in iopt_table_remove_domain()
956 xa_store(&iopt->domains, index, iter_domain, GFP_KERNEL); in iopt_table_remove_domain()
958 iopt_unfill_domain(iopt, domain); in iopt_table_remove_domain()
959 __iopt_remove_reserved_iova(iopt, domain); in iopt_table_remove_domain()
961 WARN_ON(iopt_calculate_iova_alignment(iopt)); in iopt_table_remove_domain()
963 up_write(&iopt->iova_rwsem); in iopt_table_remove_domain()
964 up_write(&iopt->domains_rwsem); in iopt_table_remove_domain()
978 unsigned long alignment = area->iopt->iova_alignment; in iopt_area_split()
982 struct io_pagetable *iopt = area->iopt; in iopt_area_split() local
988 lockdep_assert_held_write(&iopt->iova_rwsem); in iopt_area_split()
1024 if (area->storage_domain && !iopt->disable_large_pages) { in iopt_area_split()
1029 interval_tree_remove(&area->node, &iopt->area_itree); in iopt_area_split()
1030 rc = iopt_insert_area(iopt, lhs, area->pages, start_iova, in iopt_area_split()
1037 rc = iopt_insert_area(iopt, rhs, area->pages, new_start, in iopt_area_split()
1058 interval_tree_remove(&lhs->node, &iopt->area_itree); in iopt_area_split()
1060 interval_tree_insert(&area->node, &iopt->area_itree); in iopt_area_split()
1069 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas, in iopt_cut_iova() argument
1075 down_write(&iopt->iova_rwsem); in iopt_cut_iova()
1079 area = iopt_area_iter_first(iopt, iovas[i], iovas[i]); in iopt_cut_iova()
1086 up_write(&iopt->iova_rwsem); in iopt_cut_iova()
1090 void iopt_enable_large_pages(struct io_pagetable *iopt) in iopt_enable_large_pages() argument
1094 down_write(&iopt->domains_rwsem); in iopt_enable_large_pages()
1095 down_write(&iopt->iova_rwsem); in iopt_enable_large_pages()
1096 WRITE_ONCE(iopt->disable_large_pages, false); in iopt_enable_large_pages()
1097 rc = iopt_calculate_iova_alignment(iopt); in iopt_enable_large_pages()
1099 up_write(&iopt->iova_rwsem); in iopt_enable_large_pages()
1100 up_write(&iopt->domains_rwsem); in iopt_enable_large_pages()
1103 int iopt_disable_large_pages(struct io_pagetable *iopt) in iopt_disable_large_pages() argument
1107 down_write(&iopt->domains_rwsem); in iopt_disable_large_pages()
1108 down_write(&iopt->iova_rwsem); in iopt_disable_large_pages()
1109 if (iopt->disable_large_pages) in iopt_disable_large_pages()
1113 if (!xa_empty(&iopt->domains) && in iopt_disable_large_pages()
1114 !RB_EMPTY_ROOT(&iopt->area_itree.rb_root)) { in iopt_disable_large_pages()
1119 WRITE_ONCE(iopt->disable_large_pages, true); in iopt_disable_large_pages()
1120 rc = iopt_calculate_iova_alignment(iopt); in iopt_disable_large_pages()
1122 WRITE_ONCE(iopt->disable_large_pages, false); in iopt_disable_large_pages()
1124 up_write(&iopt->iova_rwsem); in iopt_disable_large_pages()
1125 up_write(&iopt->domains_rwsem); in iopt_disable_large_pages()
1129 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access) in iopt_add_access() argument
1133 down_write(&iopt->domains_rwsem); in iopt_add_access()
1134 down_write(&iopt->iova_rwsem); in iopt_add_access()
1135 rc = xa_alloc(&iopt->access_list, &access->iopt_access_list_id, access, in iopt_add_access()
1140 rc = iopt_calculate_iova_alignment(iopt); in iopt_add_access()
1142 xa_erase(&iopt->access_list, access->iopt_access_list_id); in iopt_add_access()
1147 up_write(&iopt->iova_rwsem); in iopt_add_access()
1148 up_write(&iopt->domains_rwsem); in iopt_add_access()
1152 void iopt_remove_access(struct io_pagetable *iopt, in iopt_remove_access() argument
1155 down_write(&iopt->domains_rwsem); in iopt_remove_access()
1156 down_write(&iopt->iova_rwsem); in iopt_remove_access()
1157 WARN_ON(xa_erase(&iopt->access_list, access->iopt_access_list_id) != in iopt_remove_access()
1159 WARN_ON(iopt_calculate_iova_alignment(iopt)); in iopt_remove_access()
1160 up_write(&iopt->iova_rwsem); in iopt_remove_access()
1161 up_write(&iopt->domains_rwsem); in iopt_remove_access()
1165 int iopt_table_enforce_group_resv_regions(struct io_pagetable *iopt, in iopt_table_enforce_group_resv_regions() argument
1177 down_write(&iopt->iova_rwsem); in iopt_table_enforce_group_resv_regions()
1193 rc = iopt_reserve_iova(iopt, resv->start, in iopt_table_enforce_group_resv_regions()
1209 __iopt_remove_reserved_iova(iopt, device); in iopt_table_enforce_group_resv_regions()
1214 up_write(&iopt->iova_rwsem); in iopt_table_enforce_group_resv_regions()