Lines Matching refs:area
24 struct iopt_area *area; member
39 iter->area = iopt_area_iter_first(iopt, iova, iova); in iopt_area_contig_init()
40 if (!iter->area) in iopt_area_contig_init()
42 if (!iter->area->pages) { in iopt_area_contig_init()
43 iter->area = NULL; in iopt_area_contig_init()
46 return iter->area; in iopt_area_contig_init()
53 if (!iter->area) in iopt_area_contig_next()
55 last_iova = iopt_area_last_iova(iter->area); in iopt_area_contig_next()
60 iter->area = iopt_area_iter_next(iter->area, iter->cur_iova, in iopt_area_contig_next()
62 if (!iter->area) in iopt_area_contig_next()
64 if (iter->cur_iova != iopt_area_iova(iter->area) || in iopt_area_contig_next()
65 !iter->area->pages) { in iopt_area_contig_next()
66 iter->area = NULL; in iopt_area_contig_next()
69 return iter->area; in iopt_area_contig_next()
189 static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, in iopt_insert_area() argument
199 area->iommu_prot = iommu_prot; in iopt_insert_area()
200 area->page_offset = start_byte % PAGE_SIZE; in iopt_insert_area()
201 if (area->page_offset & (iopt->iova_alignment - 1)) in iopt_insert_area()
204 area->node.start = iova; in iopt_insert_area()
205 if (check_add_overflow(iova, length - 1, &area->node.last)) in iopt_insert_area()
208 area->pages_node.start = start_byte / PAGE_SIZE; in iopt_insert_area()
209 if (check_add_overflow(start_byte, length - 1, &area->pages_node.last)) in iopt_insert_area()
211 area->pages_node.last = area->pages_node.last / PAGE_SIZE; in iopt_insert_area()
212 if (WARN_ON(area->pages_node.last >= pages->npages)) in iopt_insert_area()
219 area->iopt = iopt; in iopt_insert_area()
220 interval_tree_insert(&area->node, &iopt->area_itree); in iopt_insert_area()
234 elm->area = kzalloc(sizeof(*elm->area), GFP_KERNEL_ACCOUNT); in iopt_alloc_area_pages()
235 if (!elm->area) in iopt_alloc_area_pages()
271 rc = iopt_insert_area(iopt, elm->area, elm->pages, iova, in iopt_alloc_area_pages()
283 static void iopt_abort_area(struct iopt_area *area) in iopt_abort_area() argument
286 WARN_ON(area->pages); in iopt_abort_area()
287 if (area->iopt) { in iopt_abort_area()
288 down_write(&area->iopt->iova_rwsem); in iopt_abort_area()
289 interval_tree_remove(&area->node, &area->iopt->area_itree); in iopt_abort_area()
290 up_write(&area->iopt->iova_rwsem); in iopt_abort_area()
292 kfree(area); in iopt_abort_area()
301 if (elm->area) in iopt_free_pages_list()
302 iopt_abort_area(elm->area); in iopt_free_pages_list()
317 rc = iopt_area_fill_domains(elm->area, elm->pages); in iopt_fill_domains_pages()
327 iopt_area_unfill_domains(undo_elm->area, undo_elm->pages); in iopt_fill_domains_pages()
356 elm->area->pages = elm->pages; in iopt_map_pages()
358 elm->area = NULL; in iopt_map_pages()
406 if (elm.area) in iopt_map_user_pages()
407 iopt_abort_area(elm.area); in iopt_map_user_pages()
420 struct iopt_area *area; in iopt_get_pages() local
429 iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) { in iopt_get_pages()
431 unsigned long last = min(last_iova, iopt_area_last_iova(area)); in iopt_get_pages()
438 elm->start_byte = iopt_area_start_byte(area, iter.cur_iova); in iopt_get_pages()
439 elm->pages = area->pages; in iopt_get_pages()
459 struct iopt_area *area; in iopt_unmap_iova_range() local
471 while ((area = iopt_area_iter_first(iopt, start, last))) { in iopt_unmap_iova_range()
472 unsigned long area_last = iopt_area_last_iova(area); in iopt_unmap_iova_range()
473 unsigned long area_first = iopt_area_iova(area); in iopt_unmap_iova_range()
477 if (!area->pages) { in iopt_unmap_iova_range()
492 if (area->num_accesses) { in iopt_unmap_iova_range()
494 area->prevent_access = true; in iopt_unmap_iova_range()
498 iopt_area_length(area)); in iopt_unmap_iova_range()
499 if (WARN_ON(READ_ONCE(area->num_accesses))) in iopt_unmap_iova_range()
504 pages = area->pages; in iopt_unmap_iova_range()
505 area->pages = NULL; in iopt_unmap_iova_range()
508 iopt_area_unfill_domains(area, pages); in iopt_unmap_iova_range()
509 iopt_abort_area(area); in iopt_unmap_iova_range()
679 struct iopt_area *area; in iopt_unfill_domain() local
693 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_unfill_domain()
694 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { in iopt_unfill_domain()
695 struct iopt_pages *pages = area->pages; in iopt_unfill_domain()
702 WARN_ON(!area->storage_domain); in iopt_unfill_domain()
703 if (area->storage_domain == domain) in iopt_unfill_domain()
704 area->storage_domain = storage_domain; in iopt_unfill_domain()
707 iopt_area_unmap_domain(area, domain); in iopt_unfill_domain()
712 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_unfill_domain()
713 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { in iopt_unfill_domain()
714 struct iopt_pages *pages = area->pages; in iopt_unfill_domain()
720 interval_tree_remove(&area->pages_node, &pages->domains_itree); in iopt_unfill_domain()
721 WARN_ON(area->storage_domain != domain); in iopt_unfill_domain()
722 area->storage_domain = NULL; in iopt_unfill_domain()
723 iopt_area_unfill_domain(area, pages, domain); in iopt_unfill_domain()
740 struct iopt_area *area; in iopt_fill_domain() local
746 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_fill_domain()
747 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { in iopt_fill_domain()
748 struct iopt_pages *pages = area->pages; in iopt_fill_domain()
754 rc = iopt_area_fill_domain(area, domain); in iopt_fill_domain()
759 if (!area->storage_domain) { in iopt_fill_domain()
761 area->storage_domain = domain; in iopt_fill_domain()
762 interval_tree_insert(&area->pages_node, in iopt_fill_domain()
770 end_area = area; in iopt_fill_domain()
771 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_fill_domain()
772 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { in iopt_fill_domain()
773 struct iopt_pages *pages = area->pages; in iopt_fill_domain()
775 if (area == end_area) in iopt_fill_domain()
781 interval_tree_remove(&area->pages_node, in iopt_fill_domain()
783 area->storage_domain = NULL; in iopt_fill_domain()
785 iopt_area_unfill_domain(area, pages, domain); in iopt_fill_domain()
796 struct iopt_area *area; in iopt_check_iova_alignment() local
801 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; in iopt_check_iova_alignment()
802 area = iopt_area_iter_next(area, 0, ULONG_MAX)) in iopt_check_iova_alignment()
803 if ((iopt_area_iova(area) & align_mask) || in iopt_check_iova_alignment()
804 (iopt_area_length(area) & align_mask) || in iopt_check_iova_alignment()
805 (area->page_offset & align_mask)) in iopt_check_iova_alignment()
976 static int iopt_area_split(struct iopt_area *area, unsigned long iova) in iopt_area_split() argument
978 unsigned long alignment = area->iopt->iova_alignment; in iopt_area_split()
979 unsigned long last_iova = iopt_area_last_iova(area); in iopt_area_split()
980 unsigned long start_iova = iopt_area_iova(area); in iopt_area_split()
982 struct io_pagetable *iopt = area->iopt; in iopt_area_split()
983 struct iopt_pages *pages = area->pages; in iopt_area_split()
993 if (!pages || area->prevent_access) in iopt_area_split()
997 iopt_area_start_byte(area, new_start) & (alignment - 1)) in iopt_area_split()
1000 lhs = kzalloc(sizeof(*area), GFP_KERNEL_ACCOUNT); in iopt_area_split()
1004 rhs = kzalloc(sizeof(*area), GFP_KERNEL_ACCOUNT); in iopt_area_split()
1015 if (area->num_accesses) { in iopt_area_split()
1024 if (area->storage_domain && !iopt->disable_large_pages) { in iopt_area_split()
1029 interval_tree_remove(&area->node, &iopt->area_itree); in iopt_area_split()
1030 rc = iopt_insert_area(iopt, lhs, area->pages, start_iova, in iopt_area_split()
1031 iopt_area_start_byte(area, start_iova), in iopt_area_split()
1033 area->iommu_prot); in iopt_area_split()
1037 rc = iopt_insert_area(iopt, rhs, area->pages, new_start, in iopt_area_split()
1038 iopt_area_start_byte(area, new_start), in iopt_area_split()
1039 last_iova - new_start + 1, area->iommu_prot); in iopt_area_split()
1043 lhs->storage_domain = area->storage_domain; in iopt_area_split()
1044 lhs->pages = area->pages; in iopt_area_split()
1045 rhs->storage_domain = area->storage_domain; in iopt_area_split()
1046 rhs->pages = area->pages; in iopt_area_split()
1048 kfree(area); in iopt_area_split()
1060 interval_tree_insert(&area->node, &iopt->area_itree); in iopt_area_split()
1077 struct iopt_area *area; in iopt_cut_iova() local
1079 area = iopt_area_iter_first(iopt, iovas[i], iovas[i]); in iopt_cut_iova()
1080 if (!area) in iopt_cut_iova()
1082 rc = iopt_area_split(area, iovas[i]); in iopt_cut_iova()