Lines Matching refs:pfns
268 unsigned long *pfns; member
279 batch->pfns[0] = 0; in batch_clear()
298 batch->pfns[0] = batch->pfns[batch->end - 1] + in batch_clear_carry()
310 batch->pfns[0] += skip_pfns; in batch_skip_carry()
318 const size_t elmsz = sizeof(*batch->pfns) + sizeof(*batch->npfns); in __batch_init()
321 batch->pfns = temp_kmalloc(&size, backup, backup_len); in __batch_init()
322 if (!batch->pfns) in __batch_init()
327 batch->npfns = (u32 *)(batch->pfns + batch->array_size); in __batch_init()
345 if (batch->pfns != backup) in batch_destroy()
346 kfree(batch->pfns); in batch_destroy()
355 pfn == batch->pfns[batch->end - 1] + batch->npfns[batch->end - 1] && in batch_add_pfn()
364 batch->pfns[batch->end] = pfn; in batch_add_pfn()
499 PFN_PHYS(batch->pfns[cur]) + page_offset, in batch_to_domain()
503 PFN_PHYS(batch->pfns[cur]) + page_offset, in batch_to_domain()
642 pfn_to_page(batch->pfns[cur] + first_page_off), in batch_unpin()
677 copy_data_page(pfn_to_page(batch->pfns[cur] + npage), data, in batch_rw()
947 static int pfn_reader_update_pinned(struct pfn_reader *pfns) in pfn_reader_update_pinned() argument
949 return pfn_reader_user_update_pinned(&pfns->user, pfns->pages); in pfn_reader_update_pinned()
956 static void pfn_reader_unpin(struct pfn_reader *pfns) in pfn_reader_unpin() argument
958 unsigned long last = pfns->batch_end_index - 1; in pfn_reader_unpin()
959 unsigned long start = pfns->batch_start_index; in pfn_reader_unpin()
961 struct iopt_pages *pages = pfns->pages; in pfn_reader_unpin()
970 batch_unpin(&pfns->batch, pages, span.start_hole - start, in pfn_reader_unpin()
976 static int pfn_reader_fill_span(struct pfn_reader *pfns) in pfn_reader_fill_span() argument
978 struct interval_tree_double_span_iter *span = &pfns->span; in pfn_reader_fill_span()
979 unsigned long start_index = pfns->batch_end_index; in pfn_reader_fill_span()
988 batch_from_xarray(&pfns->batch, &pfns->pages->pinned_pfns, in pfn_reader_fill_span()
999 area = iopt_pages_find_domain_area(pfns->pages, start_index); in pfn_reader_fill_span()
1005 &pfns->batch, area->storage_domain, area, start_index, in pfn_reader_fill_span()
1010 if (start_index >= pfns->user.upages_end) { in pfn_reader_fill_span()
1011 rc = pfn_reader_user_pin(&pfns->user, pfns->pages, start_index, in pfn_reader_fill_span()
1017 batch_from_pages(&pfns->batch, in pfn_reader_fill_span()
1018 pfns->user.upages + in pfn_reader_fill_span()
1019 (start_index - pfns->user.upages_start), in pfn_reader_fill_span()
1020 pfns->user.upages_end - start_index); in pfn_reader_fill_span()
1024 static bool pfn_reader_done(struct pfn_reader *pfns) in pfn_reader_done() argument
1026 return pfns->batch_start_index == pfns->last_index + 1; in pfn_reader_done()
1029 static int pfn_reader_next(struct pfn_reader *pfns) in pfn_reader_next() argument
1033 batch_clear(&pfns->batch); in pfn_reader_next()
1034 pfns->batch_start_index = pfns->batch_end_index; in pfn_reader_next()
1036 while (pfns->batch_end_index != pfns->last_index + 1) { in pfn_reader_next()
1037 unsigned int npfns = pfns->batch.total_pfns; in pfn_reader_next()
1040 WARN_ON(interval_tree_double_span_iter_done(&pfns->span))) in pfn_reader_next()
1043 rc = pfn_reader_fill_span(pfns); in pfn_reader_next()
1047 if (WARN_ON(!pfns->batch.total_pfns)) in pfn_reader_next()
1050 pfns->batch_end_index = in pfn_reader_next()
1051 pfns->batch_start_index + pfns->batch.total_pfns; in pfn_reader_next()
1052 if (pfns->batch_end_index == pfns->span.last_used + 1) in pfn_reader_next()
1053 interval_tree_double_span_iter_next(&pfns->span); in pfn_reader_next()
1056 if (npfns == pfns->batch.total_pfns) in pfn_reader_next()
1062 static int pfn_reader_init(struct pfn_reader *pfns, struct iopt_pages *pages, in pfn_reader_init() argument
1069 pfns->pages = pages; in pfn_reader_init()
1070 pfns->batch_start_index = start_index; in pfn_reader_init()
1071 pfns->batch_end_index = start_index; in pfn_reader_init()
1072 pfns->last_index = last_index; in pfn_reader_init()
1073 pfn_reader_user_init(&pfns->user, pages); in pfn_reader_init()
1074 rc = batch_init(&pfns->batch, last_index - start_index + 1); in pfn_reader_init()
1077 interval_tree_double_span_iter_first(&pfns->span, &pages->access_itree, in pfn_reader_init()
1091 static void pfn_reader_release_pins(struct pfn_reader *pfns) in pfn_reader_release_pins() argument
1093 struct iopt_pages *pages = pfns->pages; in pfn_reader_release_pins()
1095 if (pfns->user.upages_end > pfns->batch_end_index) { in pfn_reader_release_pins()
1096 size_t npages = pfns->user.upages_end - pfns->batch_end_index; in pfn_reader_release_pins()
1099 unpin_user_pages(pfns->user.upages + (pfns->batch_end_index - in pfn_reader_release_pins()
1100 pfns->user.upages_start), in pfn_reader_release_pins()
1103 pfns->user.upages_end = pfns->batch_end_index; in pfn_reader_release_pins()
1105 if (pfns->batch_start_index != pfns->batch_end_index) { in pfn_reader_release_pins()
1106 pfn_reader_unpin(pfns); in pfn_reader_release_pins()
1107 pfns->batch_start_index = pfns->batch_end_index; in pfn_reader_release_pins()
1111 static void pfn_reader_destroy(struct pfn_reader *pfns) in pfn_reader_destroy() argument
1113 struct iopt_pages *pages = pfns->pages; in pfn_reader_destroy()
1115 pfn_reader_release_pins(pfns); in pfn_reader_destroy()
1116 pfn_reader_user_destroy(&pfns->user, pfns->pages); in pfn_reader_destroy()
1117 batch_destroy(&pfns->batch, NULL); in pfn_reader_destroy()
1121 static int pfn_reader_first(struct pfn_reader *pfns, struct iopt_pages *pages, in pfn_reader_first() argument
1130 rc = pfn_reader_init(pfns, pages, start_index, last_index); in pfn_reader_first()
1133 rc = pfn_reader_next(pfns); in pfn_reader_first()
1135 pfn_reader_destroy(pfns); in pfn_reader_first()
1348 struct pfn_reader pfns; in iopt_area_fill_domain() local
1353 rc = pfn_reader_first(&pfns, area->pages, iopt_area_index(area), in iopt_area_fill_domain()
1358 while (!pfn_reader_done(&pfns)) { in iopt_area_fill_domain()
1359 done_end_index = pfns.batch_start_index; in iopt_area_fill_domain()
1360 rc = batch_to_domain(&pfns.batch, domain, area, in iopt_area_fill_domain()
1361 pfns.batch_start_index); in iopt_area_fill_domain()
1364 done_end_index = pfns.batch_end_index; in iopt_area_fill_domain()
1366 rc = pfn_reader_next(&pfns); in iopt_area_fill_domain()
1371 rc = pfn_reader_update_pinned(&pfns); in iopt_area_fill_domain()
1377 pfn_reader_release_pins(&pfns); in iopt_area_fill_domain()
1381 pfn_reader_destroy(&pfns); in iopt_area_fill_domain()
1402 struct pfn_reader pfns; in iopt_area_fill_domains() local
1412 rc = pfn_reader_first(&pfns, pages, iopt_area_index(area), in iopt_area_fill_domains()
1417 while (!pfn_reader_done(&pfns)) { in iopt_area_fill_domains()
1418 done_first_end_index = pfns.batch_end_index; in iopt_area_fill_domains()
1419 done_all_end_index = pfns.batch_start_index; in iopt_area_fill_domains()
1421 rc = batch_to_domain(&pfns.batch, domain, area, in iopt_area_fill_domains()
1422 pfns.batch_start_index); in iopt_area_fill_domains()
1428 rc = pfn_reader_next(&pfns); in iopt_area_fill_domains()
1432 rc = pfn_reader_update_pinned(&pfns); in iopt_area_fill_domains()
1441 pfn_reader_release_pins(&pfns); in iopt_area_fill_domains()
1466 pfn_reader_destroy(&pfns); in iopt_area_fill_domains()
1735 struct pfn_reader pfns; in iopt_pages_rw_slow() local
1740 rc = pfn_reader_first(&pfns, pages, start_index, last_index); in iopt_pages_rw_slow()
1744 while (!pfn_reader_done(&pfns)) { in iopt_pages_rw_slow()
1747 done = batch_rw(&pfns.batch, data, offset, length, flags); in iopt_pages_rw_slow()
1751 pfn_reader_unpin(&pfns); in iopt_pages_rw_slow()
1753 rc = pfn_reader_next(&pfns); in iopt_pages_rw_slow()
1760 pfn_reader_destroy(&pfns); in iopt_pages_rw_slow()