Lines Matching refs:offset
62 unsigned long offset);
120 unsigned long offset[SWAP_NR_ORDERS]; member
126 .offset = { SWAP_ENTRY_INVALID },
174 unsigned long offset, int nr_pages) in swap_only_has_cache() argument
176 unsigned char *map = si->swap_map + offset; in swap_only_has_cache()
189 unsigned long offset, int nr_pages, bool *has_cache) in swap_is_last_map() argument
191 unsigned char *map = si->swap_map + offset; in swap_is_last_map()
213 unsigned long offset, unsigned long flags) in __try_to_reclaim_swap() argument
215 swp_entry_t entry = swp_entry(si->type, offset); in __try_to_reclaim_swap()
245 if (offset < swp_offset(entry) || offset >= swp_offset(entry) + nr_pages) { in __try_to_reclaim_swap()
250 offset = swp_offset(entry); in __try_to_reclaim_swap()
263 ci = lock_cluster(si, offset); in __try_to_reclaim_swap()
264 need_reclaim = swap_only_has_cache(si, offset, nr_pages); in __try_to_reclaim_swap()
329 offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) in offset_to_swap_extent() argument
337 if (offset < se->start_page) in offset_to_swap_extent()
339 else if (offset >= se->start_page + se->nr_pages) in offset_to_swap_extent()
353 pgoff_t offset; in swap_folio_sector() local
355 offset = swp_offset(folio->swap); in swap_folio_sector()
356 se = offset_to_swap_extent(sis, offset); in swap_folio_sector()
357 sector = se->start_block + (offset - se->start_page); in swap_folio_sector()
371 pgoff_t offset = start_page - se->start_page; in discard_swap_cluster() local
372 sector_t start_block = se->start_block + offset; in discard_swap_cluster()
373 sector_t nr_blocks = se->nr_pages - offset; in discard_swap_cluster()
431 unsigned long offset) in offset_to_cluster() argument
433 return &si->cluster_info[offset / SWAPFILE_CLUSTER]; in offset_to_cluster()
443 unsigned long offset) in lock_cluster() argument
447 ci = offset_to_cluster(si, offset); in lock_cluster()
684 unsigned long offset = start; in cluster_reclaim_range() local
689 switch (READ_ONCE(map[offset])) { in cluster_reclaim_range()
691 offset++; in cluster_reclaim_range()
694 nr_reclaim = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY); in cluster_reclaim_range()
696 offset += nr_reclaim; in cluster_reclaim_range()
703 } while (offset < end); in cluster_reclaim_range()
710 for (offset = start; offset < end; offset++) in cluster_reclaim_range()
711 if (READ_ONCE(map[offset])) in cluster_reclaim_range()
722 unsigned long offset, end = start + nr_pages; in cluster_scan_range() local
728 for (offset = start; offset < end; offset++) { in cluster_scan_range()
729 switch (READ_ONCE(map[offset])) { in cluster_scan_range()
773 unsigned long offset, in alloc_swap_scan_cluster() argument
778 unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER); in alloc_swap_scan_cluster()
788 for (end -= nr_pages; offset <= end; offset += nr_pages) { in alloc_swap_scan_cluster()
790 if (!cluster_scan_range(si, ci, offset, nr_pages, &need_reclaim)) in alloc_swap_scan_cluster()
793 ret = cluster_reclaim_range(si, ci, offset, offset + nr_pages); in alloc_swap_scan_cluster()
803 offset = start; in alloc_swap_scan_cluster()
808 if (!cluster_alloc_range(si, ci, offset, usage, order)) in alloc_swap_scan_cluster()
810 found = offset; in alloc_swap_scan_cluster()
811 offset += nr_pages; in alloc_swap_scan_cluster()
812 if (ci->count < SWAPFILE_CLUSTER && offset <= end) in alloc_swap_scan_cluster()
813 next = offset; in alloc_swap_scan_cluster()
820 this_cpu_write(percpu_swap_cluster.offset[order], next); in alloc_swap_scan_cluster()
831 unsigned long offset, end; in swap_reclaim_full_clusters() local
840 offset = cluster_offset(si, ci); in swap_reclaim_full_clusters()
841 end = min(si->max, offset + SWAPFILE_CLUSTER); in swap_reclaim_full_clusters()
844 while (offset < end) { in swap_reclaim_full_clusters()
845 if (READ_ONCE(map[offset]) == SWAP_HAS_CACHE) { in swap_reclaim_full_clusters()
847 nr_reclaim = __try_to_reclaim_swap(si, offset, in swap_reclaim_full_clusters()
851 offset += abs(nr_reclaim); in swap_reclaim_full_clusters()
855 offset++; in swap_reclaim_full_clusters()
885 unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; in cluster_alloc_swap_entry() local
897 offset = si->global_cluster->next[order]; in cluster_alloc_swap_entry()
898 if (offset == SWAP_ENTRY_INVALID) in cluster_alloc_swap_entry()
901 ci = lock_cluster(si, offset); in cluster_alloc_swap_entry()
905 offset = cluster_offset(si, ci); in cluster_alloc_swap_entry()
906 found = alloc_swap_scan_cluster(si, ci, offset, in cluster_alloc_swap_entry()
1120 static void swap_range_free(struct swap_info_struct *si, unsigned long offset, in swap_range_free() argument
1123 unsigned long begin = offset; in swap_range_free()
1124 unsigned long end = offset + nr_entries - 1; in swap_range_free()
1133 clear_bit(offset + i, si->zeromap); in swap_range_free()
1134 zswap_invalidate(swp_entry(si->type, offset + i)); in swap_range_free()
1142 while (offset <= end) { in swap_range_free()
1143 arch_swap_invalidate_page(si->type, offset); in swap_range_free()
1145 swap_slot_free_notify(si->bdev, offset); in swap_range_free()
1146 offset++; in swap_range_free()
1184 unsigned int offset, found = SWAP_ENTRY_INVALID; in swap_alloc_fast() local
1191 offset = this_cpu_read(percpu_swap_cluster.offset[order]); in swap_alloc_fast()
1192 if (!si || !offset || !get_swap_device_info(si)) in swap_alloc_fast()
1195 ci = lock_cluster(si, offset); in swap_alloc_fast()
1198 offset = cluster_offset(si, ci); in swap_alloc_fast()
1199 found = alloc_swap_scan_cluster(si, ci, offset, order, SWAP_HAS_CACHE); in swap_alloc_fast()
1215 unsigned long offset; in swap_alloc_slow() local
1226 offset = cluster_alloc_swap_entry(si, order, SWAP_HAS_CACHE); in swap_alloc_slow()
1228 if (offset) { in swap_alloc_slow()
1229 *entry = swp_entry(si->type, offset); in swap_alloc_slow()
1326 unsigned long offset; in _swap_info_get() local
1335 offset = swp_offset(entry); in _swap_info_get()
1336 if (offset >= si->max) in _swap_info_get()
1362 unsigned long offset = swp_offset(entry); in swap_entry_put_locked() local
1366 count = si->swap_map[offset]; in swap_entry_put_locked()
1382 if (swap_count_continued(si, offset, count)) in swap_entry_put_locked()
1392 WRITE_ONCE(si->swap_map[offset], usage); in swap_entry_put_locked()
1441 unsigned long offset; in get_swap_device() local
1450 offset = swp_offset(entry); in get_swap_device()
1451 if (offset >= si->max) in get_swap_device()
1468 unsigned long offset = swp_offset(entry); in swap_entries_put_cache() local
1471 ci = lock_cluster(si, offset); in swap_entries_put_cache()
1472 if (swap_only_has_cache(si, offset, nr)) in swap_entries_put_cache()
1484 unsigned long offset = swp_offset(entry); in swap_entries_put_map() local
1492 count = swap_count(data_race(si->swap_map[offset])); in swap_entries_put_map()
1496 ci = lock_cluster(si, offset); in swap_entries_put_map()
1497 if (!swap_is_last_map(si, offset, nr, &has_cache)) { in swap_entries_put_map()
1504 WRITE_ONCE(si->swap_map[offset + i], SWAP_HAS_CACHE); in swap_entries_put_map()
1510 ci = lock_cluster(si, offset); in swap_entries_put_map()
1531 unsigned long offset = swp_offset(entry); in swap_entries_put_map_nr() local
1534 cluster_rest = SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER; in swap_entries_put_map_nr()
1564 unsigned long offset = swp_offset(entry); in swap_entries_free() local
1565 unsigned char *map = si->swap_map + offset; in swap_entries_free()
1569 VM_BUG_ON(ci != offset_to_cluster(si, offset + nr_pages - 1)); in swap_entries_free()
1580 swap_range_free(si, offset, nr_pages); in swap_entries_free()
1596 unsigned long offset = swp_offset(entry); in swap_free_nr() local
1603 nr = min_t(int, nr_pages, SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER); in swap_free_nr()
1604 swap_entries_put_map(sis, swp_entry(sis->type, offset), nr); in swap_free_nr()
1605 offset += nr; in swap_free_nr()
1628 pgoff_t offset = swp_offset(entry); in __swap_count() local
1630 return swap_count(si->swap_map[offset]); in __swap_count()
1640 pgoff_t offset = swp_offset(entry); in swap_entry_swapped() local
1644 ci = lock_cluster(si, offset); in swap_entry_swapped()
1645 count = swap_count(si->swap_map[offset]); in swap_entry_swapped()
1660 pgoff_t offset; in swp_swapcount() local
1667 offset = swp_offset(entry); in swp_swapcount()
1669 ci = lock_cluster(si, offset); in swp_swapcount()
1671 count = swap_count(si->swap_map[offset]); in swp_swapcount()
1678 page = vmalloc_to_page(si->swap_map + offset); in swp_swapcount()
1679 offset &= ~PAGE_MASK; in swp_swapcount()
1685 tmp_count = map[offset]; in swp_swapcount()
1703 unsigned long offset = round_down(roffset, nr_pages); in swap_page_trans_huge_swapped() local
1707 ci = lock_cluster(si, offset); in swap_page_trans_huge_swapped()
1714 if (swap_count(map[offset + i])) { in swap_page_trans_huge_swapped()
1805 unsigned long offset; in free_swap_and_cache_nr() local
1829 for (offset = start_offset; offset < end_offset; offset += nr) { in free_swap_and_cache_nr()
1831 if (READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { in free_swap_and_cache_nr()
1840 nr = __try_to_reclaim_swap(si, offset, in free_swap_and_cache_nr()
1846 nr = ALIGN(offset + 1, nr) - offset; in free_swap_and_cache_nr()
1859 unsigned long offset; in get_swap_page_of_type() local
1868 offset = cluster_alloc_swap_entry(si, 0, 1); in get_swap_page_of_type()
1869 if (offset) { in get_swap_page_of_type()
1870 entry = swp_entry(si->type, offset); in get_swap_page_of_type()
1888 int swap_type_of(dev_t device, sector_t offset) in swap_type_of() argument
1905 if (se->start_block == offset) { in swap_type_of()
1937 sector_t swapdev_block(int type, pgoff_t offset) in swapdev_block() argument
1944 se = offset_to_swap_extent(si, offset); in swapdev_block()
1945 return se->start_block + (offset - se->start_page); in swapdev_block()
2096 unsigned long offset; in unuse_pte_range() local
2117 offset = swp_offset(entry); in unuse_pte_range()
2134 swp_count = READ_ONCE(si->swap_map[offset]); in unuse_pte_range()
2640 unsigned long offset; in wait_for_allocation() local
2646 for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) { in wait_for_allocation()
2647 ci = lock_cluster(si, offset); in wait_for_allocation()
3551 unsigned long offset; in __swap_duplicate() local
3562 offset = swp_offset(entry); in __swap_duplicate()
3563 VM_WARN_ON(nr > SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER); in __swap_duplicate()
3565 ci = lock_cluster(si, offset); in __swap_duplicate()
3569 count = si->swap_map[offset + i]; in __swap_duplicate()
3597 count = si->swap_map[offset + i]; in __swap_duplicate()
3605 else if (swap_count_continued(si, offset + i, count)) in __swap_duplicate()
3616 WRITE_ONCE(si->swap_map[offset + i], count | has_cache); in __swap_duplicate()
3698 pgoff_t offset; in add_swap_count_continuation() local
3717 offset = swp_offset(entry); in add_swap_count_continuation()
3719 ci = lock_cluster(si, offset); in add_swap_count_continuation()
3721 count = swap_count(si->swap_map[offset]); in add_swap_count_continuation()
3737 head = vmalloc_to_page(si->swap_map + offset); in add_swap_count_continuation()
3738 offset &= ~PAGE_MASK; in add_swap_count_continuation()
3762 map = kmap_local_page(list_page) + offset; in add_swap_count_continuation()
3797 pgoff_t offset, unsigned char count) in swap_count_continued() argument
3804 head = vmalloc_to_page(si->swap_map + offset); in swap_count_continued()
3811 offset &= ~PAGE_MASK; in swap_count_continued()
3813 map = kmap_local_page(page) + offset; in swap_count_continued()
3826 map = kmap_local_page(page) + offset; in swap_count_continued()
3835 map = kmap_local_page(page) + offset; in swap_count_continued()
3841 map = kmap_local_page(page) + offset; in swap_count_continued()
3856 map = kmap_local_page(page) + offset; in swap_count_continued()
3864 map = kmap_local_page(page) + offset; in swap_count_continued()
3882 pgoff_t offset; in free_swap_count_continuations() local
3884 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { in free_swap_count_continuations()
3886 head = vmalloc_to_page(si->swap_map + offset); in free_swap_count_continuations()