Lines Matching refs:stable_node
155 struct stable_node { struct
209 struct stable_node *head;
310 stable_node_cache = KSM_KMEM_CACHE(stable_node, 0); in ksm_slab_init()
336 static __always_inline bool is_stable_node_chain(struct stable_node *chain) in is_stable_node_chain()
341 static __always_inline bool is_stable_node_dup(struct stable_node *dup) in is_stable_node_dup()
346 static inline void stable_node_chain_add_dup(struct stable_node *dup, in stable_node_chain_add_dup()
347 struct stable_node *chain) in stable_node_chain_add_dup()
356 static inline void __stable_node_dup_del(struct stable_node *dup) in __stable_node_dup_del()
363 static inline void stable_node_dup_del(struct stable_node *dup) in stable_node_dup_del()
393 static inline struct stable_node *alloc_stable_node(void) in alloc_stable_node()
403 static inline void free_stable_node(struct stable_node *stable_node) in free_stable_node() argument
405 VM_BUG_ON(stable_node->rmap_hlist_len && in free_stable_node()
406 !is_stable_node_chain(stable_node)); in free_stable_node()
407 kmem_cache_free(stable_node_cache, stable_node); in free_stable_node()
587 static struct stable_node *alloc_stable_node_chain(struct stable_node *dup, in alloc_stable_node_chain()
590 struct stable_node *chain = alloc_stable_node(); in alloc_stable_node_chain()
620 static inline void free_stable_node_chain(struct stable_node *chain, in free_stable_node_chain()
628 static void remove_node_from_stable_tree(struct stable_node *stable_node) in remove_node_from_stable_tree() argument
633 BUG_ON(stable_node->rmap_hlist_len < 0); in remove_node_from_stable_tree()
635 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in remove_node_from_stable_tree()
640 VM_BUG_ON(stable_node->rmap_hlist_len <= 0); in remove_node_from_stable_tree()
641 stable_node->rmap_hlist_len--; in remove_node_from_stable_tree()
657 if (stable_node->head == &migrate_nodes) in remove_node_from_stable_tree()
658 list_del(&stable_node->list); in remove_node_from_stable_tree()
660 stable_node_dup_del(stable_node); in remove_node_from_stable_tree()
661 free_stable_node(stable_node); in remove_node_from_stable_tree()
689 static struct page *get_ksm_page(struct stable_node *stable_node, in get_ksm_page() argument
696 expected_mapping = (void *)((unsigned long)stable_node | in get_ksm_page()
699 kpfn = READ_ONCE(stable_node->kpfn); /* Address dependency. */ in get_ksm_page()
758 if (READ_ONCE(stable_node->kpfn) != kpfn) in get_ksm_page()
760 remove_node_from_stable_tree(stable_node); in get_ksm_page()
771 struct stable_node *stable_node; in remove_rmap_item_from_tree() local
774 stable_node = rmap_item->head; in remove_rmap_item_from_tree()
775 page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); in remove_rmap_item_from_tree()
783 if (!hlist_empty(&stable_node->hlist)) in remove_rmap_item_from_tree()
787 VM_BUG_ON(stable_node->rmap_hlist_len <= 0); in remove_rmap_item_from_tree()
788 stable_node->rmap_hlist_len--; in remove_rmap_item_from_tree()
855 static inline struct stable_node *folio_stable_node(struct folio *folio) in folio_stable_node()
860 static inline struct stable_node *page_stable_node(struct page *page) in page_stable_node()
866 struct stable_node *stable_node) in set_page_stable_node() argument
868 page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM); in set_page_stable_node()
875 static int remove_stable_node(struct stable_node *stable_node) in remove_stable_node() argument
880 page = get_ksm_page(stable_node, GET_KSM_PAGE_LOCK); in remove_stable_node()
904 remove_node_from_stable_tree(stable_node); in remove_stable_node()
913 static int remove_stable_node_chain(struct stable_node *stable_node, in remove_stable_node_chain() argument
916 struct stable_node *dup; in remove_stable_node_chain()
919 if (!is_stable_node_chain(stable_node)) { in remove_stable_node_chain()
920 VM_BUG_ON(is_stable_node_dup(stable_node)); in remove_stable_node_chain()
921 if (remove_stable_node(stable_node)) in remove_stable_node_chain()
928 &stable_node->hlist, hlist_dup) { in remove_stable_node_chain()
933 BUG_ON(!hlist_empty(&stable_node->hlist)); in remove_stable_node_chain()
934 free_stable_node_chain(stable_node, root); in remove_stable_node_chain()
940 struct stable_node *stable_node, *next; in remove_all_stable_nodes() local
946 stable_node = rb_entry(root_stable_tree[nid].rb_node, in remove_all_stable_nodes()
947 struct stable_node, node); in remove_all_stable_nodes()
948 if (remove_stable_node_chain(stable_node, in remove_all_stable_nodes()
956 list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) { in remove_all_stable_nodes()
957 if (remove_stable_node(stable_node)) in remove_all_stable_nodes()
1335 bool __is_page_sharing_candidate(struct stable_node *stable_node, int offset) in __is_page_sharing_candidate() argument
1337 VM_BUG_ON(stable_node->rmap_hlist_len < 0); in __is_page_sharing_candidate()
1344 return stable_node->rmap_hlist_len && in __is_page_sharing_candidate()
1345 stable_node->rmap_hlist_len + offset < ksm_max_page_sharing; in __is_page_sharing_candidate()
1349 bool is_page_sharing_candidate(struct stable_node *stable_node) in is_page_sharing_candidate() argument
1351 return __is_page_sharing_candidate(stable_node, 0); in is_page_sharing_candidate()
1354 static struct page *stable_node_dup(struct stable_node **_stable_node_dup, in stable_node_dup()
1355 struct stable_node **_stable_node, in stable_node_dup()
1359 struct stable_node *dup, *found = NULL, *stable_node = *_stable_node; in stable_node_dup() local
1366 time_before(jiffies, stable_node->chain_prune_time + in stable_node_dup()
1371 stable_node->chain_prune_time = jiffies; in stable_node_dup()
1374 &stable_node->hlist, hlist_dup) { in stable_node_dup()
1422 BUG_ON(stable_node->hlist.first->next); in stable_node_dup()
1428 rb_replace_node(&stable_node->node, &found->node, in stable_node_dup()
1430 free_stable_node(stable_node); in stable_node_dup()
1445 stable_node = NULL; in stable_node_dup()
1446 } else if (stable_node->hlist.first != &found->hlist_dup && in stable_node_dup()
1465 &stable_node->hlist); in stable_node_dup()
1473 static struct stable_node *stable_node_dup_any(struct stable_node *stable_node, in stable_node_dup_any() argument
1476 if (!is_stable_node_chain(stable_node)) in stable_node_dup_any()
1477 return stable_node; in stable_node_dup_any()
1478 if (hlist_empty(&stable_node->hlist)) { in stable_node_dup_any()
1479 free_stable_node_chain(stable_node, root); in stable_node_dup_any()
1482 return hlist_entry(stable_node->hlist.first, in stable_node_dup_any()
1483 typeof(*stable_node), hlist_dup); in stable_node_dup_any()
1500 static struct page *__stable_node_chain(struct stable_node **_stable_node_dup, in __stable_node_chain()
1501 struct stable_node **_stable_node, in __stable_node_chain()
1505 struct stable_node *stable_node = *_stable_node; in __stable_node_chain() local
1506 if (!is_stable_node_chain(stable_node)) { in __stable_node_chain()
1507 if (is_page_sharing_candidate(stable_node)) { in __stable_node_chain()
1508 *_stable_node_dup = stable_node; in __stable_node_chain()
1509 return get_ksm_page(stable_node, GET_KSM_PAGE_NOLOCK); in __stable_node_chain()
1522 static __always_inline struct page *chain_prune(struct stable_node **s_n_d, in chain_prune()
1523 struct stable_node **s_n, in chain_prune()
1529 static __always_inline struct page *chain(struct stable_node **s_n_d, in chain()
1530 struct stable_node *s_n, in chain()
1533 struct stable_node *old_stable_node = s_n; in chain()
1557 struct stable_node *stable_node, *stable_node_dup, *stable_node_any; in stable_tree_search() local
1558 struct stable_node *page_node; in stable_tree_search()
1578 stable_node = rb_entry(*new, struct stable_node, node); in stable_tree_search()
1580 tree_page = chain_prune(&stable_node_dup, &stable_node, root); in stable_tree_search()
1599 stable_node_any = stable_node_dup_any(stable_node, in stable_tree_search()
1721 if (stable_node_dup == stable_node) { in stable_tree_search()
1741 VM_BUG_ON(!is_stable_node_chain(stable_node)); in stable_tree_search()
1747 stable_node_chain_add_dup(page_node, stable_node); in stable_tree_search()
1772 if (stable_node_dup == stable_node) { in stable_tree_search()
1775 stable_node = alloc_stable_node_chain(stable_node_dup, in stable_tree_search()
1777 if (!stable_node) in stable_tree_search()
1790 stable_node_chain_add_dup(page_node, stable_node); in stable_tree_search()
1801 static struct stable_node *stable_tree_insert(struct page *kpage) in stable_tree_insert()
1808 struct stable_node *stable_node, *stable_node_dup, *stable_node_any; in stable_tree_insert() local
1823 stable_node = rb_entry(*new, struct stable_node, node); in stable_tree_insert()
1825 tree_page = chain(&stable_node_dup, stable_node, root); in stable_tree_insert()
1832 stable_node_any = stable_node_dup_any(stable_node, in stable_tree_insert()
1891 if (!is_stable_node_chain(stable_node)) { in stable_tree_insert()
1892 struct stable_node *orig = stable_node; in stable_tree_insert()
1894 stable_node = alloc_stable_node_chain(orig, root); in stable_tree_insert()
1895 if (!stable_node) { in stable_tree_insert()
1900 stable_node_chain_add_dup(stable_node_dup, stable_node); in stable_tree_insert()
1993 struct stable_node *stable_node, in stable_tree_append() argument
2006 BUG_ON(stable_node->rmap_hlist_len < 0); in stable_tree_append()
2008 stable_node->rmap_hlist_len++; in stable_tree_append()
2011 WARN_ON_ONCE(stable_node->rmap_hlist_len > in stable_tree_append()
2014 rmap_item->head = stable_node; in stable_tree_append()
2016 hlist_add_head(&rmap_item->hlist, &stable_node->hlist); in stable_tree_append()
2038 struct stable_node *stable_node; in cmp_and_merge_page() local
2044 stable_node = page_stable_node(page); in cmp_and_merge_page()
2045 if (stable_node) { in cmp_and_merge_page()
2046 if (stable_node->head != &migrate_nodes && in cmp_and_merge_page()
2047 get_kpfn_nid(READ_ONCE(stable_node->kpfn)) != in cmp_and_merge_page()
2048 NUMA(stable_node->nid)) { in cmp_and_merge_page()
2049 stable_node_dup_del(stable_node); in cmp_and_merge_page()
2050 stable_node->head = &migrate_nodes; in cmp_and_merge_page()
2051 list_add(&stable_node->list, stable_node->head); in cmp_and_merge_page()
2053 if (stable_node->head != &migrate_nodes && in cmp_and_merge_page()
2054 rmap_item->head == stable_node) in cmp_and_merge_page()
2060 if (!is_page_sharing_candidate(stable_node)) in cmp_and_merge_page()
2066 if (kpage == page && rmap_item->head == stable_node) { in cmp_and_merge_page()
2157 stable_node = stable_tree_insert(kpage); in cmp_and_merge_page()
2158 if (stable_node) { in cmp_and_merge_page()
2159 stable_tree_append(tree_rmap_item, stable_node, in cmp_and_merge_page()
2161 stable_tree_append(rmap_item, stable_node, in cmp_and_merge_page()
2172 if (!stable_node) { in cmp_and_merge_page()
2254 struct stable_node *stable_node, *next; in scan_get_next_rmap_item() local
2257 list_for_each_entry_safe(stable_node, next, in scan_get_next_rmap_item()
2259 page = get_ksm_page(stable_node, in scan_get_next_rmap_item()
2604 struct stable_node *stable_node; in rmap_walk_ksm() local
2616 stable_node = page_stable_node(page); in rmap_walk_ksm()
2617 if (!stable_node) in rmap_walk_ksm()
2620 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) { in rmap_walk_ksm()
2669 struct stable_node *stable_node; in folio_migrate_ksm() local
2675 stable_node = folio_stable_node(folio); in folio_migrate_ksm()
2676 if (stable_node) { in folio_migrate_ksm()
2677 VM_BUG_ON_FOLIO(stable_node->kpfn != folio_pfn(folio), folio); in folio_migrate_ksm()
2678 stable_node->kpfn = folio_pfn(newfolio); in folio_migrate_ksm()
2702 static bool stable_node_dup_remove_range(struct stable_node *stable_node, in stable_node_dup_remove_range() argument
2706 if (stable_node->kpfn >= start_pfn && in stable_node_dup_remove_range()
2707 stable_node->kpfn < end_pfn) { in stable_node_dup_remove_range()
2712 remove_node_from_stable_tree(stable_node); in stable_node_dup_remove_range()
2718 static bool stable_node_chain_remove_range(struct stable_node *stable_node, in stable_node_chain_remove_range() argument
2723 struct stable_node *dup; in stable_node_chain_remove_range()
2726 if (!is_stable_node_chain(stable_node)) { in stable_node_chain_remove_range()
2727 VM_BUG_ON(is_stable_node_dup(stable_node)); in stable_node_chain_remove_range()
2728 return stable_node_dup_remove_range(stable_node, start_pfn, in stable_node_chain_remove_range()
2733 &stable_node->hlist, hlist_dup) { in stable_node_chain_remove_range()
2737 if (hlist_empty(&stable_node->hlist)) { in stable_node_chain_remove_range()
2738 free_stable_node_chain(stable_node, root); in stable_node_chain_remove_range()
2747 struct stable_node *stable_node, *next; in ksm_check_stable_tree() local
2754 stable_node = rb_entry(node, struct stable_node, node); in ksm_check_stable_tree()
2755 if (stable_node_chain_remove_range(stable_node, in ksm_check_stable_tree()
2765 list_for_each_entry_safe(stable_node, next, &migrate_nodes, list) { in ksm_check_stable_tree()
2766 if (stable_node->kpfn >= start_pfn && in ksm_check_stable_tree()
2767 stable_node->kpfn < end_pfn) in ksm_check_stable_tree()
2768 remove_node_from_stable_tree(stable_node); in ksm_check_stable_tree()