Lines Matching refs:subscriptions
89 mn_itree_is_invalidating(struct mmu_notifier_subscriptions *subscriptions) in mn_itree_is_invalidating() argument
91 lockdep_assert_held(&subscriptions->lock); in mn_itree_is_invalidating()
92 return subscriptions->invalidate_seq & 1; in mn_itree_is_invalidating()
96 mn_itree_inv_start_range(struct mmu_notifier_subscriptions *subscriptions, in mn_itree_inv_start_range() argument
103 spin_lock(&subscriptions->lock); in mn_itree_inv_start_range()
104 subscriptions->active_invalidate_ranges++; in mn_itree_inv_start_range()
105 node = interval_tree_iter_first(&subscriptions->itree, range->start, in mn_itree_inv_start_range()
108 subscriptions->invalidate_seq |= 1; in mn_itree_inv_start_range()
113 *seq = subscriptions->invalidate_seq; in mn_itree_inv_start_range()
114 spin_unlock(&subscriptions->lock); in mn_itree_inv_start_range()
131 static void mn_itree_inv_end(struct mmu_notifier_subscriptions *subscriptions) in mn_itree_inv_end() argument
136 spin_lock(&subscriptions->lock); in mn_itree_inv_end()
137 if (--subscriptions->active_invalidate_ranges || in mn_itree_inv_end()
138 !mn_itree_is_invalidating(subscriptions)) { in mn_itree_inv_end()
139 spin_unlock(&subscriptions->lock); in mn_itree_inv_end()
144 subscriptions->invalidate_seq++; in mn_itree_inv_end()
153 &subscriptions->deferred_list, in mn_itree_inv_end()
157 &subscriptions->itree); in mn_itree_inv_end()
160 &subscriptions->itree); in mn_itree_inv_end()
163 spin_unlock(&subscriptions->lock); in mn_itree_inv_end()
165 wake_up_all(&subscriptions->wq); in mn_itree_inv_end()
190 struct mmu_notifier_subscriptions *subscriptions = in mmu_interval_read_begin() local
234 spin_lock(&subscriptions->lock); in mmu_interval_read_begin()
237 is_invalidating = seq == subscriptions->invalidate_seq; in mmu_interval_read_begin()
238 spin_unlock(&subscriptions->lock); in mmu_interval_read_begin()
250 wait_event(subscriptions->wq, in mmu_interval_read_begin()
251 READ_ONCE(subscriptions->invalidate_seq) != seq); in mmu_interval_read_begin()
263 static void mn_itree_release(struct mmu_notifier_subscriptions *subscriptions, in mn_itree_release() argument
278 mn_itree_inv_start_range(subscriptions, &range, &cur_seq); in mn_itree_release()
286 mn_itree_inv_end(subscriptions); in mn_itree_release()
301 static void mn_hlist_release(struct mmu_notifier_subscriptions *subscriptions, in mn_hlist_release() argument
312 hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist, in mn_hlist_release()
323 spin_lock(&subscriptions->lock); in mn_hlist_release()
324 while (unlikely(!hlist_empty(&subscriptions->list))) { in mn_hlist_release()
325 subscription = hlist_entry(subscriptions->list.first, in mn_hlist_release()
335 spin_unlock(&subscriptions->lock); in mn_hlist_release()
352 struct mmu_notifier_subscriptions *subscriptions = in __mmu_notifier_release() local
355 if (subscriptions->has_itree) in __mmu_notifier_release()
356 mn_itree_release(subscriptions, mm); in __mmu_notifier_release()
358 if (!hlist_empty(&subscriptions->list)) in __mmu_notifier_release()
359 mn_hlist_release(subscriptions, mm); in __mmu_notifier_release()
429 static int mn_itree_invalidate(struct mmu_notifier_subscriptions *subscriptions, in mn_itree_invalidate() argument
436 mn_itree_inv_start_range(subscriptions, range, &cur_seq); in mn_itree_invalidate()
456 mn_itree_inv_end(subscriptions); in mn_itree_invalidate()
461 struct mmu_notifier_subscriptions *subscriptions, in mn_hlist_invalidate_range_start() argument
469 hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist, in mn_hlist_invalidate_range_start()
507 hlist_for_each_entry_rcu(subscription, &subscriptions->list, in mn_hlist_invalidate_range_start()
523 struct mmu_notifier_subscriptions *subscriptions = in __mmu_notifier_invalidate_range_start() local
527 if (subscriptions->has_itree) { in __mmu_notifier_invalidate_range_start()
528 ret = mn_itree_invalidate(subscriptions, range); in __mmu_notifier_invalidate_range_start()
532 if (!hlist_empty(&subscriptions->list)) in __mmu_notifier_invalidate_range_start()
533 return mn_hlist_invalidate_range_start(subscriptions, range); in __mmu_notifier_invalidate_range_start()
538 mn_hlist_invalidate_end(struct mmu_notifier_subscriptions *subscriptions, in mn_hlist_invalidate_end() argument
545 hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist, in mn_hlist_invalidate_end()
561 struct mmu_notifier_subscriptions *subscriptions = in __mmu_notifier_invalidate_range_end() local
565 if (subscriptions->has_itree) in __mmu_notifier_invalidate_range_end()
566 mn_itree_inv_end(subscriptions); in __mmu_notifier_invalidate_range_end()
568 if (!hlist_empty(&subscriptions->list)) in __mmu_notifier_invalidate_range_end()
569 mn_hlist_invalidate_end(subscriptions, range); in __mmu_notifier_invalidate_range_end()
599 struct mmu_notifier_subscriptions *subscriptions = NULL; in __mmu_notifier_register() local
621 subscriptions = kzalloc( in __mmu_notifier_register()
623 if (!subscriptions) in __mmu_notifier_register()
626 INIT_HLIST_HEAD(&subscriptions->list); in __mmu_notifier_register()
627 spin_lock_init(&subscriptions->lock); in __mmu_notifier_register()
628 subscriptions->invalidate_seq = 2; in __mmu_notifier_register()
629 subscriptions->itree = RB_ROOT_CACHED; in __mmu_notifier_register()
630 init_waitqueue_head(&subscriptions->wq); in __mmu_notifier_register()
631 INIT_HLIST_HEAD(&subscriptions->deferred_list); in __mmu_notifier_register()
654 if (subscriptions) in __mmu_notifier_register()
655 smp_store_release(&mm->notifier_subscriptions, subscriptions); in __mmu_notifier_register()
675 kfree(subscriptions); in __mmu_notifier_register()
892 struct mmu_notifier_subscriptions *subscriptions, unsigned long start, in __mmu_interval_notifier_insert() argument
928 spin_lock(&subscriptions->lock); in __mmu_interval_notifier_insert()
929 if (subscriptions->active_invalidate_ranges) { in __mmu_interval_notifier_insert()
930 if (mn_itree_is_invalidating(subscriptions)) in __mmu_interval_notifier_insert()
932 &subscriptions->deferred_list); in __mmu_interval_notifier_insert()
934 subscriptions->invalidate_seq |= 1; in __mmu_interval_notifier_insert()
936 &subscriptions->itree); in __mmu_interval_notifier_insert()
938 interval_sub->invalidate_seq = subscriptions->invalidate_seq; in __mmu_interval_notifier_insert()
940 WARN_ON(mn_itree_is_invalidating(subscriptions)); in __mmu_interval_notifier_insert()
948 subscriptions->invalidate_seq - 1; in __mmu_interval_notifier_insert()
950 &subscriptions->itree); in __mmu_interval_notifier_insert()
952 spin_unlock(&subscriptions->lock); in __mmu_interval_notifier_insert()
977 struct mmu_notifier_subscriptions *subscriptions; in mmu_interval_notifier_insert() local
982 subscriptions = smp_load_acquire(&mm->notifier_subscriptions); in mmu_interval_notifier_insert()
983 if (!subscriptions || !subscriptions->has_itree) { in mmu_interval_notifier_insert()
987 subscriptions = mm->notifier_subscriptions; in mmu_interval_notifier_insert()
989 return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions, in mmu_interval_notifier_insert()
999 struct mmu_notifier_subscriptions *subscriptions = in mmu_interval_notifier_insert_locked() local
1005 if (!subscriptions || !subscriptions->has_itree) { in mmu_interval_notifier_insert_locked()
1009 subscriptions = mm->notifier_subscriptions; in mmu_interval_notifier_insert_locked()
1011 return __mmu_interval_notifier_insert(interval_sub, mm, subscriptions, in mmu_interval_notifier_insert_locked()
1017 mmu_interval_seq_released(struct mmu_notifier_subscriptions *subscriptions, in mmu_interval_seq_released() argument
1022 spin_lock(&subscriptions->lock); in mmu_interval_seq_released()
1023 ret = subscriptions->invalidate_seq != seq; in mmu_interval_seq_released()
1024 spin_unlock(&subscriptions->lock); in mmu_interval_seq_released()
1041 struct mmu_notifier_subscriptions *subscriptions = in mmu_interval_notifier_remove() local
1047 spin_lock(&subscriptions->lock); in mmu_interval_notifier_remove()
1048 if (mn_itree_is_invalidating(subscriptions)) { in mmu_interval_notifier_remove()
1057 &subscriptions->deferred_list); in mmu_interval_notifier_remove()
1058 seq = subscriptions->invalidate_seq; in mmu_interval_notifier_remove()
1063 &subscriptions->itree); in mmu_interval_notifier_remove()
1065 spin_unlock(&subscriptions->lock); in mmu_interval_notifier_remove()
1074 wait_event(subscriptions->wq, in mmu_interval_notifier_remove()
1075 mmu_interval_seq_released(subscriptions, seq)); in mmu_interval_notifier_remove()