Lines Matching refs:mm
70 void *mm; member
153 wake_up_var(fph->mm); in futex_private_hash_put()
190 fph = rcu_dereference(key->private.mm->futex_phash); in __futex_hash_private()
235 static bool __futex_pivot_hash(struct mm_struct *mm, in __futex_pivot_hash() argument
240 WARN_ON_ONCE(mm->futex_phash_new); in __futex_pivot_hash()
242 fph = rcu_dereference_protected(mm->futex_phash, in __futex_pivot_hash()
243 lockdep_is_held(&mm->futex_hash_lock)); in __futex_pivot_hash()
246 mm->futex_phash_new = new; in __futex_pivot_hash()
254 mm->futex_batches = get_state_synchronize_rcu(); in __futex_pivot_hash()
255 rcu_assign_pointer(mm->futex_phash, new); in __futex_pivot_hash()
261 static void futex_pivot_hash(struct mm_struct *mm) in futex_pivot_hash() argument
263 scoped_guard(mutex, &mm->futex_hash_lock) { in futex_pivot_hash()
266 fph = mm->futex_phash_new; in futex_pivot_hash()
268 mm->futex_phash_new = NULL; in futex_pivot_hash()
269 __futex_pivot_hash(mm, fph); in futex_pivot_hash()
276 struct mm_struct *mm = current->mm; in futex_private_hash() local
291 fph = rcu_dereference(mm->futex_phash); in futex_private_hash()
298 futex_pivot_hash(mm); in futex_private_hash()
315 futex_pivot_hash(key->private.mm); in futex_hash()
336 static int __futex_key_to_node(struct mm_struct *mm, unsigned long addr) in __futex_key_to_node() argument
338 struct vm_area_struct *vma = vma_lookup(mm, addr); in __futex_key_to_node()
365 static int futex_key_to_node_opt(struct mm_struct *mm, unsigned long addr) in futex_key_to_node_opt() argument
371 if (!mmap_lock_speculate_try_begin(mm, &seq)) in futex_key_to_node_opt()
374 node = __futex_key_to_node(mm, addr); in futex_key_to_node_opt()
376 if (mmap_lock_speculate_retry(mm, seq)) in futex_key_to_node_opt()
382 static int futex_mpol(struct mm_struct *mm, unsigned long addr) in futex_mpol() argument
386 node = futex_key_to_node_opt(mm, addr); in futex_mpol()
390 guard(mmap_read_lock)(mm); in futex_mpol()
391 return __futex_key_to_node(mm, addr); in futex_mpol()
396 static int futex_mpol(struct mm_struct *mm, unsigned long addr) in futex_mpol() argument
552 struct mm_struct *mm = current->mm; in get_futex_key() local
593 node = futex_mpol(mm, address); in get_futex_key()
625 key->private.mm = mm; in get_futex_key()
627 key->private.mm = NULL; in get_futex_key()
727 key->private.mm = mm; in get_futex_key()
786 struct mm_struct *mm = current->mm; in fault_in_user_writeable() local
789 mmap_read_lock(mm); in fault_in_user_writeable()
790 ret = fixup_user_fault(mm, (unsigned long)uaddr, in fault_in_user_writeable()
792 mmap_read_unlock(mm); in fault_in_user_writeable()
1546 struct mm_struct *mm = fph->mm; in __futex_ref_atomic_begin() local
1553 WARN_ON_ONCE(atomic_long_read(&mm->futex_atomic) != 0); in __futex_ref_atomic_begin()
1560 atomic_long_set(&mm->futex_atomic, LONG_MAX); in __futex_ref_atomic_begin()
1563 call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); in __futex_ref_atomic_begin()
1568 struct mm_struct *mm = fph->mm; in __futex_ref_atomic_end() local
1584 unsigned int *ptr = per_cpu_ptr(mm->futex_ref, cpu); in __futex_ref_atomic_end()
1592 this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ in __futex_ref_atomic_end()
1600 ret = atomic_long_add_return(count - LONG_MAX - 1, &mm->futex_atomic); in __futex_ref_atomic_end()
1602 wake_up_var(mm); in __futex_ref_atomic_end()
1605 mmput_async(mm); in __futex_ref_atomic_end()
1610 struct mm_struct *mm = container_of(head, struct mm_struct, futex_rcu); in futex_ref_rcu() local
1611 struct futex_private_hash *fph = rcu_dereference_raw(mm->futex_phash); in futex_ref_rcu()
1635 struct mm_struct *mm = fph->mm; in futex_ref_drop() local
1640 WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) != fph); in futex_ref_drop()
1645 mmget(mm); in futex_ref_drop()
1668 if (poll_state_synchronize_rcu(mm->futex_batches)) { in futex_ref_drop()
1676 call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); in futex_ref_drop()
1681 struct mm_struct *mm = fph->mm; in futex_ref_get() local
1686 this_cpu_inc(*mm->futex_ref); in futex_ref_get()
1690 return atomic_long_inc_not_zero(&mm->futex_atomic); in futex_ref_get()
1695 struct mm_struct *mm = fph->mm; in futex_ref_put() local
1700 this_cpu_dec(*mm->futex_ref); in futex_ref_put()
1704 return atomic_long_dec_and_test(&mm->futex_atomic); in futex_ref_put()
1709 struct mm_struct *mm = fph->mm; in futex_ref_is_dead() local
1716 return atomic_long_read(&mm->futex_atomic) == 0; in futex_ref_is_dead()
1719 int futex_mm_init(struct mm_struct *mm) in futex_mm_init() argument
1721 mutex_init(&mm->futex_hash_lock); in futex_mm_init()
1722 RCU_INIT_POINTER(mm->futex_phash, NULL); in futex_mm_init()
1723 mm->futex_phash_new = NULL; in futex_mm_init()
1725 atomic_long_set(&mm->futex_atomic, 0); in futex_mm_init()
1726 mm->futex_batches = get_state_synchronize_rcu(); in futex_mm_init()
1727 mm->futex_ref = alloc_percpu(unsigned int); in futex_mm_init()
1728 if (!mm->futex_ref) in futex_mm_init()
1730 this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ in futex_mm_init()
1734 void futex_hash_free(struct mm_struct *mm) in futex_hash_free() argument
1738 free_percpu(mm->futex_ref); in futex_hash_free()
1739 kvfree(mm->futex_phash_new); in futex_hash_free()
1740 fph = rcu_dereference_raw(mm->futex_phash); in futex_hash_free()
1745 static bool futex_pivot_pending(struct mm_struct *mm) in futex_pivot_pending() argument
1751 if (!mm->futex_phash_new) in futex_pivot_pending()
1754 fph = rcu_dereference(mm->futex_phash); in futex_pivot_pending()
1784 struct mm_struct *mm = current->mm; in futex_hash_allocate() local
1796 fph = rcu_dereference(mm->futex_phash); in futex_hash_allocate()
1811 fph->mm = mm; in futex_hash_allocate()
1821 wait_var_event(mm, futex_pivot_pending(mm)); in futex_hash_allocate()
1824 scoped_guard(mutex, &mm->futex_hash_lock) { in futex_hash_allocate()
1828 cur = rcu_dereference_protected(mm->futex_phash, in futex_hash_allocate()
1829 lockdep_is_held(&mm->futex_hash_lock)); in futex_hash_allocate()
1830 new = mm->futex_phash_new; in futex_hash_allocate()
1831 mm->futex_phash_new = NULL; in futex_hash_allocate()
1841 mm->futex_phash_new = new; in futex_hash_allocate()
1874 if (!__futex_pivot_hash(mm, new) && custom) in futex_hash_allocate()
1886 if (!current->mm) in futex_hash_allocate_default()
1894 fph = rcu_dereference(current->mm->futex_phash); in futex_hash_allocate_default()
1921 fph = rcu_dereference(current->mm->futex_phash); in futex_hash_get_slots()