Lines Matching refs:membarrier_state
206 this_cpu_write(runqueues.membarrier_state, in ipi_sync_rq_state()
207 atomic_read(&mm->membarrier_state)); in ipi_sync_rq_state()
225 atomic_set(&mm->membarrier_state, 0); in membarrier_exec_mmap()
230 this_cpu_write(runqueues.membarrier_state, 0); in membarrier_exec_mmap()
236 int membarrier_state = 0; in membarrier_update_current_mm() local
239 membarrier_state = atomic_read(&next_mm->membarrier_state); in membarrier_update_current_mm()
240 if (READ_ONCE(rq->membarrier_state) == membarrier_state) in membarrier_update_current_mm()
242 WRITE_ONCE(rq->membarrier_state, membarrier_state); in membarrier_update_current_mm()
278 if (!(READ_ONCE(cpu_rq(cpu)->membarrier_state) & in membarrier_global_expedited()
319 if (!(atomic_read(&mm->membarrier_state) & in membarrier_private_expedited()
326 if (!(atomic_read(&mm->membarrier_state) & in membarrier_private_expedited()
332 if (!(atomic_read(&mm->membarrier_state) & in membarrier_private_expedited()
428 int membarrier_state = atomic_read(&mm->membarrier_state); in sync_runqueues_membarrier_state() local
433 this_cpu_write(runqueues.membarrier_state, membarrier_state); in sync_runqueues_membarrier_state()
489 if (atomic_read(&mm->membarrier_state) & in membarrier_register_global_expedited()
492 atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED, &mm->membarrier_state); in membarrier_register_global_expedited()
497 &mm->membarrier_state); in membarrier_register_global_expedited()
529 if ((atomic_read(&mm->membarrier_state) & ready_state) == ready_state) in membarrier_register_private_expedited()
535 atomic_or(set_state, &mm->membarrier_state); in membarrier_register_private_expedited()
539 atomic_or(ready_state, &mm->membarrier_state); in membarrier_register_private_expedited()
548 int registrations_mask = 0, membarrier_state, i; in membarrier_get_registrations() local
567 membarrier_state = atomic_read(&mm->membarrier_state); in membarrier_get_registrations()
569 if (membarrier_state & states[i]) { in membarrier_get_registrations()
571 membarrier_state &= ~states[i]; in membarrier_get_registrations()
574 WARN_ON_ONCE(membarrier_state != 0); in membarrier_get_registrations()