Lines Matching refs:csd
41 call_single_data_t __percpu *csd; member
66 cfd->csd = alloc_percpu(call_single_data_t); in smpcfd_prepare_cpu()
67 if (!cfd->csd) { in smpcfd_prepare_cpu()
82 free_percpu(cfd->csd); in smpcfd_dead_cpu()
131 csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd) in csd_do_func() argument
133 trace_csd_function_entry(func, csd); in csd_do_func()
135 trace_csd_function_exit(func, csd); in csd_do_func()
180 static void __csd_lock_record(call_single_data_t *csd) in __csd_lock_record() argument
182 if (!csd) { in __csd_lock_record()
187 __this_cpu_write(cur_csd_func, csd->func); in __csd_lock_record()
188 __this_cpu_write(cur_csd_info, csd->info); in __csd_lock_record()
190 __this_cpu_write(cur_csd, csd); in __csd_lock_record()
195 static __always_inline void csd_lock_record(call_single_data_t *csd) in csd_lock_record() argument
198 __csd_lock_record(csd); in csd_lock_record()
201 static int csd_lock_wait_getcpu(call_single_data_t *csd) in csd_lock_wait_getcpu() argument
205 csd_type = CSD_TYPE(csd); in csd_lock_wait_getcpu()
207 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */ in csd_lock_wait_getcpu()
229 static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id, unsigned… in csd_lock_wait_toolong() argument
236 unsigned int flags = READ_ONCE(csd->node.u_flags); in csd_lock_wait_toolong()
242 cpu = csd_lock_wait_getcpu(csd); in csd_lock_wait_toolong()
268 cpu = csd_lock_wait_getcpu(csd); in csd_lock_wait_toolong()
278 cpu, csd->func, csd->info); in csd_lock_wait_toolong()
288 if (cpu_cur_csd && csd != cpu_cur_csd) { in csd_lock_wait_toolong()
318 static void __csd_lock_wait(call_single_data_t *csd) in __csd_lock_wait() argument
326 if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages)) in __csd_lock_wait()
333 static __always_inline void csd_lock_wait(call_single_data_t *csd) in csd_lock_wait() argument
336 __csd_lock_wait(csd); in csd_lock_wait()
340 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
343 static void csd_lock_record(call_single_data_t *csd) in csd_lock_record() argument
347 static __always_inline void csd_lock_wait(call_single_data_t *csd) in csd_lock_wait() argument
349 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
353 static __always_inline void csd_lock(call_single_data_t *csd) in csd_lock() argument
355 csd_lock_wait(csd); in csd_lock()
356 csd->node.u_flags |= CSD_FLAG_LOCK; in csd_lock()
366 static __always_inline void csd_unlock(call_single_data_t *csd) in csd_unlock() argument
368 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); in csd_unlock()
373 smp_store_release(&csd->node.u_flags, 0); in csd_unlock()
388 call_single_data_t *csd; in __smp_call_single_queue() local
391 csd = container_of(node, call_single_data_t, node.llist); in __smp_call_single_queue()
392 func = CSD_TYPE(csd) == CSD_TYPE_TTWU ? in __smp_call_single_queue()
393 sched_ttwu_pending : csd->func; in __smp_call_single_queue()
395 trace_csd_queue_cpu(cpu, _RET_IP_, func, csd); in __smp_call_single_queue()
419 static int generic_exec_single(int cpu, call_single_data_t *csd) in generic_exec_single() argument
422 smp_call_func_t func = csd->func; in generic_exec_single()
423 void *info = csd->info; in generic_exec_single()
430 csd_lock_record(csd); in generic_exec_single()
431 csd_unlock(csd); in generic_exec_single()
440 csd_unlock(csd); in generic_exec_single()
444 __smp_call_single_queue(cpu, &csd->node.llist); in generic_exec_single()
476 call_single_data_t *csd, *csd_next; in __flush_smp_call_function_queue() local
502 llist_for_each_entry(csd, entry, node.llist) { in __flush_smp_call_function_queue()
503 switch (CSD_TYPE(csd)) { in __flush_smp_call_function_queue()
508 csd->func); in __flush_smp_call_function_queue()
517 CSD_TYPE(csd)); in __flush_smp_call_function_queue()
527 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { in __flush_smp_call_function_queue()
529 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) { in __flush_smp_call_function_queue()
530 smp_call_func_t func = csd->func; in __flush_smp_call_function_queue()
531 void *info = csd->info; in __flush_smp_call_function_queue()
539 csd_lock_record(csd); in __flush_smp_call_function_queue()
540 csd_do_func(func, info, csd); in __flush_smp_call_function_queue()
541 csd_unlock(csd); in __flush_smp_call_function_queue()
544 prev = &csd->node.llist; in __flush_smp_call_function_queue()
555 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { in __flush_smp_call_function_queue()
556 int type = CSD_TYPE(csd); in __flush_smp_call_function_queue()
566 smp_call_func_t func = csd->func; in __flush_smp_call_function_queue()
567 void *info = csd->info; in __flush_smp_call_function_queue()
569 csd_lock_record(csd); in __flush_smp_call_function_queue()
570 csd_unlock(csd); in __flush_smp_call_function_queue()
571 csd_do_func(func, info, csd); in __flush_smp_call_function_queue()
574 irq_work_single(csd); in __flush_smp_call_function_queue()
578 prev = &csd->node.llist; in __flush_smp_call_function_queue()
586 csd = llist_entry(entry, typeof(*csd), node.llist); in __flush_smp_call_function_queue()
587 csd_do_func(sched_ttwu_pending, entry, csd); in __flush_smp_call_function_queue()
633 call_single_data_t *csd; in smp_call_function_single() local
663 csd = &csd_stack; in smp_call_function_single()
665 csd = this_cpu_ptr(&csd_data); in smp_call_function_single()
666 csd_lock(csd); in smp_call_function_single()
669 csd->func = func; in smp_call_function_single()
670 csd->info = info; in smp_call_function_single()
672 csd->node.src = smp_processor_id(); in smp_call_function_single()
673 csd->node.dst = cpu; in smp_call_function_single()
676 err = generic_exec_single(cpu, csd); in smp_call_function_single()
679 csd_lock_wait(csd); in smp_call_function_single()
710 int smp_call_function_single_async(int cpu, call_single_data_t *csd) in smp_call_function_single_async() argument
716 if (csd->node.u_flags & CSD_FLAG_LOCK) { in smp_call_function_single_async()
721 csd->node.u_flags = CSD_FLAG_LOCK; in smp_call_function_single_async()
724 err = generic_exec_single(cpu, csd); in smp_call_function_single_async()
835 call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu); in smp_call_function_many_cond() local
842 csd_lock(csd); in smp_call_function_many_cond()
844 csd->node.u_flags |= CSD_TYPE_SYNC; in smp_call_function_many_cond()
845 csd->func = func; in smp_call_function_many_cond()
846 csd->info = info; in smp_call_function_many_cond()
848 csd->node.src = smp_processor_id(); in smp_call_function_many_cond()
849 csd->node.dst = cpu; in smp_call_function_many_cond()
851 trace_csd_queue_cpu(cpu, _RET_IP_, func, csd); in smp_call_function_many_cond()
853 if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) { in smp_call_function_many_cond()
881 call_single_data_t *csd; in smp_call_function_many_cond() local
883 csd = per_cpu_ptr(cfd->csd, cpu); in smp_call_function_many_cond()
884 csd_lock_wait(csd); in smp_call_function_many_cond()