Lines Matching refs:csd
81 call_single_data_t csd; member
214 static void __csd_lock_record(struct __call_single_data *csd) in __csd_lock_record() argument
216 if (!csd) { in __csd_lock_record()
221 __this_cpu_write(cur_csd_func, csd->func); in __csd_lock_record()
222 __this_cpu_write(cur_csd_info, csd->info); in __csd_lock_record()
224 __this_cpu_write(cur_csd, csd); in __csd_lock_record()
229 static __always_inline void csd_lock_record(struct __call_single_data *csd) in csd_lock_record() argument
232 __csd_lock_record(csd); in csd_lock_record()
235 static int csd_lock_wait_getcpu(struct __call_single_data *csd) in csd_lock_wait_getcpu() argument
239 csd_type = CSD_TYPE(csd); in csd_lock_wait_getcpu()
241 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */ in csd_lock_wait_getcpu()
285 static void csd_lock_print_extended(struct __call_single_data *csd, int cpu) in csd_lock_print_extended() argument
288 unsigned int srccpu = csd->node.src; in csd_lock_print_extended()
324 static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id) in csd_lock_wait_toolong() argument
331 unsigned int flags = READ_ONCE(csd->node.u_flags); in csd_lock_wait_toolong()
336 cpu = csd_lock_wait_getcpu(csd); in csd_lock_wait_toolong()
350 cpu = csd_lock_wait_getcpu(csd); in csd_lock_wait_toolong()
358 cpu, csd->func, csd->info); in csd_lock_wait_toolong()
359 if (cpu_cur_csd && csd != cpu_cur_csd) { in csd_lock_wait_toolong()
369 csd_lock_print_extended(csd, cpu); in csd_lock_wait_toolong()
390 static void __csd_lock_wait(struct __call_single_data *csd) in __csd_lock_wait() argument
397 if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id)) in __csd_lock_wait()
404 static __always_inline void csd_lock_wait(struct __call_single_data *csd) in csd_lock_wait() argument
407 __csd_lock_wait(csd); in csd_lock_wait()
411 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
434 static void csd_lock_record(struct __call_single_data *csd) in csd_lock_record() argument
438 static __always_inline void csd_lock_wait(struct __call_single_data *csd) in csd_lock_wait() argument
440 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
444 static __always_inline void csd_lock(struct __call_single_data *csd) in csd_lock() argument
446 csd_lock_wait(csd); in csd_lock()
447 csd->node.u_flags |= CSD_FLAG_LOCK; in csd_lock()
457 static __always_inline void csd_unlock(struct __call_single_data *csd) in csd_unlock() argument
459 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); in csd_unlock()
464 smp_store_release(&csd->node.u_flags, 0); in csd_unlock()
504 static int generic_exec_single(int cpu, struct __call_single_data *csd) in generic_exec_single() argument
507 smp_call_func_t func = csd->func; in generic_exec_single()
508 void *info = csd->info; in generic_exec_single()
515 csd_lock_record(csd); in generic_exec_single()
516 csd_unlock(csd); in generic_exec_single()
525 csd_unlock(csd); in generic_exec_single()
529 __smp_call_single_queue(cpu, &csd->node.llist); in generic_exec_single()
563 call_single_data_t *csd, *csd_next; in flush_smp_call_function_queue() local
590 llist_for_each_entry(csd, entry, node.llist) { in flush_smp_call_function_queue()
591 switch (CSD_TYPE(csd)) { in flush_smp_call_function_queue()
596 csd->func); in flush_smp_call_function_queue()
605 CSD_TYPE(csd)); in flush_smp_call_function_queue()
615 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { in flush_smp_call_function_queue()
617 if (CSD_TYPE(csd) == CSD_TYPE_SYNC) { in flush_smp_call_function_queue()
618 smp_call_func_t func = csd->func; in flush_smp_call_function_queue()
619 void *info = csd->info; in flush_smp_call_function_queue()
627 csd_lock_record(csd); in flush_smp_call_function_queue()
629 csd_unlock(csd); in flush_smp_call_function_queue()
632 prev = &csd->node.llist; in flush_smp_call_function_queue()
647 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { in flush_smp_call_function_queue()
648 int type = CSD_TYPE(csd); in flush_smp_call_function_queue()
658 smp_call_func_t func = csd->func; in flush_smp_call_function_queue()
659 void *info = csd->info; in flush_smp_call_function_queue()
661 csd_lock_record(csd); in flush_smp_call_function_queue()
662 csd_unlock(csd); in flush_smp_call_function_queue()
666 irq_work_single(csd); in flush_smp_call_function_queue()
670 prev = &csd->node.llist; in flush_smp_call_function_queue()
712 call_single_data_t *csd; in smp_call_function_single() local
742 csd = &csd_stack; in smp_call_function_single()
744 csd = this_cpu_ptr(&csd_data); in smp_call_function_single()
745 csd_lock(csd); in smp_call_function_single()
748 csd->func = func; in smp_call_function_single()
749 csd->info = info; in smp_call_function_single()
751 csd->node.src = smp_processor_id(); in smp_call_function_single()
752 csd->node.dst = cpu; in smp_call_function_single()
755 err = generic_exec_single(cpu, csd); in smp_call_function_single()
758 csd_lock_wait(csd); in smp_call_function_single()
789 int smp_call_function_single_async(int cpu, struct __call_single_data *csd) in smp_call_function_single_async() argument
795 if (csd->node.u_flags & CSD_FLAG_LOCK) { in smp_call_function_single_async()
800 csd->node.u_flags = CSD_FLAG_LOCK; in smp_call_function_single_async()
803 err = generic_exec_single(cpu, csd); in smp_call_function_single_async()
915 call_single_data_t *csd = &pcpu->csd; in smp_call_function_many_cond() local
920 csd_lock(csd); in smp_call_function_many_cond()
922 csd->node.u_flags |= CSD_TYPE_SYNC; in smp_call_function_many_cond()
923 csd->func = func; in smp_call_function_many_cond()
924 csd->info = info; in smp_call_function_many_cond()
926 csd->node.src = smp_processor_id(); in smp_call_function_many_cond()
927 csd->node.dst = cpu; in smp_call_function_many_cond()
930 if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) { in smp_call_function_many_cond()
966 call_single_data_t *csd; in smp_call_function_many_cond() local
968 csd = &per_cpu_ptr(cfd->pcpu, cpu)->csd; in smp_call_function_many_cond()
969 csd_lock_wait(csd); in smp_call_function_many_cond()