Lines Matching refs:node
38 #define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK)
209 return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */ in csd_lock_wait_getcpu()
238 unsigned int flags = READ_ONCE(csd->node.u_flags); in csd_lock_wait_toolong()
342 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
351 smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); in csd_lock_wait()
358 csd->node.u_flags |= CSD_FLAG_LOCK; in csd_lock()
370 WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); in csd_unlock()
375 smp_store_release(&csd->node.u_flags, 0); in csd_unlock()
380 void __smp_call_single_queue(int cpu, struct llist_node *node) in __smp_call_single_queue() argument
393 csd = container_of(node, call_single_data_t, node.llist); in __smp_call_single_queue()
412 if (llist_add(node, &per_cpu(call_single_queue, cpu))) in __smp_call_single_queue()
450 __smp_call_single_queue(cpu, &csd->node.llist); in generic_exec_single()
508 llist_for_each_entry(csd, entry, node.llist) { in __flush_smp_call_function_queue()
533 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { in __flush_smp_call_function_queue()
540 prev->next = &csd_next->node.llist; in __flush_smp_call_function_queue()
542 entry = &csd_next->node.llist; in __flush_smp_call_function_queue()
550 prev = &csd->node.llist; in __flush_smp_call_function_queue()
561 llist_for_each_entry_safe(csd, csd_next, entry, node.llist) { in __flush_smp_call_function_queue()
566 prev->next = &csd_next->node.llist; in __flush_smp_call_function_queue()
568 entry = &csd_next->node.llist; in __flush_smp_call_function_queue()
584 prev = &csd->node.llist; in __flush_smp_call_function_queue()
592 csd = llist_entry(entry, typeof(*csd), node.llist); in __flush_smp_call_function_queue()
641 .node = { .u_flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, }, in smp_call_function_single()
680 csd->node.src = smp_processor_id(); in smp_call_function_single()
681 csd->node.dst = cpu; in smp_call_function_single()
724 if (csd->node.u_flags & CSD_FLAG_LOCK) { in smp_call_function_single_async()
729 csd->node.u_flags = CSD_FLAG_LOCK; in smp_call_function_single_async()
831 csd->node.u_flags |= CSD_TYPE_SYNC; in smp_call_function_many_cond()
835 csd->node.src = smp_processor_id(); in smp_call_function_many_cond()
836 csd->node.dst = cpu; in smp_call_function_many_cond()
844 if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) { in smp_call_function_many_cond()