Lines Matching refs:node
60 static struct scx_idle_cpus *idle_cpumask(int node) in idle_cpumask() argument
62 return node == NUMA_NO_NODE ? &scx_idle_global_masks : scx_idle_node_masks[node]; in idle_cpumask()
79 int node = scx_cpu_node_if_enabled(cpu); in scx_idle_test_and_clear_cpu() local
80 struct cpumask *idle_cpus = idle_cpumask(node)->cpu; in scx_idle_test_and_clear_cpu()
90 struct cpumask *idle_smts = idle_cpumask(node)->smt; in scx_idle_test_and_clear_cpu()
115 static s32 pick_idle_cpu_in_node(const struct cpumask *cpus_allowed, int node, u64 flags) in pick_idle_cpu_in_node() argument
121 cpu = cpumask_any_and_distribute(idle_cpumask(node)->smt, cpus_allowed); in pick_idle_cpu_in_node()
129 cpu = cpumask_any_and_distribute(idle_cpumask(node)->cpu, cpus_allowed); in pick_idle_cpu_in_node()
150 static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) in pick_idle_cpu_from_online_nodes() argument
163 node_clear(node, *unvisited); in pick_idle_cpu_from_online_nodes()
180 for_each_node_numadist(node, *unvisited) { in pick_idle_cpu_from_online_nodes()
181 cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); in pick_idle_cpu_from_online_nodes()
191 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, int node, u64 flags) in pick_idle_cpu_from_online_nodes() argument
200 static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, int node, u64 flags) in scx_pick_idle_cpu() argument
209 cpu = pick_idle_cpu_in_node(cpus_allowed, node, flags); in scx_pick_idle_cpu()
218 if (node == NUMA_NO_NODE || flags & SCX_PICK_IDLE_IN_NODE) in scx_pick_idle_cpu()
224 return pick_idle_cpu_from_online_nodes(cpus_allowed, node, flags); in scx_pick_idle_cpu()
456 int node = scx_cpu_node_if_enabled(prev_cpu); in scx_select_cpu_dfl() local
549 (!(flags & SCX_PICK_IDLE_IN_NODE) || (waker_node == node)) && in scx_select_cpu_dfl()
565 cpumask_test_cpu(prev_cpu, idle_cpumask(node)->smt) && in scx_select_cpu_dfl()
575 cpu = pick_idle_cpu_in_node(llc_cpus, node, SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
584 cpu = pick_idle_cpu_in_node(numa_cpus, node, SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
597 cpu = scx_pick_idle_cpu(allowed, node, flags | SCX_PICK_IDLE_CORE); in scx_select_cpu_dfl()
623 cpu = pick_idle_cpu_in_node(llc_cpus, node, 0); in scx_select_cpu_dfl()
632 cpu = pick_idle_cpu_in_node(numa_cpus, node, 0); in scx_select_cpu_dfl()
645 cpu = scx_pick_idle_cpu(allowed, node, flags); in scx_select_cpu_dfl()
693 int node = scx_cpu_node_if_enabled(cpu); in update_builtin_idle() local
694 struct cpumask *idle_cpus = idle_cpumask(node)->cpu; in update_builtin_idle()
701 struct cpumask *idle_smts = idle_cpumask(node)->smt; in update_builtin_idle()
777 int node; in reset_idle_masks() local
789 for_each_node(node) { in reset_idle_masks()
790 const struct cpumask *node_mask = cpumask_of_node(node); in reset_idle_masks()
792 cpumask_and(idle_cpumask(node)->cpu, cpu_online_mask, node_mask); in reset_idle_masks()
793 cpumask_and(idle_cpumask(node)->smt, cpu_online_mask, node_mask); in reset_idle_masks()
822 static int validate_node(int node) in validate_node() argument
830 if (node == NUMA_NO_NODE) in validate_node()
834 if (node < 0 || node >= nr_node_ids) { in validate_node()
835 scx_kf_error("invalid node %d", node); in validate_node()
840 if (!node_possible(node)) { in validate_node()
841 scx_kf_error("unavailable node %d", node); in validate_node()
845 return node; in validate_node()
996 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask_node(int node) in scx_bpf_get_idle_cpumask_node() argument
998 node = validate_node(node); in scx_bpf_get_idle_cpumask_node()
999 if (node < 0) in scx_bpf_get_idle_cpumask_node()
1002 return idle_cpumask(node)->cpu; in scx_bpf_get_idle_cpumask_node()
1035 __bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask_node(int node) in scx_bpf_get_idle_smtmask_node() argument
1037 node = validate_node(node); in scx_bpf_get_idle_smtmask_node()
1038 if (node < 0) in scx_bpf_get_idle_smtmask_node()
1042 return idle_cpumask(node)->smt; in scx_bpf_get_idle_smtmask_node()
1044 return idle_cpumask(node)->cpu; in scx_bpf_get_idle_smtmask_node()
1127 int node, u64 flags) in scx_bpf_pick_idle_cpu_node() argument
1129 node = validate_node(node); in scx_bpf_pick_idle_cpu_node()
1130 if (node < 0) in scx_bpf_pick_idle_cpu_node()
1131 return node; in scx_bpf_pick_idle_cpu_node()
1133 return scx_pick_idle_cpu(cpus_allowed, node, flags); in scx_bpf_pick_idle_cpu_node()
1194 int node, u64 flags) in scx_bpf_pick_any_cpu_node() argument
1198 node = validate_node(node); in scx_bpf_pick_any_cpu_node()
1199 if (node < 0) in scx_bpf_pick_any_cpu_node()
1200 return node; in scx_bpf_pick_any_cpu_node()
1202 cpu = scx_pick_idle_cpu(cpus_allowed, node, flags); in scx_bpf_pick_any_cpu_node()
1207 cpu = cpumask_any_and_distribute(cpumask_of_node(node), cpus_allowed); in scx_bpf_pick_any_cpu_node()