1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8 *
9 */
10 #include <linux/sched/isolation.h>
11 #include "sched.h"
12
13 enum hk_flags {
14 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
15 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
16 HK_FLAG_KERNEL_NOISE = BIT(HK_TYPE_KERNEL_NOISE),
17 };
18
19 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
20 EXPORT_SYMBOL_GPL(housekeeping_overridden);
21
22 struct housekeeping {
23 cpumask_var_t cpumasks[HK_TYPE_MAX];
24 unsigned long flags;
25 };
26
27 static struct housekeeping housekeeping;
28
housekeeping_enabled(enum hk_type type)29 bool housekeeping_enabled(enum hk_type type)
30 {
31 return !!(housekeeping.flags & BIT(type));
32 }
33 EXPORT_SYMBOL_GPL(housekeeping_enabled);
34
housekeeping_any_cpu(enum hk_type type)35 int housekeeping_any_cpu(enum hk_type type)
36 {
37 int cpu;
38
39 if (static_branch_unlikely(&housekeeping_overridden)) {
40 if (housekeeping.flags & BIT(type)) {
41 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
42 if (cpu < nr_cpu_ids)
43 return cpu;
44
45 cpu = cpumask_any_and_distribute(housekeeping.cpumasks[type], cpu_online_mask);
46 if (likely(cpu < nr_cpu_ids))
47 return cpu;
48 /*
49 * Unless we have another problem this can only happen
50 * at boot time before start_secondary() brings the 1st
51 * housekeeping CPU up.
52 */
53 WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
54 type != HK_TYPE_TIMER);
55 }
56 }
57 return smp_processor_id();
58 }
59 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
60
housekeeping_cpumask(enum hk_type type)61 const struct cpumask *housekeeping_cpumask(enum hk_type type)
62 {
63 if (static_branch_unlikely(&housekeeping_overridden))
64 if (housekeeping.flags & BIT(type))
65 return housekeeping.cpumasks[type];
66 return cpu_possible_mask;
67 }
68 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
69
housekeeping_affine(struct task_struct * t,enum hk_type type)70 void housekeeping_affine(struct task_struct *t, enum hk_type type)
71 {
72 if (static_branch_unlikely(&housekeeping_overridden))
73 if (housekeeping.flags & BIT(type))
74 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
75 }
76 EXPORT_SYMBOL_GPL(housekeeping_affine);
77
housekeeping_test_cpu(int cpu,enum hk_type type)78 bool housekeeping_test_cpu(int cpu, enum hk_type type)
79 {
80 if (static_branch_unlikely(&housekeeping_overridden))
81 if (housekeeping.flags & BIT(type))
82 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
83 return true;
84 }
85 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
86
housekeeping_init(void)87 void __init housekeeping_init(void)
88 {
89 enum hk_type type;
90
91 if (!housekeeping.flags)
92 return;
93
94 static_branch_enable(&housekeeping_overridden);
95
96 if (housekeeping.flags & HK_FLAG_KERNEL_NOISE)
97 sched_tick_offload_init();
98
99 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
100 /* We need at least one CPU to handle housekeeping work */
101 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
102 }
103 }
104
housekeeping_setup_type(enum hk_type type,cpumask_var_t housekeeping_staging)105 static void __init housekeeping_setup_type(enum hk_type type,
106 cpumask_var_t housekeeping_staging)
107 {
108
109 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
110 cpumask_copy(housekeeping.cpumasks[type],
111 housekeeping_staging);
112 }
113
housekeeping_setup(char * str,unsigned long flags)114 static int __init housekeeping_setup(char *str, unsigned long flags)
115 {
116 cpumask_var_t non_housekeeping_mask, housekeeping_staging;
117 unsigned int first_cpu;
118 int err = 0;
119
120 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE)) {
121 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
122 pr_warn("Housekeeping: nohz unsupported."
123 " Build with CONFIG_NO_HZ_FULL\n");
124 return 0;
125 }
126 }
127
128 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
129 if (cpulist_parse(str, non_housekeeping_mask) < 0) {
130 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
131 goto free_non_housekeeping_mask;
132 }
133
134 alloc_bootmem_cpumask_var(&housekeeping_staging);
135 cpumask_andnot(housekeeping_staging,
136 cpu_possible_mask, non_housekeeping_mask);
137
138 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
139 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
140 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
141 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
142 if (!housekeeping.flags) {
143 pr_warn("Housekeeping: must include one present CPU, "
144 "using boot CPU:%d\n", smp_processor_id());
145 }
146 }
147
148 if (cpumask_empty(non_housekeeping_mask))
149 goto free_housekeeping_staging;
150
151 if (!housekeeping.flags) {
152 /* First setup call ("nohz_full=" or "isolcpus=") */
153 enum hk_type type;
154
155 for_each_set_bit(type, &flags, HK_TYPE_MAX)
156 housekeeping_setup_type(type, housekeeping_staging);
157 } else {
158 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
159 enum hk_type type;
160 unsigned long iter_flags = flags & housekeeping.flags;
161
162 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
163 if (!cpumask_equal(housekeeping_staging,
164 housekeeping.cpumasks[type])) {
165 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
166 goto free_housekeeping_staging;
167 }
168 }
169
170 iter_flags = flags & ~housekeeping.flags;
171
172 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
173 housekeeping_setup_type(type, housekeeping_staging);
174 }
175
176 if ((flags & HK_FLAG_KERNEL_NOISE) && !(housekeeping.flags & HK_FLAG_KERNEL_NOISE))
177 tick_nohz_full_setup(non_housekeeping_mask);
178
179 housekeeping.flags |= flags;
180 err = 1;
181
182 free_housekeeping_staging:
183 free_bootmem_cpumask_var(housekeeping_staging);
184 free_non_housekeeping_mask:
185 free_bootmem_cpumask_var(non_housekeeping_mask);
186
187 return err;
188 }
189
housekeeping_nohz_full_setup(char * str)190 static int __init housekeeping_nohz_full_setup(char *str)
191 {
192 unsigned long flags;
193
194 flags = HK_FLAG_KERNEL_NOISE;
195
196 return housekeeping_setup(str, flags);
197 }
198 __setup("nohz_full=", housekeeping_nohz_full_setup);
199
housekeeping_isolcpus_setup(char * str)200 static int __init housekeeping_isolcpus_setup(char *str)
201 {
202 unsigned long flags = 0;
203 bool illegal = false;
204 char *par;
205 int len;
206
207 while (isalpha(*str)) {
208 /*
209 * isolcpus=nohz is equivalent to nohz_full.
210 */
211 if (!strncmp(str, "nohz,", 5)) {
212 str += 5;
213 flags |= HK_FLAG_KERNEL_NOISE;
214 continue;
215 }
216
217 if (!strncmp(str, "domain,", 7)) {
218 str += 7;
219 flags |= HK_FLAG_DOMAIN;
220 continue;
221 }
222
223 if (!strncmp(str, "managed_irq,", 12)) {
224 str += 12;
225 flags |= HK_FLAG_MANAGED_IRQ;
226 continue;
227 }
228
229 /*
230 * Skip unknown sub-parameter and validate that it is not
231 * containing an invalid character.
232 */
233 for (par = str, len = 0; *str && *str != ','; str++, len++) {
234 if (!isalpha(*str) && *str != '_')
235 illegal = true;
236 }
237
238 if (illegal) {
239 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
240 return 0;
241 }
242
243 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
244 str++;
245 }
246
247 /* Default behaviour for isolcpus without flags */
248 if (!flags)
249 flags |= HK_FLAG_DOMAIN;
250
251 return housekeeping_setup(str, flags);
252 }
253 __setup("isolcpus=", housekeeping_isolcpus_setup);
254