1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * SMP related functions
4 *
5 * Copyright IBM Corp. 1999, 2012
6 * Author(s): Denis Joseph Barrow,
7 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
8 *
9 * based on other smp stuff by
10 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
11 * (c) 1998 Ingo Molnar
12 *
13 * The code outside of smp.c uses logical cpu numbers, only smp.c does
14 * the translation of logical to physical cpu ids. All new code that
15 * operates on physical cpu numbers needs to go into smp.c.
16 */
17
18 #define KMSG_COMPONENT "cpu"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20
21 #include <linux/workqueue.h>
22 #include <linux/memblock.h>
23 #include <linux/export.h>
24 #include <linux/init.h>
25 #include <linux/mm.h>
26 #include <linux/err.h>
27 #include <linux/spinlock.h>
28 #include <linux/kernel_stat.h>
29 #include <linux/delay.h>
30 #include <linux/interrupt.h>
31 #include <linux/irqflags.h>
32 #include <linux/irq_work.h>
33 #include <linux/cpu.h>
34 #include <linux/slab.h>
35 #include <linux/sched/hotplug.h>
36 #include <linux/sched/task_stack.h>
37 #include <linux/crash_dump.h>
38 #include <linux/kprobes.h>
39 #include <asm/asm-offsets.h>
40 #include <asm/diag.h>
41 #include <asm/switch_to.h>
42 #include <asm/facility.h>
43 #include <asm/ipl.h>
44 #include <asm/setup.h>
45 #include <asm/irq.h>
46 #include <asm/tlbflush.h>
47 #include <asm/vtimer.h>
48 #include <asm/abs_lowcore.h>
49 #include <asm/sclp.h>
50 #include <asm/debug.h>
51 #include <asm/os_info.h>
52 #include <asm/sigp.h>
53 #include <asm/idle.h>
54 #include <asm/nmi.h>
55 #include <asm/stacktrace.h>
56 #include <asm/topology.h>
57 #include <asm/vdso.h>
58 #include <asm/maccess.h>
59 #include "entry.h"
60
61 enum {
62 ec_schedule = 0,
63 ec_call_function_single,
64 ec_stop_cpu,
65 ec_mcck_pending,
66 ec_irq_work,
67 };
68
69 enum {
70 CPU_STATE_STANDBY,
71 CPU_STATE_CONFIGURED,
72 };
73
74 static DEFINE_PER_CPU(struct cpu *, cpu_device);
75
76 struct pcpu {
77 unsigned long ec_mask; /* bit mask for ec_xxx functions */
78 unsigned long ec_clk; /* sigp timestamp for ec_xxx */
79 signed char state; /* physical cpu state */
80 signed char polarization; /* physical polarization */
81 u16 address; /* physical cpu address */
82 };
83
84 static u8 boot_core_type;
85 static struct pcpu pcpu_devices[NR_CPUS];
86
87 unsigned int smp_cpu_mt_shift;
88 EXPORT_SYMBOL(smp_cpu_mt_shift);
89
90 unsigned int smp_cpu_mtid;
91 EXPORT_SYMBOL(smp_cpu_mtid);
92
93 #ifdef CONFIG_CRASH_DUMP
94 __vector128 __initdata boot_cpu_vector_save_area[__NUM_VXRS];
95 #endif
96
97 static unsigned int smp_max_threads __initdata = -1U;
98 cpumask_t cpu_setup_mask;
99
early_nosmt(char * s)100 static int __init early_nosmt(char *s)
101 {
102 smp_max_threads = 1;
103 return 0;
104 }
105 early_param("nosmt", early_nosmt);
106
early_smt(char * s)107 static int __init early_smt(char *s)
108 {
109 get_option(&s, &smp_max_threads);
110 return 0;
111 }
112 early_param("smt", early_smt);
113
114 /*
115 * The smp_cpu_state_mutex must be held when changing the state or polarization
116 * member of a pcpu data structure within the pcpu_devices arreay.
117 */
118 DEFINE_MUTEX(smp_cpu_state_mutex);
119
120 /*
121 * Signal processor helper functions.
122 */
__pcpu_sigp_relax(u16 addr,u8 order,unsigned long parm)123 static inline int __pcpu_sigp_relax(u16 addr, u8 order, unsigned long parm)
124 {
125 int cc;
126
127 while (1) {
128 cc = __pcpu_sigp(addr, order, parm, NULL);
129 if (cc != SIGP_CC_BUSY)
130 return cc;
131 cpu_relax();
132 }
133 }
134
pcpu_sigp_retry(struct pcpu * pcpu,u8 order,u32 parm)135 static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
136 {
137 int cc, retry;
138
139 for (retry = 0; ; retry++) {
140 cc = __pcpu_sigp(pcpu->address, order, parm, NULL);
141 if (cc != SIGP_CC_BUSY)
142 break;
143 if (retry >= 3)
144 udelay(10);
145 }
146 return cc;
147 }
148
pcpu_stopped(struct pcpu * pcpu)149 static inline int pcpu_stopped(struct pcpu *pcpu)
150 {
151 u32 status;
152
153 if (__pcpu_sigp(pcpu->address, SIGP_SENSE,
154 0, &status) != SIGP_CC_STATUS_STORED)
155 return 0;
156 return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED));
157 }
158
pcpu_running(struct pcpu * pcpu)159 static inline int pcpu_running(struct pcpu *pcpu)
160 {
161 if (__pcpu_sigp(pcpu->address, SIGP_SENSE_RUNNING,
162 0, NULL) != SIGP_CC_STATUS_STORED)
163 return 1;
164 /* Status stored condition code is equivalent to cpu not running. */
165 return 0;
166 }
167
168 /*
169 * Find struct pcpu by cpu address.
170 */
pcpu_find_address(const struct cpumask * mask,u16 address)171 static struct pcpu *pcpu_find_address(const struct cpumask *mask, u16 address)
172 {
173 int cpu;
174
175 for_each_cpu(cpu, mask)
176 if (pcpu_devices[cpu].address == address)
177 return pcpu_devices + cpu;
178 return NULL;
179 }
180
pcpu_ec_call(struct pcpu * pcpu,int ec_bit)181 static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
182 {
183 int order;
184
185 if (test_and_set_bit(ec_bit, &pcpu->ec_mask))
186 return;
187 order = pcpu_running(pcpu) ? SIGP_EXTERNAL_CALL : SIGP_EMERGENCY_SIGNAL;
188 pcpu->ec_clk = get_tod_clock_fast();
189 pcpu_sigp_retry(pcpu, order, 0);
190 }
191
pcpu_alloc_lowcore(struct pcpu * pcpu,int cpu)192 static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
193 {
194 unsigned long async_stack, nodat_stack, mcck_stack;
195 struct lowcore *lc;
196
197 lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
198 nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
199 async_stack = stack_alloc();
200 mcck_stack = stack_alloc();
201 if (!lc || !nodat_stack || !async_stack || !mcck_stack)
202 goto out;
203 memcpy(lc, &S390_lowcore, 512);
204 memset((char *) lc + 512, 0, sizeof(*lc) - 512);
205 lc->async_stack = async_stack + STACK_INIT_OFFSET;
206 lc->nodat_stack = nodat_stack + STACK_INIT_OFFSET;
207 lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET;
208 lc->cpu_nr = cpu;
209 lc->spinlock_lockval = arch_spin_lockval(cpu);
210 lc->spinlock_index = 0;
211 lc->return_lpswe = gen_lpswe(__LC_RETURN_PSW);
212 lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW);
213 lc->preempt_count = PREEMPT_DISABLED;
214 if (nmi_alloc_mcesa(&lc->mcesad))
215 goto out;
216 if (abs_lowcore_map(cpu, lc, true))
217 goto out_mcesa;
218 lowcore_ptr[cpu] = lc;
219 pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc));
220 return 0;
221
222 out_mcesa:
223 nmi_free_mcesa(&lc->mcesad);
224 out:
225 stack_free(mcck_stack);
226 stack_free(async_stack);
227 free_pages(nodat_stack, THREAD_SIZE_ORDER);
228 free_pages((unsigned long) lc, LC_ORDER);
229 return -ENOMEM;
230 }
231
pcpu_free_lowcore(struct pcpu * pcpu)232 static void pcpu_free_lowcore(struct pcpu *pcpu)
233 {
234 unsigned long async_stack, nodat_stack, mcck_stack;
235 struct lowcore *lc;
236 int cpu;
237
238 cpu = pcpu - pcpu_devices;
239 lc = lowcore_ptr[cpu];
240 nodat_stack = lc->nodat_stack - STACK_INIT_OFFSET;
241 async_stack = lc->async_stack - STACK_INIT_OFFSET;
242 mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET;
243 pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0);
244 lowcore_ptr[cpu] = NULL;
245 abs_lowcore_unmap(cpu);
246 nmi_free_mcesa(&lc->mcesad);
247 stack_free(async_stack);
248 stack_free(mcck_stack);
249 free_pages(nodat_stack, THREAD_SIZE_ORDER);
250 free_pages((unsigned long) lc, LC_ORDER);
251 }
252
pcpu_prepare_secondary(struct pcpu * pcpu,int cpu)253 static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
254 {
255 struct lowcore *lc = lowcore_ptr[cpu];
256
257 cpumask_set_cpu(cpu, &init_mm.context.cpu_attach_mask);
258 cpumask_set_cpu(cpu, mm_cpumask(&init_mm));
259 lc->cpu_nr = cpu;
260 lc->restart_flags = RESTART_FLAG_CTLREGS;
261 lc->spinlock_lockval = arch_spin_lockval(cpu);
262 lc->spinlock_index = 0;
263 lc->percpu_offset = __per_cpu_offset[cpu];
264 lc->kernel_asce = S390_lowcore.kernel_asce;
265 lc->user_asce = s390_invalid_asce;
266 lc->machine_flags = S390_lowcore.machine_flags;
267 lc->user_timer = lc->system_timer =
268 lc->steal_timer = lc->avg_steal_timer = 0;
269 __ctl_store(lc->cregs_save_area, 0, 15);
270 lc->cregs_save_area[1] = lc->kernel_asce;
271 lc->cregs_save_area[7] = lc->user_asce;
272 save_access_regs((unsigned int *) lc->access_regs_save_area);
273 arch_spin_lock_setup(cpu);
274 }
275
pcpu_attach_task(struct pcpu * pcpu,struct task_struct * tsk)276 static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
277 {
278 struct lowcore *lc;
279 int cpu;
280
281 cpu = pcpu - pcpu_devices;
282 lc = lowcore_ptr[cpu];
283 lc->kernel_stack = (unsigned long) task_stack_page(tsk)
284 + THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
285 lc->current_task = (unsigned long) tsk;
286 lc->lpp = LPP_MAGIC;
287 lc->current_pid = tsk->pid;
288 lc->user_timer = tsk->thread.user_timer;
289 lc->guest_timer = tsk->thread.guest_timer;
290 lc->system_timer = tsk->thread.system_timer;
291 lc->hardirq_timer = tsk->thread.hardirq_timer;
292 lc->softirq_timer = tsk->thread.softirq_timer;
293 lc->steal_timer = 0;
294 }
295
pcpu_start_fn(struct pcpu * pcpu,void (* func)(void *),void * data)296 static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
297 {
298 struct lowcore *lc;
299 int cpu;
300
301 cpu = pcpu - pcpu_devices;
302 lc = lowcore_ptr[cpu];
303 lc->restart_stack = lc->kernel_stack;
304 lc->restart_fn = (unsigned long) func;
305 lc->restart_data = (unsigned long) data;
306 lc->restart_source = -1U;
307 pcpu_sigp_retry(pcpu, SIGP_RESTART, 0);
308 }
309
310 typedef void (pcpu_delegate_fn)(void *);
311
312 /*
313 * Call function via PSW restart on pcpu and stop the current cpu.
314 */
__pcpu_delegate(pcpu_delegate_fn * func,void * data)315 static void __pcpu_delegate(pcpu_delegate_fn *func, void *data)
316 {
317 func(data); /* should not return */
318 }
319
pcpu_delegate(struct pcpu * pcpu,pcpu_delegate_fn * func,void * data,unsigned long stack)320 static void pcpu_delegate(struct pcpu *pcpu,
321 pcpu_delegate_fn *func,
322 void *data, unsigned long stack)
323 {
324 struct lowcore *lc, *abs_lc;
325 unsigned int source_cpu;
326
327 lc = lowcore_ptr[pcpu - pcpu_devices];
328 source_cpu = stap();
329
330 if (pcpu->address == source_cpu) {
331 call_on_stack(2, stack, void, __pcpu_delegate,
332 pcpu_delegate_fn *, func, void *, data);
333 }
334 /* Stop target cpu (if func returns this stops the current cpu). */
335 pcpu_sigp_retry(pcpu, SIGP_STOP, 0);
336 pcpu_sigp_retry(pcpu, SIGP_CPU_RESET, 0);
337 /* Restart func on the target cpu and stop the current cpu. */
338 if (lc) {
339 lc->restart_stack = stack;
340 lc->restart_fn = (unsigned long)func;
341 lc->restart_data = (unsigned long)data;
342 lc->restart_source = source_cpu;
343 } else {
344 abs_lc = get_abs_lowcore();
345 abs_lc->restart_stack = stack;
346 abs_lc->restart_fn = (unsigned long)func;
347 abs_lc->restart_data = (unsigned long)data;
348 abs_lc->restart_source = source_cpu;
349 put_abs_lowcore(abs_lc);
350 }
351 __bpon();
352 asm volatile(
353 "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
354 " brc 2,0b # busy, try again\n"
355 "1: sigp 0,%1,%3 # sigp stop to current cpu\n"
356 " brc 2,1b # busy, try again\n"
357 : : "d" (pcpu->address), "d" (source_cpu),
358 "K" (SIGP_RESTART), "K" (SIGP_STOP)
359 : "0", "1", "cc");
360 for (;;) ;
361 }
362
363 /*
364 * Enable additional logical cpus for multi-threading.
365 */
pcpu_set_smt(unsigned int mtid)366 static int pcpu_set_smt(unsigned int mtid)
367 {
368 int cc;
369
370 if (smp_cpu_mtid == mtid)
371 return 0;
372 cc = __pcpu_sigp(0, SIGP_SET_MULTI_THREADING, mtid, NULL);
373 if (cc == 0) {
374 smp_cpu_mtid = mtid;
375 smp_cpu_mt_shift = 0;
376 while (smp_cpu_mtid >= (1U << smp_cpu_mt_shift))
377 smp_cpu_mt_shift++;
378 pcpu_devices[0].address = stap();
379 }
380 return cc;
381 }
382
383 /*
384 * Call function on an online CPU.
385 */
smp_call_online_cpu(void (* func)(void *),void * data)386 void smp_call_online_cpu(void (*func)(void *), void *data)
387 {
388 struct pcpu *pcpu;
389
390 /* Use the current cpu if it is online. */
391 pcpu = pcpu_find_address(cpu_online_mask, stap());
392 if (!pcpu)
393 /* Use the first online cpu. */
394 pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
395 pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
396 }
397
398 /*
399 * Call function on the ipl CPU.
400 */
smp_call_ipl_cpu(void (* func)(void *),void * data)401 void smp_call_ipl_cpu(void (*func)(void *), void *data)
402 {
403 struct lowcore *lc = lowcore_ptr[0];
404
405 if (pcpu_devices[0].address == stap())
406 lc = &S390_lowcore;
407
408 pcpu_delegate(&pcpu_devices[0], func, data,
409 lc->nodat_stack);
410 }
411
smp_find_processor_id(u16 address)412 int smp_find_processor_id(u16 address)
413 {
414 int cpu;
415
416 for_each_present_cpu(cpu)
417 if (pcpu_devices[cpu].address == address)
418 return cpu;
419 return -1;
420 }
421
schedule_mcck_handler(void)422 void schedule_mcck_handler(void)
423 {
424 pcpu_ec_call(pcpu_devices + smp_processor_id(), ec_mcck_pending);
425 }
426
arch_vcpu_is_preempted(int cpu)427 bool notrace arch_vcpu_is_preempted(int cpu)
428 {
429 if (test_cpu_flag_of(CIF_ENABLED_WAIT, cpu))
430 return false;
431 if (pcpu_running(pcpu_devices + cpu))
432 return false;
433 return true;
434 }
435 EXPORT_SYMBOL(arch_vcpu_is_preempted);
436
smp_yield_cpu(int cpu)437 void notrace smp_yield_cpu(int cpu)
438 {
439 if (!MACHINE_HAS_DIAG9C)
440 return;
441 diag_stat_inc_norecursion(DIAG_STAT_X09C);
442 asm volatile("diag %0,0,0x9c"
443 : : "d" (pcpu_devices[cpu].address));
444 }
445 EXPORT_SYMBOL_GPL(smp_yield_cpu);
446
447 /*
448 * Send cpus emergency shutdown signal. This gives the cpus the
449 * opportunity to complete outstanding interrupts.
450 */
smp_emergency_stop(void)451 void notrace smp_emergency_stop(void)
452 {
453 static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED;
454 static cpumask_t cpumask;
455 u64 end;
456 int cpu;
457
458 arch_spin_lock(&lock);
459 cpumask_copy(&cpumask, cpu_online_mask);
460 cpumask_clear_cpu(smp_processor_id(), &cpumask);
461
462 end = get_tod_clock() + (1000000UL << 12);
463 for_each_cpu(cpu, &cpumask) {
464 struct pcpu *pcpu = pcpu_devices + cpu;
465 set_bit(ec_stop_cpu, &pcpu->ec_mask);
466 while (__pcpu_sigp(pcpu->address, SIGP_EMERGENCY_SIGNAL,
467 0, NULL) == SIGP_CC_BUSY &&
468 get_tod_clock() < end)
469 cpu_relax();
470 }
471 while (get_tod_clock() < end) {
472 for_each_cpu(cpu, &cpumask)
473 if (pcpu_stopped(pcpu_devices + cpu))
474 cpumask_clear_cpu(cpu, &cpumask);
475 if (cpumask_empty(&cpumask))
476 break;
477 cpu_relax();
478 }
479 arch_spin_unlock(&lock);
480 }
481 NOKPROBE_SYMBOL(smp_emergency_stop);
482
483 /*
484 * Stop all cpus but the current one.
485 */
smp_send_stop(void)486 void smp_send_stop(void)
487 {
488 int cpu;
489
490 /* Disable all interrupts/machine checks */
491 __load_psw_mask(PSW_KERNEL_BITS);
492 trace_hardirqs_off();
493
494 debug_set_critical();
495
496 if (oops_in_progress)
497 smp_emergency_stop();
498
499 /* stop all processors */
500 for_each_online_cpu(cpu) {
501 if (cpu == smp_processor_id())
502 continue;
503 pcpu_sigp_retry(pcpu_devices + cpu, SIGP_STOP, 0);
504 while (!pcpu_stopped(pcpu_devices + cpu))
505 cpu_relax();
506 }
507 }
508
509 /*
510 * This is the main routine where commands issued by other
511 * cpus are handled.
512 */
smp_handle_ext_call(void)513 static void smp_handle_ext_call(void)
514 {
515 unsigned long bits;
516
517 /* handle bit signal external calls */
518 bits = xchg(&pcpu_devices[smp_processor_id()].ec_mask, 0);
519 if (test_bit(ec_stop_cpu, &bits))
520 smp_stop_cpu();
521 if (test_bit(ec_schedule, &bits))
522 scheduler_ipi();
523 if (test_bit(ec_call_function_single, &bits))
524 generic_smp_call_function_single_interrupt();
525 if (test_bit(ec_mcck_pending, &bits))
526 s390_handle_mcck();
527 if (test_bit(ec_irq_work, &bits))
528 irq_work_run();
529 }
530
do_ext_call_interrupt(struct ext_code ext_code,unsigned int param32,unsigned long param64)531 static void do_ext_call_interrupt(struct ext_code ext_code,
532 unsigned int param32, unsigned long param64)
533 {
534 inc_irq_stat(ext_code.code == 0x1202 ? IRQEXT_EXC : IRQEXT_EMS);
535 smp_handle_ext_call();
536 }
537
arch_send_call_function_ipi_mask(const struct cpumask * mask)538 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
539 {
540 int cpu;
541
542 for_each_cpu(cpu, mask)
543 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
544 }
545
arch_send_call_function_single_ipi(int cpu)546 void arch_send_call_function_single_ipi(int cpu)
547 {
548 pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
549 }
550
551 /*
552 * this function sends a 'reschedule' IPI to another CPU.
553 * it goes straight through and wastes no time serializing
554 * anything. Worst case is that we lose a reschedule ...
555 */
smp_send_reschedule(int cpu)556 void smp_send_reschedule(int cpu)
557 {
558 pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
559 }
560
561 #ifdef CONFIG_IRQ_WORK
arch_irq_work_raise(void)562 void arch_irq_work_raise(void)
563 {
564 pcpu_ec_call(pcpu_devices + smp_processor_id(), ec_irq_work);
565 }
566 #endif
567
568 /*
569 * parameter area for the set/clear control bit callbacks
570 */
571 struct ec_creg_mask_parms {
572 unsigned long orval;
573 unsigned long andval;
574 int cr;
575 };
576
577 /*
578 * callback for setting/clearing control bits
579 */
smp_ctl_bit_callback(void * info)580 static void smp_ctl_bit_callback(void *info)
581 {
582 struct ec_creg_mask_parms *pp = info;
583 unsigned long cregs[16];
584
585 __ctl_store(cregs, 0, 15);
586 cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
587 __ctl_load(cregs, 0, 15);
588 }
589
590 static DEFINE_SPINLOCK(ctl_lock);
591
smp_ctl_set_clear_bit(int cr,int bit,bool set)592 void smp_ctl_set_clear_bit(int cr, int bit, bool set)
593 {
594 struct ec_creg_mask_parms parms = { .cr = cr, };
595 struct lowcore *abs_lc;
596 u64 ctlreg;
597
598 if (set) {
599 parms.orval = 1UL << bit;
600 parms.andval = -1UL;
601 } else {
602 parms.orval = 0;
603 parms.andval = ~(1UL << bit);
604 }
605 spin_lock(&ctl_lock);
606 abs_lc = get_abs_lowcore();
607 ctlreg = abs_lc->cregs_save_area[cr];
608 ctlreg = (ctlreg & parms.andval) | parms.orval;
609 abs_lc->cregs_save_area[cr] = ctlreg;
610 put_abs_lowcore(abs_lc);
611 spin_unlock(&ctl_lock);
612 on_each_cpu(smp_ctl_bit_callback, &parms, 1);
613 }
614 EXPORT_SYMBOL(smp_ctl_set_clear_bit);
615
616 #ifdef CONFIG_CRASH_DUMP
617
smp_store_status(int cpu)618 int smp_store_status(int cpu)
619 {
620 struct lowcore *lc;
621 struct pcpu *pcpu;
622 unsigned long pa;
623
624 pcpu = pcpu_devices + cpu;
625 lc = lowcore_ptr[cpu];
626 pa = __pa(&lc->floating_pt_save_area);
627 if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_STATUS_AT_ADDRESS,
628 pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
629 return -EIO;
630 if (!MACHINE_HAS_VX && !MACHINE_HAS_GS)
631 return 0;
632 pa = lc->mcesad & MCESA_ORIGIN_MASK;
633 if (MACHINE_HAS_GS)
634 pa |= lc->mcesad & MCESA_LC_MASK;
635 if (__pcpu_sigp_relax(pcpu->address, SIGP_STORE_ADDITIONAL_STATUS,
636 pa) != SIGP_CC_ORDER_CODE_ACCEPTED)
637 return -EIO;
638 return 0;
639 }
640
641 /*
642 * Collect CPU state of the previous, crashed system.
643 * There are four cases:
644 * 1) standard zfcp/nvme dump
645 * condition: OLDMEM_BASE == NULL && is_ipl_type_dump() == true
646 * The state for all CPUs except the boot CPU needs to be collected
647 * with sigp stop-and-store-status. The boot CPU state is located in
648 * the absolute lowcore of the memory stored in the HSA. The zcore code
649 * will copy the boot CPU state from the HSA.
650 * 2) stand-alone kdump for SCSI/NVMe (zfcp/nvme dump with swapped memory)
651 * condition: OLDMEM_BASE != NULL && is_ipl_type_dump() == true
652 * The state for all CPUs except the boot CPU needs to be collected
653 * with sigp stop-and-store-status. The firmware or the boot-loader
654 * stored the registers of the boot CPU in the absolute lowcore in the
655 * memory of the old system.
656 * 3) kdump and the old kernel did not store the CPU state,
657 * or stand-alone kdump for DASD
658 * condition: OLDMEM_BASE != NULL && !is_kdump_kernel()
659 * The state for all CPUs except the boot CPU needs to be collected
660 * with sigp stop-and-store-status. The kexec code or the boot-loader
661 * stored the registers of the boot CPU in the memory of the old system.
662 * 4) kdump and the old kernel stored the CPU state
663 * condition: OLDMEM_BASE != NULL && is_kdump_kernel()
664 * This case does not exist for s390 anymore, setup_arch explicitly
665 * deactivates the elfcorehdr= kernel parameter
666 */
dump_available(void)667 static bool dump_available(void)
668 {
669 return oldmem_data.start || is_ipl_type_dump();
670 }
671
smp_save_dump_ipl_cpu(void)672 void __init smp_save_dump_ipl_cpu(void)
673 {
674 struct save_area *sa;
675 void *regs;
676
677 if (!dump_available())
678 return;
679 sa = save_area_alloc(true);
680 regs = memblock_alloc(512, 8);
681 if (!sa || !regs)
682 panic("could not allocate memory for boot CPU save area\n");
683 copy_oldmem_kernel(regs, __LC_FPREGS_SAVE_AREA, 512);
684 save_area_add_regs(sa, regs);
685 memblock_free(regs, 512);
686 if (MACHINE_HAS_VX)
687 save_area_add_vxrs(sa, boot_cpu_vector_save_area);
688 }
689
smp_save_dump_secondary_cpus(void)690 void __init smp_save_dump_secondary_cpus(void)
691 {
692 int addr, boot_cpu_addr, max_cpu_addr;
693 struct save_area *sa;
694 void *page;
695
696 if (!dump_available())
697 return;
698 /* Allocate a page as dumping area for the store status sigps */
699 page = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
700 if (!page)
701 panic("ERROR: Failed to allocate %lx bytes below %lx\n",
702 PAGE_SIZE, 1UL << 31);
703
704 /* Set multi-threading state to the previous system. */
705 pcpu_set_smt(sclp.mtid_prev);
706 boot_cpu_addr = stap();
707 max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev;
708 for (addr = 0; addr <= max_cpu_addr; addr++) {
709 if (addr == boot_cpu_addr)
710 continue;
711 if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0) ==
712 SIGP_CC_NOT_OPERATIONAL)
713 continue;
714 sa = save_area_alloc(false);
715 if (!sa)
716 panic("could not allocate memory for save area\n");
717 __pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, __pa(page));
718 save_area_add_regs(sa, page);
719 if (MACHINE_HAS_VX) {
720 __pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, __pa(page));
721 save_area_add_vxrs(sa, page);
722 }
723 }
724 memblock_free(page, PAGE_SIZE);
725 diag_amode31_ops.diag308_reset();
726 pcpu_set_smt(0);
727 }
728 #endif /* CONFIG_CRASH_DUMP */
729
smp_cpu_set_polarization(int cpu,int val)730 void smp_cpu_set_polarization(int cpu, int val)
731 {
732 pcpu_devices[cpu].polarization = val;
733 }
734
smp_cpu_get_polarization(int cpu)735 int smp_cpu_get_polarization(int cpu)
736 {
737 return pcpu_devices[cpu].polarization;
738 }
739
smp_cpu_get_cpu_address(int cpu)740 int smp_cpu_get_cpu_address(int cpu)
741 {
742 return pcpu_devices[cpu].address;
743 }
744
smp_get_core_info(struct sclp_core_info * info,int early)745 static void __ref smp_get_core_info(struct sclp_core_info *info, int early)
746 {
747 static int use_sigp_detection;
748 int address;
749
750 if (use_sigp_detection || sclp_get_core_info(info, early)) {
751 use_sigp_detection = 1;
752 for (address = 0;
753 address < (SCLP_MAX_CORES << smp_cpu_mt_shift);
754 address += (1U << smp_cpu_mt_shift)) {
755 if (__pcpu_sigp_relax(address, SIGP_SENSE, 0) ==
756 SIGP_CC_NOT_OPERATIONAL)
757 continue;
758 info->core[info->configured].core_id =
759 address >> smp_cpu_mt_shift;
760 info->configured++;
761 }
762 info->combined = info->configured;
763 }
764 }
765
766 static int smp_add_present_cpu(int cpu);
767
smp_add_core(struct sclp_core_entry * core,cpumask_t * avail,bool configured,bool early)768 static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail,
769 bool configured, bool early)
770 {
771 struct pcpu *pcpu;
772 int cpu, nr, i;
773 u16 address;
774
775 nr = 0;
776 if (sclp.has_core_type && core->type != boot_core_type)
777 return nr;
778 cpu = cpumask_first(avail);
779 address = core->core_id << smp_cpu_mt_shift;
780 for (i = 0; (i <= smp_cpu_mtid) && (cpu < nr_cpu_ids); i++) {
781 if (pcpu_find_address(cpu_present_mask, address + i))
782 continue;
783 pcpu = pcpu_devices + cpu;
784 pcpu->address = address + i;
785 if (configured)
786 pcpu->state = CPU_STATE_CONFIGURED;
787 else
788 pcpu->state = CPU_STATE_STANDBY;
789 smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
790 set_cpu_present(cpu, true);
791 if (!early && smp_add_present_cpu(cpu) != 0)
792 set_cpu_present(cpu, false);
793 else
794 nr++;
795 cpumask_clear_cpu(cpu, avail);
796 cpu = cpumask_next(cpu, avail);
797 }
798 return nr;
799 }
800
__smp_rescan_cpus(struct sclp_core_info * info,bool early)801 static int __smp_rescan_cpus(struct sclp_core_info *info, bool early)
802 {
803 struct sclp_core_entry *core;
804 static cpumask_t avail;
805 bool configured;
806 u16 core_id;
807 int nr, i;
808
809 cpus_read_lock();
810 mutex_lock(&smp_cpu_state_mutex);
811 nr = 0;
812 cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
813 /*
814 * Add IPL core first (which got logical CPU number 0) to make sure
815 * that all SMT threads get subsequent logical CPU numbers.
816 */
817 if (early) {
818 core_id = pcpu_devices[0].address >> smp_cpu_mt_shift;
819 for (i = 0; i < info->configured; i++) {
820 core = &info->core[i];
821 if (core->core_id == core_id) {
822 nr += smp_add_core(core, &avail, true, early);
823 break;
824 }
825 }
826 }
827 for (i = 0; i < info->combined; i++) {
828 configured = i < info->configured;
829 nr += smp_add_core(&info->core[i], &avail, configured, early);
830 }
831 mutex_unlock(&smp_cpu_state_mutex);
832 cpus_read_unlock();
833 return nr;
834 }
835
smp_detect_cpus(void)836 void __init smp_detect_cpus(void)
837 {
838 unsigned int cpu, mtid, c_cpus, s_cpus;
839 struct sclp_core_info *info;
840 u16 address;
841
842 /* Get CPU information */
843 info = memblock_alloc(sizeof(*info), 8);
844 if (!info)
845 panic("%s: Failed to allocate %zu bytes align=0x%x\n",
846 __func__, sizeof(*info), 8);
847 smp_get_core_info(info, 1);
848 /* Find boot CPU type */
849 if (sclp.has_core_type) {
850 address = stap();
851 for (cpu = 0; cpu < info->combined; cpu++)
852 if (info->core[cpu].core_id == address) {
853 /* The boot cpu dictates the cpu type. */
854 boot_core_type = info->core[cpu].type;
855 break;
856 }
857 if (cpu >= info->combined)
858 panic("Could not find boot CPU type");
859 }
860
861 /* Set multi-threading state for the current system */
862 mtid = boot_core_type ? sclp.mtid : sclp.mtid_cp;
863 mtid = (mtid < smp_max_threads) ? mtid : smp_max_threads - 1;
864 pcpu_set_smt(mtid);
865
866 /* Print number of CPUs */
867 c_cpus = s_cpus = 0;
868 for (cpu = 0; cpu < info->combined; cpu++) {
869 if (sclp.has_core_type &&
870 info->core[cpu].type != boot_core_type)
871 continue;
872 if (cpu < info->configured)
873 c_cpus += smp_cpu_mtid + 1;
874 else
875 s_cpus += smp_cpu_mtid + 1;
876 }
877 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
878
879 /* Add CPUs present at boot */
880 __smp_rescan_cpus(info, true);
881 memblock_free(info, sizeof(*info));
882 }
883
884 /*
885 * Activate a secondary processor.
886 */
smp_start_secondary(void * cpuvoid)887 static void smp_start_secondary(void *cpuvoid)
888 {
889 int cpu = raw_smp_processor_id();
890
891 S390_lowcore.last_update_clock = get_tod_clock();
892 S390_lowcore.restart_stack = (unsigned long)restart_stack;
893 S390_lowcore.restart_fn = (unsigned long)do_restart;
894 S390_lowcore.restart_data = 0;
895 S390_lowcore.restart_source = -1U;
896 S390_lowcore.restart_flags = 0;
897 restore_access_regs(S390_lowcore.access_regs_save_area);
898 cpu_init();
899 rcu_cpu_starting(cpu);
900 init_cpu_timer();
901 vtime_init();
902 vdso_getcpu_init();
903 pfault_init();
904 cpumask_set_cpu(cpu, &cpu_setup_mask);
905 update_cpu_masks();
906 notify_cpu_starting(cpu);
907 if (topology_cpu_dedicated(cpu))
908 set_cpu_flag(CIF_DEDICATED_CPU);
909 else
910 clear_cpu_flag(CIF_DEDICATED_CPU);
911 set_cpu_online(cpu, true);
912 inc_irq_stat(CPU_RST);
913 local_irq_enable();
914 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
915 }
916
917 /* Upping and downing of CPUs */
__cpu_up(unsigned int cpu,struct task_struct * tidle)918 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
919 {
920 struct pcpu *pcpu = pcpu_devices + cpu;
921 int rc;
922
923 if (pcpu->state != CPU_STATE_CONFIGURED)
924 return -EIO;
925 if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) !=
926 SIGP_CC_ORDER_CODE_ACCEPTED)
927 return -EIO;
928
929 rc = pcpu_alloc_lowcore(pcpu, cpu);
930 if (rc)
931 return rc;
932 pcpu_prepare_secondary(pcpu, cpu);
933 pcpu_attach_task(pcpu, tidle);
934 pcpu_start_fn(pcpu, smp_start_secondary, NULL);
935 /* Wait until cpu puts itself in the online & active maps */
936 while (!cpu_online(cpu))
937 cpu_relax();
938 return 0;
939 }
940
941 static unsigned int setup_possible_cpus __initdata;
942
_setup_possible_cpus(char * s)943 static int __init _setup_possible_cpus(char *s)
944 {
945 get_option(&s, &setup_possible_cpus);
946 return 0;
947 }
948 early_param("possible_cpus", _setup_possible_cpus);
949
__cpu_disable(void)950 int __cpu_disable(void)
951 {
952 unsigned long cregs[16];
953 int cpu;
954
955 /* Handle possible pending IPIs */
956 smp_handle_ext_call();
957 cpu = smp_processor_id();
958 set_cpu_online(cpu, false);
959 cpumask_clear_cpu(cpu, &cpu_setup_mask);
960 update_cpu_masks();
961 /* Disable pseudo page faults on this cpu. */
962 pfault_fini();
963 /* Disable interrupt sources via control register. */
964 __ctl_store(cregs, 0, 15);
965 cregs[0] &= ~0x0000ee70UL; /* disable all external interrupts */
966 cregs[6] &= ~0xff000000UL; /* disable all I/O interrupts */
967 cregs[14] &= ~0x1f000000UL; /* disable most machine checks */
968 __ctl_load(cregs, 0, 15);
969 clear_cpu_flag(CIF_NOHZ_DELAY);
970 return 0;
971 }
972
__cpu_die(unsigned int cpu)973 void __cpu_die(unsigned int cpu)
974 {
975 struct pcpu *pcpu;
976
977 /* Wait until target cpu is down */
978 pcpu = pcpu_devices + cpu;
979 while (!pcpu_stopped(pcpu))
980 cpu_relax();
981 pcpu_free_lowcore(pcpu);
982 cpumask_clear_cpu(cpu, mm_cpumask(&init_mm));
983 cpumask_clear_cpu(cpu, &init_mm.context.cpu_attach_mask);
984 }
985
cpu_die(void)986 void __noreturn cpu_die(void)
987 {
988 idle_task_exit();
989 __bpon();
990 pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
991 for (;;) ;
992 }
993
smp_fill_possible_mask(void)994 void __init smp_fill_possible_mask(void)
995 {
996 unsigned int possible, sclp_max, cpu;
997
998 sclp_max = max(sclp.mtid, sclp.mtid_cp) + 1;
999 sclp_max = min(smp_max_threads, sclp_max);
1000 sclp_max = (sclp.max_cores * sclp_max) ?: nr_cpu_ids;
1001 possible = setup_possible_cpus ?: nr_cpu_ids;
1002 possible = min(possible, sclp_max);
1003 for (cpu = 0; cpu < possible && cpu < nr_cpu_ids; cpu++)
1004 set_cpu_possible(cpu, true);
1005 }
1006
smp_prepare_cpus(unsigned int max_cpus)1007 void __init smp_prepare_cpus(unsigned int max_cpus)
1008 {
1009 /* request the 0x1201 emergency signal external interrupt */
1010 if (register_external_irq(EXT_IRQ_EMERGENCY_SIG, do_ext_call_interrupt))
1011 panic("Couldn't request external interrupt 0x1201");
1012 /* request the 0x1202 external call external interrupt */
1013 if (register_external_irq(EXT_IRQ_EXTERNAL_CALL, do_ext_call_interrupt))
1014 panic("Couldn't request external interrupt 0x1202");
1015 }
1016
smp_prepare_boot_cpu(void)1017 void __init smp_prepare_boot_cpu(void)
1018 {
1019 struct pcpu *pcpu = pcpu_devices;
1020
1021 WARN_ON(!cpu_present(0) || !cpu_online(0));
1022 pcpu->state = CPU_STATE_CONFIGURED;
1023 S390_lowcore.percpu_offset = __per_cpu_offset[0];
1024 smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
1025 }
1026
smp_setup_processor_id(void)1027 void __init smp_setup_processor_id(void)
1028 {
1029 pcpu_devices[0].address = stap();
1030 S390_lowcore.cpu_nr = 0;
1031 S390_lowcore.spinlock_lockval = arch_spin_lockval(0);
1032 S390_lowcore.spinlock_index = 0;
1033 }
1034
1035 /*
1036 * the frequency of the profiling timer can be changed
1037 * by writing a multiplier value into /proc/profile.
1038 *
1039 * usually you want to run this on all CPUs ;)
1040 */
setup_profiling_timer(unsigned int multiplier)1041 int setup_profiling_timer(unsigned int multiplier)
1042 {
1043 return 0;
1044 }
1045
cpu_configure_show(struct device * dev,struct device_attribute * attr,char * buf)1046 static ssize_t cpu_configure_show(struct device *dev,
1047 struct device_attribute *attr, char *buf)
1048 {
1049 ssize_t count;
1050
1051 mutex_lock(&smp_cpu_state_mutex);
1052 count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
1053 mutex_unlock(&smp_cpu_state_mutex);
1054 return count;
1055 }
1056
cpu_configure_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1057 static ssize_t cpu_configure_store(struct device *dev,
1058 struct device_attribute *attr,
1059 const char *buf, size_t count)
1060 {
1061 struct pcpu *pcpu;
1062 int cpu, val, rc, i;
1063 char delim;
1064
1065 if (sscanf(buf, "%d %c", &val, &delim) != 1)
1066 return -EINVAL;
1067 if (val != 0 && val != 1)
1068 return -EINVAL;
1069 cpus_read_lock();
1070 mutex_lock(&smp_cpu_state_mutex);
1071 rc = -EBUSY;
1072 /* disallow configuration changes of online cpus and cpu 0 */
1073 cpu = dev->id;
1074 cpu = smp_get_base_cpu(cpu);
1075 if (cpu == 0)
1076 goto out;
1077 for (i = 0; i <= smp_cpu_mtid; i++)
1078 if (cpu_online(cpu + i))
1079 goto out;
1080 pcpu = pcpu_devices + cpu;
1081 rc = 0;
1082 switch (val) {
1083 case 0:
1084 if (pcpu->state != CPU_STATE_CONFIGURED)
1085 break;
1086 rc = sclp_core_deconfigure(pcpu->address >> smp_cpu_mt_shift);
1087 if (rc)
1088 break;
1089 for (i = 0; i <= smp_cpu_mtid; i++) {
1090 if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1091 continue;
1092 pcpu[i].state = CPU_STATE_STANDBY;
1093 smp_cpu_set_polarization(cpu + i,
1094 POLARIZATION_UNKNOWN);
1095 }
1096 topology_expect_change();
1097 break;
1098 case 1:
1099 if (pcpu->state != CPU_STATE_STANDBY)
1100 break;
1101 rc = sclp_core_configure(pcpu->address >> smp_cpu_mt_shift);
1102 if (rc)
1103 break;
1104 for (i = 0; i <= smp_cpu_mtid; i++) {
1105 if (cpu + i >= nr_cpu_ids || !cpu_present(cpu + i))
1106 continue;
1107 pcpu[i].state = CPU_STATE_CONFIGURED;
1108 smp_cpu_set_polarization(cpu + i,
1109 POLARIZATION_UNKNOWN);
1110 }
1111 topology_expect_change();
1112 break;
1113 default:
1114 break;
1115 }
1116 out:
1117 mutex_unlock(&smp_cpu_state_mutex);
1118 cpus_read_unlock();
1119 return rc ? rc : count;
1120 }
1121 static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
1122
show_cpu_address(struct device * dev,struct device_attribute * attr,char * buf)1123 static ssize_t show_cpu_address(struct device *dev,
1124 struct device_attribute *attr, char *buf)
1125 {
1126 return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
1127 }
1128 static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
1129
1130 static struct attribute *cpu_common_attrs[] = {
1131 &dev_attr_configure.attr,
1132 &dev_attr_address.attr,
1133 NULL,
1134 };
1135
1136 static struct attribute_group cpu_common_attr_group = {
1137 .attrs = cpu_common_attrs,
1138 };
1139
1140 static struct attribute *cpu_online_attrs[] = {
1141 &dev_attr_idle_count.attr,
1142 &dev_attr_idle_time_us.attr,
1143 NULL,
1144 };
1145
1146 static struct attribute_group cpu_online_attr_group = {
1147 .attrs = cpu_online_attrs,
1148 };
1149
smp_cpu_online(unsigned int cpu)1150 static int smp_cpu_online(unsigned int cpu)
1151 {
1152 struct device *s = &per_cpu(cpu_device, cpu)->dev;
1153
1154 return sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1155 }
1156
smp_cpu_pre_down(unsigned int cpu)1157 static int smp_cpu_pre_down(unsigned int cpu)
1158 {
1159 struct device *s = &per_cpu(cpu_device, cpu)->dev;
1160
1161 sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1162 return 0;
1163 }
1164
smp_add_present_cpu(int cpu)1165 static int smp_add_present_cpu(int cpu)
1166 {
1167 struct device *s;
1168 struct cpu *c;
1169 int rc;
1170
1171 c = kzalloc(sizeof(*c), GFP_KERNEL);
1172 if (!c)
1173 return -ENOMEM;
1174 per_cpu(cpu_device, cpu) = c;
1175 s = &c->dev;
1176 c->hotpluggable = 1;
1177 rc = register_cpu(c, cpu);
1178 if (rc)
1179 goto out;
1180 rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
1181 if (rc)
1182 goto out_cpu;
1183 rc = topology_cpu_init(c);
1184 if (rc)
1185 goto out_topology;
1186 return 0;
1187
1188 out_topology:
1189 sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
1190 out_cpu:
1191 unregister_cpu(c);
1192 out:
1193 return rc;
1194 }
1195
smp_rescan_cpus(void)1196 int __ref smp_rescan_cpus(void)
1197 {
1198 struct sclp_core_info *info;
1199 int nr;
1200
1201 info = kzalloc(sizeof(*info), GFP_KERNEL);
1202 if (!info)
1203 return -ENOMEM;
1204 smp_get_core_info(info, 0);
1205 nr = __smp_rescan_cpus(info, false);
1206 kfree(info);
1207 if (nr)
1208 topology_schedule_update();
1209 return 0;
1210 }
1211
rescan_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1212 static ssize_t __ref rescan_store(struct device *dev,
1213 struct device_attribute *attr,
1214 const char *buf,
1215 size_t count)
1216 {
1217 int rc;
1218
1219 rc = lock_device_hotplug_sysfs();
1220 if (rc)
1221 return rc;
1222 rc = smp_rescan_cpus();
1223 unlock_device_hotplug();
1224 return rc ? rc : count;
1225 }
1226 static DEVICE_ATTR_WO(rescan);
1227
s390_smp_init(void)1228 static int __init s390_smp_init(void)
1229 {
1230 int cpu, rc = 0;
1231
1232 rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan);
1233 if (rc)
1234 return rc;
1235 for_each_present_cpu(cpu) {
1236 rc = smp_add_present_cpu(cpu);
1237 if (rc)
1238 goto out;
1239 }
1240
1241 rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
1242 smp_cpu_online, smp_cpu_pre_down);
1243 rc = rc <= 0 ? rc : 0;
1244 out:
1245 return rc;
1246 }
1247 subsys_initcall(s390_smp_init);
1248
set_new_lowcore(struct lowcore * lc)1249 static __always_inline void set_new_lowcore(struct lowcore *lc)
1250 {
1251 union register_pair dst, src;
1252 u32 pfx;
1253
1254 src.even = (unsigned long) &S390_lowcore;
1255 src.odd = sizeof(S390_lowcore);
1256 dst.even = (unsigned long) lc;
1257 dst.odd = sizeof(*lc);
1258 pfx = __pa(lc);
1259
1260 asm volatile(
1261 " mvcl %[dst],%[src]\n"
1262 " spx %[pfx]\n"
1263 : [dst] "+&d" (dst.pair), [src] "+&d" (src.pair)
1264 : [pfx] "Q" (pfx)
1265 : "memory", "cc");
1266 }
1267
smp_reinit_ipl_cpu(void)1268 int __init smp_reinit_ipl_cpu(void)
1269 {
1270 unsigned long async_stack, nodat_stack, mcck_stack;
1271 struct lowcore *lc, *lc_ipl;
1272 unsigned long flags, cr0;
1273 u64 mcesad;
1274
1275 lc_ipl = lowcore_ptr[0];
1276 lc = (struct lowcore *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
1277 nodat_stack = __get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
1278 async_stack = stack_alloc();
1279 mcck_stack = stack_alloc();
1280 if (!lc || !nodat_stack || !async_stack || !mcck_stack || nmi_alloc_mcesa(&mcesad))
1281 panic("Couldn't allocate memory");
1282
1283 local_irq_save(flags);
1284 local_mcck_disable();
1285 set_new_lowcore(lc);
1286 S390_lowcore.nodat_stack = nodat_stack + STACK_INIT_OFFSET;
1287 S390_lowcore.async_stack = async_stack + STACK_INIT_OFFSET;
1288 S390_lowcore.mcck_stack = mcck_stack + STACK_INIT_OFFSET;
1289 __ctl_store(cr0, 0, 0);
1290 __ctl_clear_bit(0, 28); /* disable lowcore protection */
1291 S390_lowcore.mcesad = mcesad;
1292 __ctl_load(cr0, 0, 0);
1293 if (abs_lowcore_map(0, lc, false))
1294 panic("Couldn't remap absolute lowcore");
1295 lowcore_ptr[0] = lc;
1296 local_mcck_enable();
1297 local_irq_restore(flags);
1298
1299 free_pages(lc_ipl->async_stack - STACK_INIT_OFFSET, THREAD_SIZE_ORDER);
1300 memblock_free_late(__pa(lc_ipl->mcck_stack - STACK_INIT_OFFSET), THREAD_SIZE);
1301 memblock_free_late(__pa(lc_ipl), sizeof(*lc_ipl));
1302
1303 return 0;
1304 }
1305