1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2014 ARM Limited
4 */
5
6 #include <linux/cpu.h>
7 #include <linux/init.h>
8 #include <linux/list.h>
9 #include <linux/perf_event.h>
10 #include <linux/sched.h>
11 #include <linux/slab.h>
12 #include <linux/sysctl.h>
13 #include <linux/uaccess.h>
14
15 #include <asm/cpufeature.h>
16 #include <asm/insn.h>
17 #include <asm/sysreg.h>
18 #include <asm/system_misc.h>
19 #include <asm/traps.h>
20
21 #define CREATE_TRACE_POINTS
22 #include "trace-events-emulation.h"
23
24 /*
25 * The runtime support for deprecated instruction support can be in one of
26 * following three states -
27 *
28 * 0 = undef
29 * 1 = emulate (software emulation)
30 * 2 = hw (supported in hardware)
31 */
32 enum insn_emulation_mode {
33 INSN_UNDEF,
34 INSN_EMULATE,
35 INSN_HW,
36 };
37
38 enum legacy_insn_status {
39 INSN_DEPRECATED,
40 INSN_OBSOLETE,
41 INSN_UNAVAILABLE,
42 };
43
44 struct insn_emulation {
45 const char *name;
46 enum legacy_insn_status status;
47 bool (*try_emulate)(struct pt_regs *regs,
48 u32 insn);
49 int (*set_hw_mode)(bool enable);
50
51 int current_mode;
52 int min;
53 int max;
54
55 /*
56 * sysctl for this emulation + a sentinal entry.
57 */
58 struct ctl_table sysctl[2];
59 };
60
61 #define ARM_OPCODE_CONDTEST_FAIL 0
62 #define ARM_OPCODE_CONDTEST_PASS 1
63 #define ARM_OPCODE_CONDTEST_UNCOND 2
64
65 #define ARM_OPCODE_CONDITION_UNCOND 0xf
66
aarch32_check_condition(u32 opcode,u32 psr)67 static unsigned int __maybe_unused aarch32_check_condition(u32 opcode, u32 psr)
68 {
69 u32 cc_bits = opcode >> 28;
70
71 if (cc_bits != ARM_OPCODE_CONDITION_UNCOND) {
72 if ((*aarch32_opcode_cond_checks[cc_bits])(psr))
73 return ARM_OPCODE_CONDTEST_PASS;
74 else
75 return ARM_OPCODE_CONDTEST_FAIL;
76 }
77 return ARM_OPCODE_CONDTEST_UNCOND;
78 }
79
80 #ifdef CONFIG_SWP_EMULATION
81 /*
82 * Implement emulation of the SWP/SWPB instructions using load-exclusive and
83 * store-exclusive.
84 *
85 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
86 * Where: Rt = destination
87 * Rt2 = source
88 * Rn = address
89 */
90
91 /*
92 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
93 */
94
95 /* Arbitrary constant to ensure forward-progress of the LL/SC loop */
96 #define __SWP_LL_SC_LOOPS 4
97
98 #define __user_swpX_asm(data, addr, res, temp, temp2, B) \
99 do { \
100 uaccess_enable_privileged(); \
101 __asm__ __volatile__( \
102 " mov %w3, %w6\n" \
103 "0: ldxr"B" %w2, [%4]\n" \
104 "1: stxr"B" %w0, %w1, [%4]\n" \
105 " cbz %w0, 2f\n" \
106 " sub %w3, %w3, #1\n" \
107 " cbnz %w3, 0b\n" \
108 " mov %w0, %w5\n" \
109 " b 3f\n" \
110 "2:\n" \
111 " mov %w1, %w2\n" \
112 "3:\n" \
113 _ASM_EXTABLE_UACCESS_ERR(0b, 3b, %w0) \
114 _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
115 : "=&r" (res), "+r" (data), "=&r" (temp), "=&r" (temp2) \
116 : "r" ((unsigned long)addr), "i" (-EAGAIN), \
117 "i" (__SWP_LL_SC_LOOPS) \
118 : "memory"); \
119 uaccess_disable_privileged(); \
120 } while (0)
121
122 #define __user_swp_asm(data, addr, res, temp, temp2) \
123 __user_swpX_asm(data, addr, res, temp, temp2, "")
124 #define __user_swpb_asm(data, addr, res, temp, temp2) \
125 __user_swpX_asm(data, addr, res, temp, temp2, "b")
126
127 /*
128 * Bit 22 of the instruction encoding distinguishes between
129 * the SWP and SWPB variants (bit set means SWPB).
130 */
131 #define TYPE_SWPB (1 << 22)
132
emulate_swpX(unsigned int address,unsigned int * data,unsigned int type)133 static int emulate_swpX(unsigned int address, unsigned int *data,
134 unsigned int type)
135 {
136 unsigned int res = 0;
137
138 if ((type != TYPE_SWPB) && (address & 0x3)) {
139 /* SWP to unaligned address not permitted */
140 pr_debug("SWP instruction on unaligned pointer!\n");
141 return -EFAULT;
142 }
143
144 while (1) {
145 unsigned long temp, temp2;
146
147 if (type == TYPE_SWPB)
148 __user_swpb_asm(*data, address, res, temp, temp2);
149 else
150 __user_swp_asm(*data, address, res, temp, temp2);
151
152 if (likely(res != -EAGAIN) || signal_pending(current))
153 break;
154
155 cond_resched();
156 }
157
158 return res;
159 }
160
161 /*
162 * swp_handler logs the id of calling process, dissects the instruction, sanity
163 * checks the memory location, calls emulate_swpX for the actual operation and
164 * deals with fixup/error handling before returning
165 */
swp_handler(struct pt_regs * regs,u32 instr)166 static int swp_handler(struct pt_regs *regs, u32 instr)
167 {
168 u32 destreg, data, type, address = 0;
169 const void __user *user_ptr;
170 int rn, rt2, res = 0;
171
172 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
173
174 type = instr & TYPE_SWPB;
175
176 switch (aarch32_check_condition(instr, regs->pstate)) {
177 case ARM_OPCODE_CONDTEST_PASS:
178 break;
179 case ARM_OPCODE_CONDTEST_FAIL:
180 /* Condition failed - return to next instruction */
181 goto ret;
182 case ARM_OPCODE_CONDTEST_UNCOND:
183 /* If unconditional encoding - not a SWP, undef */
184 return -EFAULT;
185 default:
186 return -EINVAL;
187 }
188
189 rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
190 rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
191
192 address = (u32)regs->user_regs.regs[rn];
193 data = (u32)regs->user_regs.regs[rt2];
194 destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
195
196 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
197 rn, address, destreg,
198 aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
199
200 /* Check access in reasonable access range for both SWP and SWPB */
201 user_ptr = (const void __user *)(unsigned long)(address & ~3);
202 if (!access_ok(user_ptr, 4)) {
203 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
204 address);
205 goto fault;
206 }
207
208 res = emulate_swpX(address, &data, type);
209 if (res == -EFAULT)
210 goto fault;
211 else if (res == 0)
212 regs->user_regs.regs[destreg] = data;
213
214 ret:
215 if (type == TYPE_SWPB)
216 trace_instruction_emulation("swpb", regs->pc);
217 else
218 trace_instruction_emulation("swp", regs->pc);
219
220 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
221 current->comm, (unsigned long)current->pid, regs->pc);
222
223 arm64_skip_faulting_instruction(regs, 4);
224 return 0;
225
226 fault:
227 pr_debug("SWP{B} emulation: access caused memory abort!\n");
228 arm64_notify_segfault(address);
229
230 return 0;
231 }
232
try_emulate_swp(struct pt_regs * regs,u32 insn)233 static bool try_emulate_swp(struct pt_regs *regs, u32 insn)
234 {
235 /* SWP{B} only exists in ARM state and does not exist in Thumb */
236 if (!compat_user_mode(regs) || compat_thumb_mode(regs))
237 return false;
238
239 if ((insn & 0x0fb00ff0) != 0x01000090)
240 return false;
241
242 return swp_handler(regs, insn) == 0;
243 }
244
245 static struct insn_emulation insn_swp = {
246 .name = "swp",
247 .status = INSN_OBSOLETE,
248 .try_emulate = try_emulate_swp,
249 .set_hw_mode = NULL,
250 };
251 #endif /* CONFIG_SWP_EMULATION */
252
253 #ifdef CONFIG_CP15_BARRIER_EMULATION
cp15barrier_handler(struct pt_regs * regs,u32 instr)254 static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
255 {
256 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
257
258 switch (aarch32_check_condition(instr, regs->pstate)) {
259 case ARM_OPCODE_CONDTEST_PASS:
260 break;
261 case ARM_OPCODE_CONDTEST_FAIL:
262 /* Condition failed - return to next instruction */
263 goto ret;
264 case ARM_OPCODE_CONDTEST_UNCOND:
265 /* If unconditional encoding - not a barrier instruction */
266 return -EFAULT;
267 default:
268 return -EINVAL;
269 }
270
271 switch (aarch32_insn_mcr_extract_crm(instr)) {
272 case 10:
273 /*
274 * dmb - mcr p15, 0, Rt, c7, c10, 5
275 * dsb - mcr p15, 0, Rt, c7, c10, 4
276 */
277 if (aarch32_insn_mcr_extract_opc2(instr) == 5) {
278 dmb(sy);
279 trace_instruction_emulation(
280 "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc);
281 } else {
282 dsb(sy);
283 trace_instruction_emulation(
284 "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc);
285 }
286 break;
287 case 5:
288 /*
289 * isb - mcr p15, 0, Rt, c7, c5, 4
290 *
291 * Taking an exception or returning from one acts as an
292 * instruction barrier. So no explicit barrier needed here.
293 */
294 trace_instruction_emulation(
295 "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc);
296 break;
297 }
298
299 ret:
300 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
301 current->comm, (unsigned long)current->pid, regs->pc);
302
303 arm64_skip_faulting_instruction(regs, 4);
304 return 0;
305 }
306
cp15_barrier_set_hw_mode(bool enable)307 static int cp15_barrier_set_hw_mode(bool enable)
308 {
309 if (enable)
310 sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
311 else
312 sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
313 return 0;
314 }
315
try_emulate_cp15_barrier(struct pt_regs * regs,u32 insn)316 static bool try_emulate_cp15_barrier(struct pt_regs *regs, u32 insn)
317 {
318 if (!compat_user_mode(regs) || compat_thumb_mode(regs))
319 return false;
320
321 if ((insn & 0x0fff0fdf) == 0x0e070f9a)
322 return cp15barrier_handler(regs, insn) == 0;
323
324 if ((insn & 0x0fff0fff) == 0x0e070f95)
325 return cp15barrier_handler(regs, insn) == 0;
326
327 return false;
328 }
329
330 static struct insn_emulation insn_cp15_barrier = {
331 .name = "cp15_barrier",
332 .status = INSN_DEPRECATED,
333 .try_emulate = try_emulate_cp15_barrier,
334 .set_hw_mode = cp15_barrier_set_hw_mode,
335 };
336 #endif /* CONFIG_CP15_BARRIER_EMULATION */
337
338 #ifdef CONFIG_SETEND_EMULATION
setend_set_hw_mode(bool enable)339 static int setend_set_hw_mode(bool enable)
340 {
341 if (!cpu_supports_mixed_endian_el0())
342 return -EINVAL;
343
344 if (enable)
345 sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
346 else
347 sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
348 return 0;
349 }
350
compat_setend_handler(struct pt_regs * regs,u32 big_endian)351 static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
352 {
353 char *insn;
354
355 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
356
357 if (big_endian) {
358 insn = "setend be";
359 regs->pstate |= PSR_AA32_E_BIT;
360 } else {
361 insn = "setend le";
362 regs->pstate &= ~PSR_AA32_E_BIT;
363 }
364
365 trace_instruction_emulation(insn, regs->pc);
366 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
367 current->comm, (unsigned long)current->pid, regs->pc);
368
369 return 0;
370 }
371
a32_setend_handler(struct pt_regs * regs,u32 instr)372 static int a32_setend_handler(struct pt_regs *regs, u32 instr)
373 {
374 int rc = compat_setend_handler(regs, (instr >> 9) & 1);
375 arm64_skip_faulting_instruction(regs, 4);
376 return rc;
377 }
378
t16_setend_handler(struct pt_regs * regs,u32 instr)379 static int t16_setend_handler(struct pt_regs *regs, u32 instr)
380 {
381 int rc = compat_setend_handler(regs, (instr >> 3) & 1);
382 arm64_skip_faulting_instruction(regs, 2);
383 return rc;
384 }
385
try_emulate_setend(struct pt_regs * regs,u32 insn)386 static bool try_emulate_setend(struct pt_regs *regs, u32 insn)
387 {
388 if (compat_thumb_mode(regs) &&
389 (insn & 0xfffffff7) == 0x0000b650)
390 return t16_setend_handler(regs, insn) == 0;
391
392 if (compat_user_mode(regs) &&
393 (insn & 0xfffffdff) == 0xf1010000)
394 return a32_setend_handler(regs, insn) == 0;
395
396 return false;
397 }
398
399 static struct insn_emulation insn_setend = {
400 .name = "setend",
401 .status = INSN_DEPRECATED,
402 .try_emulate = try_emulate_setend,
403 .set_hw_mode = setend_set_hw_mode,
404 };
405 #endif /* CONFIG_SETEND_EMULATION */
406
407 static struct insn_emulation *insn_emulations[] = {
408 #ifdef CONFIG_SWP_EMULATION
409 &insn_swp,
410 #endif
411 #ifdef CONFIG_CP15_BARRIER_EMULATION
412 &insn_cp15_barrier,
413 #endif
414 #ifdef CONFIG_SETEND_EMULATION
415 &insn_setend,
416 #endif
417 };
418
419 static DEFINE_MUTEX(insn_emulation_mutex);
420
enable_insn_hw_mode(void * data)421 static void enable_insn_hw_mode(void *data)
422 {
423 struct insn_emulation *insn = (struct insn_emulation *)data;
424 if (insn->set_hw_mode)
425 insn->set_hw_mode(true);
426 }
427
disable_insn_hw_mode(void * data)428 static void disable_insn_hw_mode(void *data)
429 {
430 struct insn_emulation *insn = (struct insn_emulation *)data;
431 if (insn->set_hw_mode)
432 insn->set_hw_mode(false);
433 }
434
435 /* Run set_hw_mode(mode) on all active CPUs */
run_all_cpu_set_hw_mode(struct insn_emulation * insn,bool enable)436 static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
437 {
438 if (!insn->set_hw_mode)
439 return -EINVAL;
440 if (enable)
441 on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
442 else
443 on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
444 return 0;
445 }
446
447 /*
448 * Run set_hw_mode for all insns on a starting CPU.
449 * Returns:
450 * 0 - If all the hooks ran successfully.
451 * -EINVAL - At least one hook is not supported by the CPU.
452 */
run_all_insn_set_hw_mode(unsigned int cpu)453 static int run_all_insn_set_hw_mode(unsigned int cpu)
454 {
455 int rc = 0;
456 unsigned long flags;
457
458 /*
459 * Disable IRQs to serialize against an IPI from
460 * run_all_cpu_set_hw_mode(), ensuring the HW is programmed to the most
461 * recent enablement state if the two race with one another.
462 */
463 local_irq_save(flags);
464 for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
465 struct insn_emulation *insn = insn_emulations[i];
466 bool enable = READ_ONCE(insn->current_mode) == INSN_HW;
467 if (insn->set_hw_mode && insn->set_hw_mode(enable)) {
468 pr_warn("CPU[%u] cannot support the emulation of %s",
469 cpu, insn->name);
470 rc = -EINVAL;
471 }
472 }
473 local_irq_restore(flags);
474
475 return rc;
476 }
477
update_insn_emulation_mode(struct insn_emulation * insn,enum insn_emulation_mode prev)478 static int update_insn_emulation_mode(struct insn_emulation *insn,
479 enum insn_emulation_mode prev)
480 {
481 int ret = 0;
482
483 switch (prev) {
484 case INSN_UNDEF: /* Nothing to be done */
485 break;
486 case INSN_EMULATE:
487 break;
488 case INSN_HW:
489 if (!run_all_cpu_set_hw_mode(insn, false))
490 pr_notice("Disabled %s support\n", insn->name);
491 break;
492 }
493
494 switch (insn->current_mode) {
495 case INSN_UNDEF:
496 break;
497 case INSN_EMULATE:
498 break;
499 case INSN_HW:
500 ret = run_all_cpu_set_hw_mode(insn, true);
501 if (!ret)
502 pr_notice("Enabled %s support\n", insn->name);
503 break;
504 }
505
506 return ret;
507 }
508
emulation_proc_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)509 static int emulation_proc_handler(struct ctl_table *table, int write,
510 void *buffer, size_t *lenp,
511 loff_t *ppos)
512 {
513 int ret = 0;
514 struct insn_emulation *insn = container_of(table->data, struct insn_emulation, current_mode);
515 enum insn_emulation_mode prev_mode = insn->current_mode;
516
517 mutex_lock(&insn_emulation_mutex);
518 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
519
520 if (ret || !write || prev_mode == insn->current_mode)
521 goto ret;
522
523 ret = update_insn_emulation_mode(insn, prev_mode);
524 if (ret) {
525 /* Mode change failed, revert to previous mode. */
526 WRITE_ONCE(insn->current_mode, prev_mode);
527 update_insn_emulation_mode(insn, INSN_UNDEF);
528 }
529 ret:
530 mutex_unlock(&insn_emulation_mutex);
531 return ret;
532 }
533
register_insn_emulation(struct insn_emulation * insn)534 static void __init register_insn_emulation(struct insn_emulation *insn)
535 {
536 struct ctl_table *sysctl;
537
538 insn->min = INSN_UNDEF;
539
540 switch (insn->status) {
541 case INSN_DEPRECATED:
542 insn->current_mode = INSN_EMULATE;
543 /* Disable the HW mode if it was turned on at early boot time */
544 run_all_cpu_set_hw_mode(insn, false);
545 insn->max = INSN_HW;
546 break;
547 case INSN_OBSOLETE:
548 insn->current_mode = INSN_UNDEF;
549 insn->max = INSN_EMULATE;
550 break;
551 case INSN_UNAVAILABLE:
552 insn->current_mode = INSN_UNDEF;
553 insn->max = INSN_UNDEF;
554 break;
555 }
556
557 /* Program the HW if required */
558 update_insn_emulation_mode(insn, INSN_UNDEF);
559
560 if (insn->status != INSN_UNAVAILABLE) {
561 sysctl = &insn->sysctl[0];
562
563 sysctl->mode = 0644;
564 sysctl->maxlen = sizeof(int);
565
566 sysctl->procname = insn->name;
567 sysctl->data = &insn->current_mode;
568 sysctl->extra1 = &insn->min;
569 sysctl->extra2 = &insn->max;
570 sysctl->proc_handler = emulation_proc_handler;
571
572 register_sysctl("abi", sysctl);
573 }
574 }
575
try_emulate_armv8_deprecated(struct pt_regs * regs,u32 insn)576 bool try_emulate_armv8_deprecated(struct pt_regs *regs, u32 insn)
577 {
578 for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
579 struct insn_emulation *ie = insn_emulations[i];
580
581 if (ie->status == INSN_UNAVAILABLE)
582 continue;
583
584 /*
585 * A trap may race with the mode being changed
586 * INSN_EMULATE<->INSN_HW. Try to emulate the instruction to
587 * avoid a spurious UNDEF.
588 */
589 if (READ_ONCE(ie->current_mode) == INSN_UNDEF)
590 continue;
591
592 if (ie->try_emulate(regs, insn))
593 return true;
594 }
595
596 return false;
597 }
598
599 /*
600 * Invoked as core_initcall, which guarantees that the instruction
601 * emulation is ready for userspace.
602 */
armv8_deprecated_init(void)603 static int __init armv8_deprecated_init(void)
604 {
605 #ifdef CONFIG_SETEND_EMULATION
606 if (!system_supports_mixed_endian_el0()) {
607 insn_setend.status = INSN_UNAVAILABLE;
608 pr_info("setend instruction emulation is not supported on this system\n");
609 }
610
611 #endif
612 for (int i = 0; i < ARRAY_SIZE(insn_emulations); i++) {
613 struct insn_emulation *ie = insn_emulations[i];
614
615 if (ie->status == INSN_UNAVAILABLE)
616 continue;
617
618 register_insn_emulation(ie);
619 }
620
621 cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING,
622 "arm64/isndep:starting",
623 run_all_insn_set_hw_mode, NULL);
624 return 0;
625 }
626
627 core_initcall(armv8_deprecated_init);
628