1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013 Huawei Ltd.
4 * Author: Jiang Liu <liuj97@gmail.com>
5 *
6 * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
7 */
8 #include <linux/bitops.h>
9 #include <linux/bug.h>
10 #include <linux/printk.h>
11 #include <linux/sizes.h>
12 #include <linux/types.h>
13
14 #include <asm/debug-monitors.h>
15 #include <asm/errno.h>
16 #include <asm/insn.h>
17 #include <asm/kprobes.h>
18
19 #define AARCH64_INSN_SF_BIT BIT(31)
20 #define AARCH64_INSN_N_BIT BIT(22)
21 #define AARCH64_INSN_LSL_12 BIT(22)
22
aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,u32 * maskp,int * shiftp)23 static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
24 u32 *maskp, int *shiftp)
25 {
26 u32 mask;
27 int shift;
28
29 switch (type) {
30 case AARCH64_INSN_IMM_26:
31 mask = BIT(26) - 1;
32 shift = 0;
33 break;
34 case AARCH64_INSN_IMM_19:
35 mask = BIT(19) - 1;
36 shift = 5;
37 break;
38 case AARCH64_INSN_IMM_16:
39 mask = BIT(16) - 1;
40 shift = 5;
41 break;
42 case AARCH64_INSN_IMM_14:
43 mask = BIT(14) - 1;
44 shift = 5;
45 break;
46 case AARCH64_INSN_IMM_12:
47 mask = BIT(12) - 1;
48 shift = 10;
49 break;
50 case AARCH64_INSN_IMM_9:
51 mask = BIT(9) - 1;
52 shift = 12;
53 break;
54 case AARCH64_INSN_IMM_7:
55 mask = BIT(7) - 1;
56 shift = 15;
57 break;
58 case AARCH64_INSN_IMM_6:
59 case AARCH64_INSN_IMM_S:
60 mask = BIT(6) - 1;
61 shift = 10;
62 break;
63 case AARCH64_INSN_IMM_R:
64 mask = BIT(6) - 1;
65 shift = 16;
66 break;
67 case AARCH64_INSN_IMM_N:
68 mask = 1;
69 shift = 22;
70 break;
71 default:
72 return -EINVAL;
73 }
74
75 *maskp = mask;
76 *shiftp = shift;
77
78 return 0;
79 }
80
81 #define ADR_IMM_HILOSPLIT 2
82 #define ADR_IMM_SIZE SZ_2M
83 #define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1)
84 #define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1)
85 #define ADR_IMM_LOSHIFT 29
86 #define ADR_IMM_HISHIFT 5
87
aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type,u32 insn)88 u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn)
89 {
90 u32 immlo, immhi, mask;
91 int shift;
92
93 switch (type) {
94 case AARCH64_INSN_IMM_ADR:
95 shift = 0;
96 immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK;
97 immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK;
98 insn = (immhi << ADR_IMM_HILOSPLIT) | immlo;
99 mask = ADR_IMM_SIZE - 1;
100 break;
101 default:
102 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
103 pr_err("%s: unknown immediate encoding %d\n", __func__,
104 type);
105 return 0;
106 }
107 }
108
109 return (insn >> shift) & mask;
110 }
111
aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,u32 insn,u64 imm)112 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
113 u32 insn, u64 imm)
114 {
115 u32 immlo, immhi, mask;
116 int shift;
117
118 if (insn == AARCH64_BREAK_FAULT)
119 return AARCH64_BREAK_FAULT;
120
121 switch (type) {
122 case AARCH64_INSN_IMM_ADR:
123 shift = 0;
124 immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT;
125 imm >>= ADR_IMM_HILOSPLIT;
126 immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT;
127 imm = immlo | immhi;
128 mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) |
129 (ADR_IMM_HIMASK << ADR_IMM_HISHIFT));
130 break;
131 default:
132 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
133 pr_err("%s: unknown immediate encoding %d\n", __func__,
134 type);
135 return AARCH64_BREAK_FAULT;
136 }
137 }
138
139 /* Update the immediate field. */
140 insn &= ~(mask << shift);
141 insn |= (imm & mask) << shift;
142
143 return insn;
144 }
145
aarch64_insn_decode_register(enum aarch64_insn_register_type type,u32 insn)146 u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type,
147 u32 insn)
148 {
149 int shift;
150
151 switch (type) {
152 case AARCH64_INSN_REGTYPE_RT:
153 case AARCH64_INSN_REGTYPE_RD:
154 shift = 0;
155 break;
156 case AARCH64_INSN_REGTYPE_RN:
157 shift = 5;
158 break;
159 case AARCH64_INSN_REGTYPE_RT2:
160 case AARCH64_INSN_REGTYPE_RA:
161 shift = 10;
162 break;
163 case AARCH64_INSN_REGTYPE_RM:
164 shift = 16;
165 break;
166 default:
167 pr_err("%s: unknown register type encoding %d\n", __func__,
168 type);
169 return 0;
170 }
171
172 return (insn >> shift) & GENMASK(4, 0);
173 }
174
aarch64_insn_encode_register(enum aarch64_insn_register_type type,u32 insn,enum aarch64_insn_register reg)175 static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
176 u32 insn,
177 enum aarch64_insn_register reg)
178 {
179 int shift;
180
181 if (insn == AARCH64_BREAK_FAULT)
182 return AARCH64_BREAK_FAULT;
183
184 if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) {
185 pr_err("%s: unknown register encoding %d\n", __func__, reg);
186 return AARCH64_BREAK_FAULT;
187 }
188
189 switch (type) {
190 case AARCH64_INSN_REGTYPE_RT:
191 case AARCH64_INSN_REGTYPE_RD:
192 shift = 0;
193 break;
194 case AARCH64_INSN_REGTYPE_RN:
195 shift = 5;
196 break;
197 case AARCH64_INSN_REGTYPE_RT2:
198 case AARCH64_INSN_REGTYPE_RA:
199 shift = 10;
200 break;
201 case AARCH64_INSN_REGTYPE_RM:
202 case AARCH64_INSN_REGTYPE_RS:
203 shift = 16;
204 break;
205 default:
206 pr_err("%s: unknown register type encoding %d\n", __func__,
207 type);
208 return AARCH64_BREAK_FAULT;
209 }
210
211 insn &= ~(GENMASK(4, 0) << shift);
212 insn |= reg << shift;
213
214 return insn;
215 }
216
217 static const u32 aarch64_insn_ldst_size[] = {
218 [AARCH64_INSN_SIZE_8] = 0,
219 [AARCH64_INSN_SIZE_16] = 1,
220 [AARCH64_INSN_SIZE_32] = 2,
221 [AARCH64_INSN_SIZE_64] = 3,
222 };
223
aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,u32 insn)224 static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,
225 u32 insn)
226 {
227 u32 size;
228
229 if (type < AARCH64_INSN_SIZE_8 || type > AARCH64_INSN_SIZE_64) {
230 pr_err("%s: unknown size encoding %d\n", __func__, type);
231 return AARCH64_BREAK_FAULT;
232 }
233
234 size = aarch64_insn_ldst_size[type];
235 insn &= ~GENMASK(31, 30);
236 insn |= size << 30;
237
238 return insn;
239 }
240
label_imm_common(unsigned long pc,unsigned long addr,long range)241 static inline long label_imm_common(unsigned long pc, unsigned long addr,
242 long range)
243 {
244 long offset;
245
246 if ((pc & 0x3) || (addr & 0x3)) {
247 pr_err("%s: A64 instructions must be word aligned\n", __func__);
248 return range;
249 }
250
251 offset = ((long)addr - (long)pc);
252
253 if (offset < -range || offset >= range) {
254 pr_err("%s: offset out of range\n", __func__);
255 return range;
256 }
257
258 return offset;
259 }
260
aarch64_insn_gen_branch_imm(unsigned long pc,unsigned long addr,enum aarch64_insn_branch_type type)261 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
262 enum aarch64_insn_branch_type type)
263 {
264 u32 insn;
265 long offset;
266
267 /*
268 * B/BL support [-128M, 128M) offset
269 * ARM64 virtual address arrangement guarantees all kernel and module
270 * texts are within +/-128M.
271 */
272 offset = label_imm_common(pc, addr, SZ_128M);
273 if (offset >= SZ_128M)
274 return AARCH64_BREAK_FAULT;
275
276 switch (type) {
277 case AARCH64_INSN_BRANCH_LINK:
278 insn = aarch64_insn_get_bl_value();
279 break;
280 case AARCH64_INSN_BRANCH_NOLINK:
281 insn = aarch64_insn_get_b_value();
282 break;
283 default:
284 pr_err("%s: unknown branch encoding %d\n", __func__, type);
285 return AARCH64_BREAK_FAULT;
286 }
287
288 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
289 offset >> 2);
290 }
291
aarch64_insn_gen_comp_branch_imm(unsigned long pc,unsigned long addr,enum aarch64_insn_register reg,enum aarch64_insn_variant variant,enum aarch64_insn_branch_type type)292 u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
293 enum aarch64_insn_register reg,
294 enum aarch64_insn_variant variant,
295 enum aarch64_insn_branch_type type)
296 {
297 u32 insn;
298 long offset;
299
300 offset = label_imm_common(pc, addr, SZ_1M);
301 if (offset >= SZ_1M)
302 return AARCH64_BREAK_FAULT;
303
304 switch (type) {
305 case AARCH64_INSN_BRANCH_COMP_ZERO:
306 insn = aarch64_insn_get_cbz_value();
307 break;
308 case AARCH64_INSN_BRANCH_COMP_NONZERO:
309 insn = aarch64_insn_get_cbnz_value();
310 break;
311 default:
312 pr_err("%s: unknown branch encoding %d\n", __func__, type);
313 return AARCH64_BREAK_FAULT;
314 }
315
316 switch (variant) {
317 case AARCH64_INSN_VARIANT_32BIT:
318 break;
319 case AARCH64_INSN_VARIANT_64BIT:
320 insn |= AARCH64_INSN_SF_BIT;
321 break;
322 default:
323 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
324 return AARCH64_BREAK_FAULT;
325 }
326
327 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
328
329 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
330 offset >> 2);
331 }
332
aarch64_insn_gen_cond_branch_imm(unsigned long pc,unsigned long addr,enum aarch64_insn_condition cond)333 u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
334 enum aarch64_insn_condition cond)
335 {
336 u32 insn;
337 long offset;
338
339 offset = label_imm_common(pc, addr, SZ_1M);
340
341 insn = aarch64_insn_get_bcond_value();
342
343 if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) {
344 pr_err("%s: unknown condition encoding %d\n", __func__, cond);
345 return AARCH64_BREAK_FAULT;
346 }
347 insn |= cond;
348
349 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
350 offset >> 2);
351 }
352
aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,enum aarch64_insn_branch_type type)353 u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
354 enum aarch64_insn_branch_type type)
355 {
356 u32 insn;
357
358 switch (type) {
359 case AARCH64_INSN_BRANCH_NOLINK:
360 insn = aarch64_insn_get_br_value();
361 break;
362 case AARCH64_INSN_BRANCH_LINK:
363 insn = aarch64_insn_get_blr_value();
364 break;
365 case AARCH64_INSN_BRANCH_RETURN:
366 insn = aarch64_insn_get_ret_value();
367 break;
368 default:
369 pr_err("%s: unknown branch encoding %d\n", __func__, type);
370 return AARCH64_BREAK_FAULT;
371 }
372
373 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg);
374 }
375
aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,enum aarch64_insn_register base,enum aarch64_insn_register offset,enum aarch64_insn_size_type size,enum aarch64_insn_ldst_type type)376 u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
377 enum aarch64_insn_register base,
378 enum aarch64_insn_register offset,
379 enum aarch64_insn_size_type size,
380 enum aarch64_insn_ldst_type type)
381 {
382 u32 insn;
383
384 switch (type) {
385 case AARCH64_INSN_LDST_LOAD_REG_OFFSET:
386 insn = aarch64_insn_get_ldr_reg_value();
387 break;
388 case AARCH64_INSN_LDST_STORE_REG_OFFSET:
389 insn = aarch64_insn_get_str_reg_value();
390 break;
391 default:
392 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
393 return AARCH64_BREAK_FAULT;
394 }
395
396 insn = aarch64_insn_encode_ldst_size(size, insn);
397
398 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
399
400 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
401 base);
402
403 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
404 offset);
405 }
406
aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg,enum aarch64_insn_register base,unsigned int imm,enum aarch64_insn_size_type size,enum aarch64_insn_ldst_type type)407 u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg,
408 enum aarch64_insn_register base,
409 unsigned int imm,
410 enum aarch64_insn_size_type size,
411 enum aarch64_insn_ldst_type type)
412 {
413 u32 insn;
414 u32 shift;
415
416 if (size < AARCH64_INSN_SIZE_8 || size > AARCH64_INSN_SIZE_64) {
417 pr_err("%s: unknown size encoding %d\n", __func__, type);
418 return AARCH64_BREAK_FAULT;
419 }
420
421 shift = aarch64_insn_ldst_size[size];
422 if (imm & ~(BIT(12 + shift) - BIT(shift))) {
423 pr_err("%s: invalid imm: %d\n", __func__, imm);
424 return AARCH64_BREAK_FAULT;
425 }
426
427 imm >>= shift;
428
429 switch (type) {
430 case AARCH64_INSN_LDST_LOAD_IMM_OFFSET:
431 insn = aarch64_insn_get_ldr_imm_value();
432 break;
433 case AARCH64_INSN_LDST_STORE_IMM_OFFSET:
434 insn = aarch64_insn_get_str_imm_value();
435 break;
436 default:
437 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
438 return AARCH64_BREAK_FAULT;
439 }
440
441 insn = aarch64_insn_encode_ldst_size(size, insn);
442
443 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
444
445 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
446 base);
447
448 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
449 }
450
aarch64_insn_gen_load_literal(unsigned long pc,unsigned long addr,enum aarch64_insn_register reg,bool is64bit)451 u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr,
452 enum aarch64_insn_register reg,
453 bool is64bit)
454 {
455 u32 insn;
456 long offset;
457
458 offset = label_imm_common(pc, addr, SZ_1M);
459 if (offset >= SZ_1M)
460 return AARCH64_BREAK_FAULT;
461
462 insn = aarch64_insn_get_ldr_lit_value();
463
464 if (is64bit)
465 insn |= BIT(30);
466
467 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
468
469 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
470 offset >> 2);
471 }
472
aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,enum aarch64_insn_register reg2,enum aarch64_insn_register base,int offset,enum aarch64_insn_variant variant,enum aarch64_insn_ldst_type type)473 u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
474 enum aarch64_insn_register reg2,
475 enum aarch64_insn_register base,
476 int offset,
477 enum aarch64_insn_variant variant,
478 enum aarch64_insn_ldst_type type)
479 {
480 u32 insn;
481 int shift;
482
483 switch (type) {
484 case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX:
485 insn = aarch64_insn_get_ldp_pre_value();
486 break;
487 case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX:
488 insn = aarch64_insn_get_stp_pre_value();
489 break;
490 case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX:
491 insn = aarch64_insn_get_ldp_post_value();
492 break;
493 case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX:
494 insn = aarch64_insn_get_stp_post_value();
495 break;
496 default:
497 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
498 return AARCH64_BREAK_FAULT;
499 }
500
501 switch (variant) {
502 case AARCH64_INSN_VARIANT_32BIT:
503 if ((offset & 0x3) || (offset < -256) || (offset > 252)) {
504 pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n",
505 __func__, offset);
506 return AARCH64_BREAK_FAULT;
507 }
508 shift = 2;
509 break;
510 case AARCH64_INSN_VARIANT_64BIT:
511 if ((offset & 0x7) || (offset < -512) || (offset > 504)) {
512 pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n",
513 __func__, offset);
514 return AARCH64_BREAK_FAULT;
515 }
516 shift = 3;
517 insn |= AARCH64_INSN_SF_BIT;
518 break;
519 default:
520 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
521 return AARCH64_BREAK_FAULT;
522 }
523
524 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
525 reg1);
526
527 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
528 reg2);
529
530 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
531 base);
532
533 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn,
534 offset >> shift);
535 }
536
aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,enum aarch64_insn_register base,enum aarch64_insn_register state,enum aarch64_insn_size_type size,enum aarch64_insn_ldst_type type)537 u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
538 enum aarch64_insn_register base,
539 enum aarch64_insn_register state,
540 enum aarch64_insn_size_type size,
541 enum aarch64_insn_ldst_type type)
542 {
543 u32 insn;
544
545 switch (type) {
546 case AARCH64_INSN_LDST_LOAD_EX:
547 case AARCH64_INSN_LDST_LOAD_ACQ_EX:
548 insn = aarch64_insn_get_load_ex_value();
549 if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX)
550 insn |= BIT(15);
551 break;
552 case AARCH64_INSN_LDST_STORE_EX:
553 case AARCH64_INSN_LDST_STORE_REL_EX:
554 insn = aarch64_insn_get_store_ex_value();
555 if (type == AARCH64_INSN_LDST_STORE_REL_EX)
556 insn |= BIT(15);
557 break;
558 default:
559 pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type);
560 return AARCH64_BREAK_FAULT;
561 }
562
563 insn = aarch64_insn_encode_ldst_size(size, insn);
564
565 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
566 reg);
567
568 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
569 base);
570
571 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
572 AARCH64_INSN_REG_ZR);
573
574 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
575 state);
576 }
577
578 #ifdef CONFIG_ARM64_LSE_ATOMICS
aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,u32 insn)579 static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,
580 u32 insn)
581 {
582 u32 order;
583
584 switch (type) {
585 case AARCH64_INSN_MEM_ORDER_NONE:
586 order = 0;
587 break;
588 case AARCH64_INSN_MEM_ORDER_ACQ:
589 order = 2;
590 break;
591 case AARCH64_INSN_MEM_ORDER_REL:
592 order = 1;
593 break;
594 case AARCH64_INSN_MEM_ORDER_ACQREL:
595 order = 3;
596 break;
597 default:
598 pr_err("%s: unknown mem order %d\n", __func__, type);
599 return AARCH64_BREAK_FAULT;
600 }
601
602 insn &= ~GENMASK(23, 22);
603 insn |= order << 22;
604
605 return insn;
606 }
607
aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,enum aarch64_insn_register address,enum aarch64_insn_register value,enum aarch64_insn_size_type size,enum aarch64_insn_mem_atomic_op op,enum aarch64_insn_mem_order_type order)608 u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
609 enum aarch64_insn_register address,
610 enum aarch64_insn_register value,
611 enum aarch64_insn_size_type size,
612 enum aarch64_insn_mem_atomic_op op,
613 enum aarch64_insn_mem_order_type order)
614 {
615 u32 insn;
616
617 switch (op) {
618 case AARCH64_INSN_MEM_ATOMIC_ADD:
619 insn = aarch64_insn_get_ldadd_value();
620 break;
621 case AARCH64_INSN_MEM_ATOMIC_CLR:
622 insn = aarch64_insn_get_ldclr_value();
623 break;
624 case AARCH64_INSN_MEM_ATOMIC_EOR:
625 insn = aarch64_insn_get_ldeor_value();
626 break;
627 case AARCH64_INSN_MEM_ATOMIC_SET:
628 insn = aarch64_insn_get_ldset_value();
629 break;
630 case AARCH64_INSN_MEM_ATOMIC_SWP:
631 insn = aarch64_insn_get_swp_value();
632 break;
633 default:
634 pr_err("%s: unimplemented mem atomic op %d\n", __func__, op);
635 return AARCH64_BREAK_FAULT;
636 }
637
638 switch (size) {
639 case AARCH64_INSN_SIZE_32:
640 case AARCH64_INSN_SIZE_64:
641 break;
642 default:
643 pr_err("%s: unimplemented size encoding %d\n", __func__, size);
644 return AARCH64_BREAK_FAULT;
645 }
646
647 insn = aarch64_insn_encode_ldst_size(size, insn);
648
649 insn = aarch64_insn_encode_ldst_order(order, insn);
650
651 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
652 result);
653
654 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
655 address);
656
657 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
658 value);
659 }
660
aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,u32 insn)661 static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,
662 u32 insn)
663 {
664 u32 order;
665
666 switch (type) {
667 case AARCH64_INSN_MEM_ORDER_NONE:
668 order = 0;
669 break;
670 case AARCH64_INSN_MEM_ORDER_ACQ:
671 order = BIT(22);
672 break;
673 case AARCH64_INSN_MEM_ORDER_REL:
674 order = BIT(15);
675 break;
676 case AARCH64_INSN_MEM_ORDER_ACQREL:
677 order = BIT(15) | BIT(22);
678 break;
679 default:
680 pr_err("%s: unknown mem order %d\n", __func__, type);
681 return AARCH64_BREAK_FAULT;
682 }
683
684 insn &= ~(BIT(15) | BIT(22));
685 insn |= order;
686
687 return insn;
688 }
689
aarch64_insn_gen_cas(enum aarch64_insn_register result,enum aarch64_insn_register address,enum aarch64_insn_register value,enum aarch64_insn_size_type size,enum aarch64_insn_mem_order_type order)690 u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
691 enum aarch64_insn_register address,
692 enum aarch64_insn_register value,
693 enum aarch64_insn_size_type size,
694 enum aarch64_insn_mem_order_type order)
695 {
696 u32 insn;
697
698 switch (size) {
699 case AARCH64_INSN_SIZE_32:
700 case AARCH64_INSN_SIZE_64:
701 break;
702 default:
703 pr_err("%s: unimplemented size encoding %d\n", __func__, size);
704 return AARCH64_BREAK_FAULT;
705 }
706
707 insn = aarch64_insn_get_cas_value();
708
709 insn = aarch64_insn_encode_ldst_size(size, insn);
710
711 insn = aarch64_insn_encode_cas_order(order, insn);
712
713 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
714 result);
715
716 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
717 address);
718
719 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
720 value);
721 }
722 #endif
723
aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,enum aarch64_insn_register src,int imm,enum aarch64_insn_variant variant,enum aarch64_insn_adsb_type type)724 u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
725 enum aarch64_insn_register src,
726 int imm, enum aarch64_insn_variant variant,
727 enum aarch64_insn_adsb_type type)
728 {
729 u32 insn;
730
731 switch (type) {
732 case AARCH64_INSN_ADSB_ADD:
733 insn = aarch64_insn_get_add_imm_value();
734 break;
735 case AARCH64_INSN_ADSB_SUB:
736 insn = aarch64_insn_get_sub_imm_value();
737 break;
738 case AARCH64_INSN_ADSB_ADD_SETFLAGS:
739 insn = aarch64_insn_get_adds_imm_value();
740 break;
741 case AARCH64_INSN_ADSB_SUB_SETFLAGS:
742 insn = aarch64_insn_get_subs_imm_value();
743 break;
744 default:
745 pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
746 return AARCH64_BREAK_FAULT;
747 }
748
749 switch (variant) {
750 case AARCH64_INSN_VARIANT_32BIT:
751 break;
752 case AARCH64_INSN_VARIANT_64BIT:
753 insn |= AARCH64_INSN_SF_BIT;
754 break;
755 default:
756 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
757 return AARCH64_BREAK_FAULT;
758 }
759
760 /* We can't encode more than a 24bit value (12bit + 12bit shift) */
761 if (imm & ~(BIT(24) - 1))
762 goto out;
763
764 /* If we have something in the top 12 bits... */
765 if (imm & ~(SZ_4K - 1)) {
766 /* ... and in the low 12 bits -> error */
767 if (imm & (SZ_4K - 1))
768 goto out;
769
770 imm >>= 12;
771 insn |= AARCH64_INSN_LSL_12;
772 }
773
774 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
775
776 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
777
778 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
779
780 out:
781 pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
782 return AARCH64_BREAK_FAULT;
783 }
784
aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,enum aarch64_insn_register src,int immr,int imms,enum aarch64_insn_variant variant,enum aarch64_insn_bitfield_type type)785 u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
786 enum aarch64_insn_register src,
787 int immr, int imms,
788 enum aarch64_insn_variant variant,
789 enum aarch64_insn_bitfield_type type)
790 {
791 u32 insn;
792 u32 mask;
793
794 switch (type) {
795 case AARCH64_INSN_BITFIELD_MOVE:
796 insn = aarch64_insn_get_bfm_value();
797 break;
798 case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED:
799 insn = aarch64_insn_get_ubfm_value();
800 break;
801 case AARCH64_INSN_BITFIELD_MOVE_SIGNED:
802 insn = aarch64_insn_get_sbfm_value();
803 break;
804 default:
805 pr_err("%s: unknown bitfield encoding %d\n", __func__, type);
806 return AARCH64_BREAK_FAULT;
807 }
808
809 switch (variant) {
810 case AARCH64_INSN_VARIANT_32BIT:
811 mask = GENMASK(4, 0);
812 break;
813 case AARCH64_INSN_VARIANT_64BIT:
814 insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT;
815 mask = GENMASK(5, 0);
816 break;
817 default:
818 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
819 return AARCH64_BREAK_FAULT;
820 }
821
822 if (immr & ~mask) {
823 pr_err("%s: invalid immr encoding %d\n", __func__, immr);
824 return AARCH64_BREAK_FAULT;
825 }
826 if (imms & ~mask) {
827 pr_err("%s: invalid imms encoding %d\n", __func__, imms);
828 return AARCH64_BREAK_FAULT;
829 }
830
831 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
832
833 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
834
835 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
836
837 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
838 }
839
aarch64_insn_gen_movewide(enum aarch64_insn_register dst,int imm,int shift,enum aarch64_insn_variant variant,enum aarch64_insn_movewide_type type)840 u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst,
841 int imm, int shift,
842 enum aarch64_insn_variant variant,
843 enum aarch64_insn_movewide_type type)
844 {
845 u32 insn;
846
847 switch (type) {
848 case AARCH64_INSN_MOVEWIDE_ZERO:
849 insn = aarch64_insn_get_movz_value();
850 break;
851 case AARCH64_INSN_MOVEWIDE_KEEP:
852 insn = aarch64_insn_get_movk_value();
853 break;
854 case AARCH64_INSN_MOVEWIDE_INVERSE:
855 insn = aarch64_insn_get_movn_value();
856 break;
857 default:
858 pr_err("%s: unknown movewide encoding %d\n", __func__, type);
859 return AARCH64_BREAK_FAULT;
860 }
861
862 if (imm & ~(SZ_64K - 1)) {
863 pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
864 return AARCH64_BREAK_FAULT;
865 }
866
867 switch (variant) {
868 case AARCH64_INSN_VARIANT_32BIT:
869 if (shift != 0 && shift != 16) {
870 pr_err("%s: invalid shift encoding %d\n", __func__,
871 shift);
872 return AARCH64_BREAK_FAULT;
873 }
874 break;
875 case AARCH64_INSN_VARIANT_64BIT:
876 insn |= AARCH64_INSN_SF_BIT;
877 if (shift != 0 && shift != 16 && shift != 32 && shift != 48) {
878 pr_err("%s: invalid shift encoding %d\n", __func__,
879 shift);
880 return AARCH64_BREAK_FAULT;
881 }
882 break;
883 default:
884 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
885 return AARCH64_BREAK_FAULT;
886 }
887
888 insn |= (shift >> 4) << 21;
889
890 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
891
892 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
893 }
894
aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg,int shift,enum aarch64_insn_variant variant,enum aarch64_insn_adsb_type type)895 u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
896 enum aarch64_insn_register src,
897 enum aarch64_insn_register reg,
898 int shift,
899 enum aarch64_insn_variant variant,
900 enum aarch64_insn_adsb_type type)
901 {
902 u32 insn;
903
904 switch (type) {
905 case AARCH64_INSN_ADSB_ADD:
906 insn = aarch64_insn_get_add_value();
907 break;
908 case AARCH64_INSN_ADSB_SUB:
909 insn = aarch64_insn_get_sub_value();
910 break;
911 case AARCH64_INSN_ADSB_ADD_SETFLAGS:
912 insn = aarch64_insn_get_adds_value();
913 break;
914 case AARCH64_INSN_ADSB_SUB_SETFLAGS:
915 insn = aarch64_insn_get_subs_value();
916 break;
917 default:
918 pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
919 return AARCH64_BREAK_FAULT;
920 }
921
922 switch (variant) {
923 case AARCH64_INSN_VARIANT_32BIT:
924 if (shift & ~(SZ_32 - 1)) {
925 pr_err("%s: invalid shift encoding %d\n", __func__,
926 shift);
927 return AARCH64_BREAK_FAULT;
928 }
929 break;
930 case AARCH64_INSN_VARIANT_64BIT:
931 insn |= AARCH64_INSN_SF_BIT;
932 if (shift & ~(SZ_64 - 1)) {
933 pr_err("%s: invalid shift encoding %d\n", __func__,
934 shift);
935 return AARCH64_BREAK_FAULT;
936 }
937 break;
938 default:
939 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
940 return AARCH64_BREAK_FAULT;
941 }
942
943
944 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
945
946 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
947
948 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
949
950 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
951 }
952
aarch64_insn_gen_data1(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_variant variant,enum aarch64_insn_data1_type type)953 u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
954 enum aarch64_insn_register src,
955 enum aarch64_insn_variant variant,
956 enum aarch64_insn_data1_type type)
957 {
958 u32 insn;
959
960 switch (type) {
961 case AARCH64_INSN_DATA1_REVERSE_16:
962 insn = aarch64_insn_get_rev16_value();
963 break;
964 case AARCH64_INSN_DATA1_REVERSE_32:
965 insn = aarch64_insn_get_rev32_value();
966 break;
967 case AARCH64_INSN_DATA1_REVERSE_64:
968 if (variant != AARCH64_INSN_VARIANT_64BIT) {
969 pr_err("%s: invalid variant for reverse64 %d\n",
970 __func__, variant);
971 return AARCH64_BREAK_FAULT;
972 }
973 insn = aarch64_insn_get_rev64_value();
974 break;
975 default:
976 pr_err("%s: unknown data1 encoding %d\n", __func__, type);
977 return AARCH64_BREAK_FAULT;
978 }
979
980 switch (variant) {
981 case AARCH64_INSN_VARIANT_32BIT:
982 break;
983 case AARCH64_INSN_VARIANT_64BIT:
984 insn |= AARCH64_INSN_SF_BIT;
985 break;
986 default:
987 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
988 return AARCH64_BREAK_FAULT;
989 }
990
991 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
992
993 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
994 }
995
aarch64_insn_gen_data2(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg,enum aarch64_insn_variant variant,enum aarch64_insn_data2_type type)996 u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst,
997 enum aarch64_insn_register src,
998 enum aarch64_insn_register reg,
999 enum aarch64_insn_variant variant,
1000 enum aarch64_insn_data2_type type)
1001 {
1002 u32 insn;
1003
1004 switch (type) {
1005 case AARCH64_INSN_DATA2_UDIV:
1006 insn = aarch64_insn_get_udiv_value();
1007 break;
1008 case AARCH64_INSN_DATA2_SDIV:
1009 insn = aarch64_insn_get_sdiv_value();
1010 break;
1011 case AARCH64_INSN_DATA2_LSLV:
1012 insn = aarch64_insn_get_lslv_value();
1013 break;
1014 case AARCH64_INSN_DATA2_LSRV:
1015 insn = aarch64_insn_get_lsrv_value();
1016 break;
1017 case AARCH64_INSN_DATA2_ASRV:
1018 insn = aarch64_insn_get_asrv_value();
1019 break;
1020 case AARCH64_INSN_DATA2_RORV:
1021 insn = aarch64_insn_get_rorv_value();
1022 break;
1023 default:
1024 pr_err("%s: unknown data2 encoding %d\n", __func__, type);
1025 return AARCH64_BREAK_FAULT;
1026 }
1027
1028 switch (variant) {
1029 case AARCH64_INSN_VARIANT_32BIT:
1030 break;
1031 case AARCH64_INSN_VARIANT_64BIT:
1032 insn |= AARCH64_INSN_SF_BIT;
1033 break;
1034 default:
1035 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1036 return AARCH64_BREAK_FAULT;
1037 }
1038
1039 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1040
1041 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1042
1043 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1044 }
1045
aarch64_insn_gen_data3(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg1,enum aarch64_insn_register reg2,enum aarch64_insn_variant variant,enum aarch64_insn_data3_type type)1046 u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
1047 enum aarch64_insn_register src,
1048 enum aarch64_insn_register reg1,
1049 enum aarch64_insn_register reg2,
1050 enum aarch64_insn_variant variant,
1051 enum aarch64_insn_data3_type type)
1052 {
1053 u32 insn;
1054
1055 switch (type) {
1056 case AARCH64_INSN_DATA3_MADD:
1057 insn = aarch64_insn_get_madd_value();
1058 break;
1059 case AARCH64_INSN_DATA3_MSUB:
1060 insn = aarch64_insn_get_msub_value();
1061 break;
1062 default:
1063 pr_err("%s: unknown data3 encoding %d\n", __func__, type);
1064 return AARCH64_BREAK_FAULT;
1065 }
1066
1067 switch (variant) {
1068 case AARCH64_INSN_VARIANT_32BIT:
1069 break;
1070 case AARCH64_INSN_VARIANT_64BIT:
1071 insn |= AARCH64_INSN_SF_BIT;
1072 break;
1073 default:
1074 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1075 return AARCH64_BREAK_FAULT;
1076 }
1077
1078 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1079
1080 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src);
1081
1082 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
1083 reg1);
1084
1085 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
1086 reg2);
1087 }
1088
aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg,int shift,enum aarch64_insn_variant variant,enum aarch64_insn_logic_type type)1089 u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
1090 enum aarch64_insn_register src,
1091 enum aarch64_insn_register reg,
1092 int shift,
1093 enum aarch64_insn_variant variant,
1094 enum aarch64_insn_logic_type type)
1095 {
1096 u32 insn;
1097
1098 switch (type) {
1099 case AARCH64_INSN_LOGIC_AND:
1100 insn = aarch64_insn_get_and_value();
1101 break;
1102 case AARCH64_INSN_LOGIC_BIC:
1103 insn = aarch64_insn_get_bic_value();
1104 break;
1105 case AARCH64_INSN_LOGIC_ORR:
1106 insn = aarch64_insn_get_orr_value();
1107 break;
1108 case AARCH64_INSN_LOGIC_ORN:
1109 insn = aarch64_insn_get_orn_value();
1110 break;
1111 case AARCH64_INSN_LOGIC_EOR:
1112 insn = aarch64_insn_get_eor_value();
1113 break;
1114 case AARCH64_INSN_LOGIC_EON:
1115 insn = aarch64_insn_get_eon_value();
1116 break;
1117 case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1118 insn = aarch64_insn_get_ands_value();
1119 break;
1120 case AARCH64_INSN_LOGIC_BIC_SETFLAGS:
1121 insn = aarch64_insn_get_bics_value();
1122 break;
1123 default:
1124 pr_err("%s: unknown logical encoding %d\n", __func__, type);
1125 return AARCH64_BREAK_FAULT;
1126 }
1127
1128 switch (variant) {
1129 case AARCH64_INSN_VARIANT_32BIT:
1130 if (shift & ~(SZ_32 - 1)) {
1131 pr_err("%s: invalid shift encoding %d\n", __func__,
1132 shift);
1133 return AARCH64_BREAK_FAULT;
1134 }
1135 break;
1136 case AARCH64_INSN_VARIANT_64BIT:
1137 insn |= AARCH64_INSN_SF_BIT;
1138 if (shift & ~(SZ_64 - 1)) {
1139 pr_err("%s: invalid shift encoding %d\n", __func__,
1140 shift);
1141 return AARCH64_BREAK_FAULT;
1142 }
1143 break;
1144 default:
1145 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1146 return AARCH64_BREAK_FAULT;
1147 }
1148
1149
1150 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1151
1152 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1153
1154 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1155
1156 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
1157 }
1158
1159 /*
1160 * MOV (register) is architecturally an alias of ORR (shifted register) where
1161 * MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m>
1162 */
aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_variant variant)1163 u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,
1164 enum aarch64_insn_register src,
1165 enum aarch64_insn_variant variant)
1166 {
1167 return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR,
1168 src, 0, variant,
1169 AARCH64_INSN_LOGIC_ORR);
1170 }
1171
aarch64_insn_gen_adr(unsigned long pc,unsigned long addr,enum aarch64_insn_register reg,enum aarch64_insn_adr_type type)1172 u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
1173 enum aarch64_insn_register reg,
1174 enum aarch64_insn_adr_type type)
1175 {
1176 u32 insn;
1177 s32 offset;
1178
1179 switch (type) {
1180 case AARCH64_INSN_ADR_TYPE_ADR:
1181 insn = aarch64_insn_get_adr_value();
1182 offset = addr - pc;
1183 break;
1184 case AARCH64_INSN_ADR_TYPE_ADRP:
1185 insn = aarch64_insn_get_adrp_value();
1186 offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12;
1187 break;
1188 default:
1189 pr_err("%s: unknown adr encoding %d\n", __func__, type);
1190 return AARCH64_BREAK_FAULT;
1191 }
1192
1193 if (offset < -SZ_1M || offset >= SZ_1M)
1194 return AARCH64_BREAK_FAULT;
1195
1196 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg);
1197
1198 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset);
1199 }
1200
1201 /*
1202 * Decode the imm field of a branch, and return the byte offset as a
1203 * signed value (so it can be used when computing a new branch
1204 * target).
1205 */
aarch64_get_branch_offset(u32 insn)1206 s32 aarch64_get_branch_offset(u32 insn)
1207 {
1208 s32 imm;
1209
1210 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) {
1211 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn);
1212 return (imm << 6) >> 4;
1213 }
1214
1215 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1216 aarch64_insn_is_bcond(insn)) {
1217 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn);
1218 return (imm << 13) >> 11;
1219 }
1220
1221 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) {
1222 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn);
1223 return (imm << 18) >> 16;
1224 }
1225
1226 /* Unhandled instruction */
1227 BUG();
1228 }
1229
1230 /*
1231 * Encode the displacement of a branch in the imm field and return the
1232 * updated instruction.
1233 */
aarch64_set_branch_offset(u32 insn,s32 offset)1234 u32 aarch64_set_branch_offset(u32 insn, s32 offset)
1235 {
1236 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn))
1237 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
1238 offset >> 2);
1239
1240 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1241 aarch64_insn_is_bcond(insn))
1242 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
1243 offset >> 2);
1244
1245 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn))
1246 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn,
1247 offset >> 2);
1248
1249 /* Unhandled instruction */
1250 BUG();
1251 }
1252
aarch64_insn_adrp_get_offset(u32 insn)1253 s32 aarch64_insn_adrp_get_offset(u32 insn)
1254 {
1255 BUG_ON(!aarch64_insn_is_adrp(insn));
1256 return aarch64_insn_decode_immediate(AARCH64_INSN_IMM_ADR, insn) << 12;
1257 }
1258
aarch64_insn_adrp_set_offset(u32 insn,s32 offset)1259 u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset)
1260 {
1261 BUG_ON(!aarch64_insn_is_adrp(insn));
1262 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn,
1263 offset >> 12);
1264 }
1265
1266 /*
1267 * Extract the Op/CR data from a msr/mrs instruction.
1268 */
aarch64_insn_extract_system_reg(u32 insn)1269 u32 aarch64_insn_extract_system_reg(u32 insn)
1270 {
1271 return (insn & 0x1FFFE0) >> 5;
1272 }
1273
aarch32_insn_is_wide(u32 insn)1274 bool aarch32_insn_is_wide(u32 insn)
1275 {
1276 return insn >= 0xe800;
1277 }
1278
1279 /*
1280 * Macros/defines for extracting register numbers from instruction.
1281 */
aarch32_insn_extract_reg_num(u32 insn,int offset)1282 u32 aarch32_insn_extract_reg_num(u32 insn, int offset)
1283 {
1284 return (insn & (0xf << offset)) >> offset;
1285 }
1286
1287 #define OPC2_MASK 0x7
1288 #define OPC2_OFFSET 5
aarch32_insn_mcr_extract_opc2(u32 insn)1289 u32 aarch32_insn_mcr_extract_opc2(u32 insn)
1290 {
1291 return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET;
1292 }
1293
1294 #define CRM_MASK 0xf
aarch32_insn_mcr_extract_crm(u32 insn)1295 u32 aarch32_insn_mcr_extract_crm(u32 insn)
1296 {
1297 return insn & CRM_MASK;
1298 }
1299
range_of_ones(u64 val)1300 static bool range_of_ones(u64 val)
1301 {
1302 /* Doesn't handle full ones or full zeroes */
1303 u64 sval = val >> __ffs64(val);
1304
1305 /* One of Sean Eron Anderson's bithack tricks */
1306 return ((sval + 1) & (sval)) == 0;
1307 }
1308
aarch64_encode_immediate(u64 imm,enum aarch64_insn_variant variant,u32 insn)1309 static u32 aarch64_encode_immediate(u64 imm,
1310 enum aarch64_insn_variant variant,
1311 u32 insn)
1312 {
1313 unsigned int immr, imms, n, ones, ror, esz, tmp;
1314 u64 mask;
1315
1316 switch (variant) {
1317 case AARCH64_INSN_VARIANT_32BIT:
1318 esz = 32;
1319 break;
1320 case AARCH64_INSN_VARIANT_64BIT:
1321 insn |= AARCH64_INSN_SF_BIT;
1322 esz = 64;
1323 break;
1324 default:
1325 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1326 return AARCH64_BREAK_FAULT;
1327 }
1328
1329 mask = GENMASK(esz - 1, 0);
1330
1331 /* Can't encode full zeroes, full ones, or value wider than the mask */
1332 if (!imm || imm == mask || imm & ~mask)
1333 return AARCH64_BREAK_FAULT;
1334
1335 /*
1336 * Inverse of Replicate(). Try to spot a repeating pattern
1337 * with a pow2 stride.
1338 */
1339 for (tmp = esz / 2; tmp >= 2; tmp /= 2) {
1340 u64 emask = BIT(tmp) - 1;
1341
1342 if ((imm & emask) != ((imm >> tmp) & emask))
1343 break;
1344
1345 esz = tmp;
1346 mask = emask;
1347 }
1348
1349 /* N is only set if we're encoding a 64bit value */
1350 n = esz == 64;
1351
1352 /* Trim imm to the element size */
1353 imm &= mask;
1354
1355 /* That's how many ones we need to encode */
1356 ones = hweight64(imm);
1357
1358 /*
1359 * imms is set to (ones - 1), prefixed with a string of ones
1360 * and a zero if they fit. Cap it to 6 bits.
1361 */
1362 imms = ones - 1;
1363 imms |= 0xf << ffs(esz);
1364 imms &= BIT(6) - 1;
1365
1366 /* Compute the rotation */
1367 if (range_of_ones(imm)) {
1368 /*
1369 * Pattern: 0..01..10..0
1370 *
1371 * Compute how many rotate we need to align it right
1372 */
1373 ror = __ffs64(imm);
1374 } else {
1375 /*
1376 * Pattern: 0..01..10..01..1
1377 *
1378 * Fill the unused top bits with ones, and check if
1379 * the result is a valid immediate (all ones with a
1380 * contiguous ranges of zeroes).
1381 */
1382 imm |= ~mask;
1383 if (!range_of_ones(~imm))
1384 return AARCH64_BREAK_FAULT;
1385
1386 /*
1387 * Compute the rotation to get a continuous set of
1388 * ones, with the first bit set at position 0
1389 */
1390 ror = fls64(~imm);
1391 }
1392
1393 /*
1394 * immr is the number of bits we need to rotate back to the
1395 * original set of ones. Note that this is relative to the
1396 * element size...
1397 */
1398 immr = (esz - ror) % esz;
1399
1400 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, n);
1401 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
1402 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
1403 }
1404
aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,enum aarch64_insn_variant variant,enum aarch64_insn_register Rn,enum aarch64_insn_register Rd,u64 imm)1405 u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,
1406 enum aarch64_insn_variant variant,
1407 enum aarch64_insn_register Rn,
1408 enum aarch64_insn_register Rd,
1409 u64 imm)
1410 {
1411 u32 insn;
1412
1413 switch (type) {
1414 case AARCH64_INSN_LOGIC_AND:
1415 insn = aarch64_insn_get_and_imm_value();
1416 break;
1417 case AARCH64_INSN_LOGIC_ORR:
1418 insn = aarch64_insn_get_orr_imm_value();
1419 break;
1420 case AARCH64_INSN_LOGIC_EOR:
1421 insn = aarch64_insn_get_eor_imm_value();
1422 break;
1423 case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1424 insn = aarch64_insn_get_ands_imm_value();
1425 break;
1426 default:
1427 pr_err("%s: unknown logical encoding %d\n", __func__, type);
1428 return AARCH64_BREAK_FAULT;
1429 }
1430
1431 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1432 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1433 return aarch64_encode_immediate(imm, variant, insn);
1434 }
1435
aarch64_insn_gen_extr(enum aarch64_insn_variant variant,enum aarch64_insn_register Rm,enum aarch64_insn_register Rn,enum aarch64_insn_register Rd,u8 lsb)1436 u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
1437 enum aarch64_insn_register Rm,
1438 enum aarch64_insn_register Rn,
1439 enum aarch64_insn_register Rd,
1440 u8 lsb)
1441 {
1442 u32 insn;
1443
1444 insn = aarch64_insn_get_extr_value();
1445
1446 switch (variant) {
1447 case AARCH64_INSN_VARIANT_32BIT:
1448 if (lsb > 31)
1449 return AARCH64_BREAK_FAULT;
1450 break;
1451 case AARCH64_INSN_VARIANT_64BIT:
1452 if (lsb > 63)
1453 return AARCH64_BREAK_FAULT;
1454 insn |= AARCH64_INSN_SF_BIT;
1455 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, 1);
1456 break;
1457 default:
1458 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1459 return AARCH64_BREAK_FAULT;
1460 }
1461
1462 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, lsb);
1463 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1464 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1465 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm);
1466 }
1467
aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)1468 u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)
1469 {
1470 u32 opt;
1471 u32 insn;
1472
1473 switch (type) {
1474 case AARCH64_INSN_MB_SY:
1475 opt = 0xf;
1476 break;
1477 case AARCH64_INSN_MB_ST:
1478 opt = 0xe;
1479 break;
1480 case AARCH64_INSN_MB_LD:
1481 opt = 0xd;
1482 break;
1483 case AARCH64_INSN_MB_ISH:
1484 opt = 0xb;
1485 break;
1486 case AARCH64_INSN_MB_ISHST:
1487 opt = 0xa;
1488 break;
1489 case AARCH64_INSN_MB_ISHLD:
1490 opt = 0x9;
1491 break;
1492 case AARCH64_INSN_MB_NSH:
1493 opt = 0x7;
1494 break;
1495 case AARCH64_INSN_MB_NSHST:
1496 opt = 0x6;
1497 break;
1498 case AARCH64_INSN_MB_NSHLD:
1499 opt = 0x5;
1500 break;
1501 default:
1502 pr_err("%s: unknown dmb type %d\n", __func__, type);
1503 return AARCH64_BREAK_FAULT;
1504 }
1505
1506 insn = aarch64_insn_get_dmb_value();
1507 insn &= ~GENMASK(11, 8);
1508 insn |= (opt << 8);
1509
1510 return insn;
1511 }
1512