1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copied from arch/arm64/kernel/cpufeature.c
4 *
5 * Copyright (C) 2015 ARM Ltd.
6 * Copyright (C) 2017 SiFive
7 */
8
9 #include <linux/bitmap.h>
10 #include <linux/ctype.h>
11 #include <linux/libfdt.h>
12 #include <linux/log2.h>
13 #include <linux/memory.h>
14 #include <linux/module.h>
15 #include <linux/of.h>
16 #include <asm/alternative.h>
17 #include <asm/cacheflush.h>
18 #include <asm/errata_list.h>
19 #include <asm/hwcap.h>
20 #include <asm/patch.h>
21 #include <asm/pgtable.h>
22 #include <asm/processor.h>
23 #include <asm/smp.h>
24 #include <asm/switch_to.h>
25
26 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
27
28 unsigned long elf_hwcap __read_mostly;
29
30 /* Host ISA bitmap */
31 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
32
33 /**
34 * riscv_isa_extension_base() - Get base extension word
35 *
36 * @isa_bitmap: ISA bitmap to use
37 * Return: base extension word as unsigned long value
38 *
39 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
40 */
riscv_isa_extension_base(const unsigned long * isa_bitmap)41 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap)
42 {
43 if (!isa_bitmap)
44 return riscv_isa[0];
45 return isa_bitmap[0];
46 }
47 EXPORT_SYMBOL_GPL(riscv_isa_extension_base);
48
49 /**
50 * __riscv_isa_extension_available() - Check whether given extension
51 * is available or not
52 *
53 * @isa_bitmap: ISA bitmap to use
54 * @bit: bit position of the desired extension
55 * Return: true or false
56 *
57 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
58 */
__riscv_isa_extension_available(const unsigned long * isa_bitmap,int bit)59 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit)
60 {
61 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa;
62
63 if (bit >= RISCV_ISA_EXT_MAX)
64 return false;
65
66 return test_bit(bit, bmap) ? true : false;
67 }
68 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
69
riscv_isa_extension_check(int id)70 static bool riscv_isa_extension_check(int id)
71 {
72 switch (id) {
73 case RISCV_ISA_EXT_ZICBOM:
74 if (!riscv_cbom_block_size) {
75 pr_err("Zicbom detected in ISA string, but no cbom-block-size found\n");
76 return false;
77 } else if (!is_power_of_2(riscv_cbom_block_size)) {
78 pr_err("cbom-block-size present, but is not a power-of-2\n");
79 return false;
80 }
81 return true;
82 }
83
84 return true;
85 }
86
riscv_fill_hwcap(void)87 void __init riscv_fill_hwcap(void)
88 {
89 struct device_node *node;
90 const char *isa;
91 char print_str[NUM_ALPHA_EXTS + 1];
92 int i, j, rc;
93 unsigned long isa2hwcap[26] = {0};
94 unsigned long hartid;
95
96 isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;
97 isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;
98 isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A;
99 isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;
100 isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;
101 isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;
102
103 elf_hwcap = 0;
104
105 bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX);
106
107 for_each_of_cpu_node(node) {
108 unsigned long this_hwcap = 0;
109 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX);
110 const char *temp;
111
112 rc = riscv_of_processor_hartid(node, &hartid);
113 if (rc < 0)
114 continue;
115
116 if (of_property_read_string(node, "riscv,isa", &isa)) {
117 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
118 continue;
119 }
120
121 temp = isa;
122 #if IS_ENABLED(CONFIG_32BIT)
123 if (!strncmp(isa, "rv32", 4))
124 isa += 4;
125 #elif IS_ENABLED(CONFIG_64BIT)
126 if (!strncmp(isa, "rv64", 4))
127 isa += 4;
128 #endif
129 /* The riscv,isa DT property must start with rv64 or rv32 */
130 if (temp == isa)
131 continue;
132 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX);
133 for (; *isa; ++isa) {
134 const char *ext = isa++;
135 const char *ext_end = isa;
136 bool ext_long = false, ext_err = false;
137
138 switch (*ext) {
139 case 's':
140 /**
141 * Workaround for invalid single-letter 's' & 'u'(QEMU).
142 * No need to set the bit in riscv_isa as 's' & 'u' are
143 * not valid ISA extensions. It works until multi-letter
144 * extension starting with "Su" appears.
145 */
146 if (ext[-1] != '_' && ext[1] == 'u') {
147 ++isa;
148 ext_err = true;
149 break;
150 }
151 fallthrough;
152 case 'x':
153 case 'z':
154 ext_long = true;
155 /* Multi-letter extension must be delimited */
156 for (; *isa && *isa != '_'; ++isa)
157 if (unlikely(!islower(*isa)
158 && !isdigit(*isa)))
159 ext_err = true;
160 /* Parse backwards */
161 ext_end = isa;
162 if (unlikely(ext_err))
163 break;
164 if (!isdigit(ext_end[-1]))
165 break;
166 /* Skip the minor version */
167 while (isdigit(*--ext_end))
168 ;
169 if (ext_end[0] != 'p'
170 || !isdigit(ext_end[-1])) {
171 /* Advance it to offset the pre-decrement */
172 ++ext_end;
173 break;
174 }
175 /* Skip the major version */
176 while (isdigit(*--ext_end))
177 ;
178 ++ext_end;
179 break;
180 default:
181 if (unlikely(!islower(*ext))) {
182 ext_err = true;
183 break;
184 }
185 /* Find next extension */
186 if (!isdigit(*isa))
187 break;
188 /* Skip the minor version */
189 while (isdigit(*++isa))
190 ;
191 if (*isa != 'p')
192 break;
193 if (!isdigit(*++isa)) {
194 --isa;
195 break;
196 }
197 /* Skip the major version */
198 while (isdigit(*++isa))
199 ;
200 break;
201 }
202 if (*isa != '_')
203 --isa;
204
205 #define SET_ISA_EXT_MAP(name, bit) \
206 do { \
207 if ((ext_end - ext == sizeof(name) - 1) && \
208 !memcmp(ext, name, sizeof(name) - 1) && \
209 riscv_isa_extension_check(bit)) \
210 set_bit(bit, this_isa); \
211 } while (false) \
212
213 if (unlikely(ext_err))
214 continue;
215 if (!ext_long) {
216 int nr = *ext - 'a';
217
218 if (riscv_isa_extension_check(nr)) {
219 this_hwcap |= isa2hwcap[nr];
220 set_bit(nr, this_isa);
221 }
222 } else {
223 /* sorted alphabetically */
224 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF);
225 SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC);
226 SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL);
227 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT);
228 SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB);
229 SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM);
230 SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE);
231 }
232 #undef SET_ISA_EXT_MAP
233 }
234
235 /*
236 * All "okay" hart should have same isa. Set HWCAP based on
237 * common capabilities of every "okay" hart, in case they don't
238 * have.
239 */
240 if (elf_hwcap)
241 elf_hwcap &= this_hwcap;
242 else
243 elf_hwcap = this_hwcap;
244
245 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
246 bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
247 else
248 bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
249 }
250
251 /* We don't support systems with F but without D, so mask those out
252 * here. */
253 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) {
254 pr_info("This kernel does not support systems with F but not D\n");
255 elf_hwcap &= ~COMPAT_HWCAP_ISA_F;
256 }
257
258 memset(print_str, 0, sizeof(print_str));
259 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
260 if (riscv_isa[0] & BIT_MASK(i))
261 print_str[j++] = (char)('a' + i);
262 pr_info("riscv: base ISA extensions %s\n", print_str);
263
264 memset(print_str, 0, sizeof(print_str));
265 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
266 if (elf_hwcap & BIT_MASK(i))
267 print_str[j++] = (char)('a' + i);
268 pr_info("riscv: ELF capabilities %s\n", print_str);
269 }
270
271 #ifdef CONFIG_RISCV_ALTERNATIVE
riscv_cpufeature_patch_func(struct alt_entry * begin,struct alt_entry * end,unsigned int stage)272 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
273 struct alt_entry *end,
274 unsigned int stage)
275 {
276 struct alt_entry *alt;
277 void *oldptr, *altptr;
278
279 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
280 return;
281
282 for (alt = begin; alt < end; alt++) {
283 if (alt->vendor_id != 0)
284 continue;
285 if (alt->errata_id >= RISCV_ISA_EXT_MAX) {
286 WARN(1, "This extension id:%d is not in ISA extension list",
287 alt->errata_id);
288 continue;
289 }
290
291 if (!__riscv_isa_extension_available(NULL, alt->errata_id))
292 continue;
293
294 oldptr = ALT_OLD_PTR(alt);
295 altptr = ALT_ALT_PTR(alt);
296
297 mutex_lock(&text_mutex);
298 patch_text_nosync(oldptr, altptr, alt->alt_len);
299 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);
300 mutex_unlock(&text_mutex);
301 }
302 }
303 #endif
304