1 /* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
2 because MTRRs can span upto 40 bits (36bits on most modern x86) */
3 #include <xen/lib.h>
4 #include <xen/init.h>
5 #include <xen/mm.h>
6 #include <xen/stdbool.h>
7 #include <asm/flushtlb.h>
8 #include <asm/io.h>
9 #include <asm/mtrr.h>
10 #include <asm/msr.h>
11 #include <asm/system.h>
12 #include <asm/cpufeature.h>
13 #include "mtrr.h"
14
15 static const struct fixed_range_block {
16 uint32_t base_msr; /* start address of an MTRR block */
17 unsigned int ranges; /* number of MTRRs in this block */
18 } fixed_range_blocks[] = {
19 { MSR_MTRRfix64K_00000, (0x80000 - 0x00000) >> (16 + 3) },
20 { MSR_MTRRfix16K_80000, (0xC0000 - 0x80000) >> (14 + 3) },
21 { MSR_MTRRfix4K_C0000, (0x100000 - 0xC0000) >> (12 + 3) },
22 {}
23 };
24
25 static unsigned long smp_changes_mask;
26 struct mtrr_state mtrr_state = {};
27
28 /* Get the MSR pair relating to a var range */
29 static void
get_mtrr_var_range(unsigned int index,struct mtrr_var_range * vr)30 get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
31 {
32 rdmsrl(MSR_IA32_MTRR_PHYSBASE(index), vr->base);
33 rdmsrl(MSR_IA32_MTRR_PHYSMASK(index), vr->mask);
34 }
35
36 static void
get_fixed_ranges(mtrr_type * frs)37 get_fixed_ranges(mtrr_type * frs)
38 {
39 uint64_t *p = (uint64_t *) frs;
40 const struct fixed_range_block *block;
41
42 if (!mtrr_state.have_fixed)
43 return;
44
45 for (block = fixed_range_blocks; block->ranges; ++block) {
46 unsigned int i;
47
48 for (i = 0; i < block->ranges; ++i, ++p)
49 rdmsrl(block->base_msr + i, *p);
50 }
51 }
52
mtrr_save_fixed_ranges(void * info)53 void mtrr_save_fixed_ranges(void *info)
54 {
55 get_fixed_ranges(mtrr_state.fixed_ranges);
56 }
57
58 /* Grab all of the MTRR state for this CPU into *state */
get_mtrr_state(void)59 void __init get_mtrr_state(void)
60 {
61 unsigned int i;
62 struct mtrr_var_range *vrs;
63 uint64_t msr_content;
64
65 if (!mtrr_state.var_ranges) {
66 mtrr_state.var_ranges = xmalloc_array(struct mtrr_var_range,
67 num_var_ranges);
68 if (!mtrr_state.var_ranges)
69 return;
70 }
71 vrs = mtrr_state.var_ranges;
72
73 rdmsrl(MSR_MTRRcap, msr_content);
74 mtrr_state.have_fixed = (msr_content >> 8) & 1;
75
76 for (i = 0; i < num_var_ranges; i++)
77 get_mtrr_var_range(i, &vrs[i]);
78 get_fixed_ranges(mtrr_state.fixed_ranges);
79
80 rdmsrl(MSR_MTRRdefType, msr_content);
81 mtrr_state.def_type = (msr_content & 0xff);
82 mtrr_state.enabled = (msr_content & 0xc00) >> 10;
83
84 /* Store mtrr_cap for HVM MTRR virtualisation. */
85 rdmsrl(MSR_MTRRcap, mtrr_state.mtrr_cap);
86 }
87
88 static bool_t __initdata mtrr_show;
89 boolean_param("mtrr.show", mtrr_show);
90
mtrr_attrib_to_str(mtrr_type x)91 static const char *__init mtrr_attrib_to_str(mtrr_type x)
92 {
93 static const char __initconst strings[MTRR_NUM_TYPES][16] =
94 {
95 [MTRR_TYPE_UNCACHABLE] = "uncachable",
96 [MTRR_TYPE_WRCOMB] = "write-combining",
97 [MTRR_TYPE_WRTHROUGH] = "write-through",
98 [MTRR_TYPE_WRPROT] = "write-protect",
99 [MTRR_TYPE_WRBACK] = "write-back",
100 };
101
102 return (x < ARRAY_SIZE(strings) && strings[x][0]) ? strings[x] : "?";
103 }
104
105 static unsigned int __initdata last_fixed_start;
106 static unsigned int __initdata last_fixed_end;
107 static mtrr_type __initdata last_fixed_type;
108
print_fixed_last(const char * level)109 static void __init print_fixed_last(const char *level)
110 {
111 if (!last_fixed_end)
112 return;
113
114 printk("%s %05x-%05x %s\n", level, last_fixed_start,
115 last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
116
117 last_fixed_end = 0;
118 }
119
update_fixed_last(unsigned int base,unsigned int end,mtrr_type type)120 static void __init update_fixed_last(unsigned int base, unsigned int end,
121 mtrr_type type)
122 {
123 last_fixed_start = base;
124 last_fixed_end = end;
125 last_fixed_type = type;
126 }
127
print_fixed(unsigned int base,unsigned int step,const mtrr_type * types,const char * level)128 static void __init print_fixed(unsigned int base, unsigned int step,
129 const mtrr_type *types, const char *level)
130 {
131 unsigned i;
132
133 for (i = 0; i < 8; ++i, ++types, base += step) {
134 if (last_fixed_end == 0) {
135 update_fixed_last(base, base + step, *types);
136 continue;
137 }
138 if (last_fixed_end == base && last_fixed_type == *types) {
139 last_fixed_end = base + step;
140 continue;
141 }
142 /* new segments: gap or different type */
143 print_fixed_last(level);
144 update_fixed_last(base, base + step, *types);
145 }
146 }
147
print_mtrr_state(const char * level)148 static void __init print_mtrr_state(const char *level)
149 {
150 unsigned int i;
151 int width;
152
153 printk("%sMTRR default type: %s\n", level,
154 mtrr_attrib_to_str(mtrr_state.def_type));
155 if (mtrr_state.have_fixed) {
156 const mtrr_type *fr = mtrr_state.fixed_ranges;
157 const struct fixed_range_block *block = fixed_range_blocks;
158 unsigned int base = 0, step = 0x10000;
159
160 printk("%sMTRR fixed ranges %sabled:\n", level,
161 mtrr_state.enabled & 1 ? "en" : "dis");
162 for (; block->ranges; ++block, step >>= 2) {
163 for (i = 0; i < block->ranges; ++i, fr += 8) {
164 print_fixed(base, step, fr, level);
165 base += 8 * step;
166 }
167 }
168 print_fixed_last(level);
169 }
170 printk("%sMTRR variable ranges %sabled:\n", level,
171 mtrr_state.enabled & 2 ? "en" : "dis");
172 width = (paddr_bits - PAGE_SHIFT + 3) / 4;
173
174 for (i = 0; i < num_var_ranges; ++i) {
175 if (mtrr_state.var_ranges[i].mask & MTRR_PHYSMASK_VALID)
176 printk("%s %u base %0*"PRIx64"000 mask %0*"PRIx64"000 %s\n",
177 level, i,
178 width, mtrr_state.var_ranges[i].base >> 12,
179 width, mtrr_state.var_ranges[i].mask >> 12,
180 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base &
181 MTRR_PHYSBASE_TYPE_MASK));
182 else
183 printk("%s %u disabled\n", level, i);
184 }
185
186 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD
187 && boot_cpu_data.x86 >= 0xf) {
188 uint64_t syscfg, tom2;
189
190 rdmsrl(MSR_K8_SYSCFG, syscfg);
191 if (syscfg & (1 << 21)) {
192 rdmsrl(MSR_K8_TOP_MEM2, tom2);
193 printk("%sTOM2: %012"PRIx64"%s\n", level, tom2,
194 syscfg & (1 << 22) ? " (WB)" : "");
195 }
196 }
197 }
198
199 /* Some BIOS's are fucked and don't set all MTRRs the same! */
mtrr_state_warn(void)200 void __init mtrr_state_warn(void)
201 {
202 unsigned long mask = smp_changes_mask;
203
204 if (mtrr_show)
205 print_mtrr_state(mask ? KERN_WARNING : "");
206 if (!mask)
207 return;
208 if (mask & MTRR_CHANGE_MASK_FIXED)
209 printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n");
210 if (mask & MTRR_CHANGE_MASK_VARIABLE)
211 printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n");
212 if (mask & MTRR_CHANGE_MASK_DEFTYPE)
213 printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n");
214 printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
215 printk(KERN_INFO "mtrr: corrected configuration.\n");
216 if (!mtrr_show)
217 print_mtrr_state(KERN_INFO);
218 }
219
220 /* Doesn't attempt to pass an error out to MTRR users
221 because it's quite complicated in some cases and probably not
222 worth it because the best error handling is to ignore it. */
mtrr_wrmsr(unsigned int msr,uint64_t msr_content)223 static void mtrr_wrmsr(unsigned int msr, uint64_t msr_content)
224 {
225 if (wrmsr_safe(msr, msr_content) < 0)
226 printk(KERN_ERR
227 "MTRR: CPU %u: Writing MSR %x to %"PRIx64" failed\n",
228 smp_processor_id(), msr, msr_content);
229 /* Cache overlap status for efficient HVM MTRR virtualisation. */
230 mtrr_state.overlapped = is_var_mtrr_overlapped(&mtrr_state);
231 }
232
233 /**
234 * Checks and updates an fixed-range MTRR if it differs from the value it
235 * should have. If K8 extenstions are wanted, update the K8 SYSCFG MSR also.
236 * see AMD publication no. 24593, chapter 7.8.1, page 233 for more information
237 * \param msr MSR address of the MTTR which should be checked and updated
238 * \param changed pointer which indicates whether the MTRR needed to be changed
239 * \param msrwords pointer to the MSR values which the MSR should have
240 */
set_fixed_range(int msr,bool * changed,unsigned int * msrwords)241 static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
242 {
243 uint64_t msr_content, val;
244
245 rdmsrl(msr, msr_content);
246 val = ((uint64_t)msrwords[1] << 32) | msrwords[0];
247
248 if (msr_content != val) {
249 mtrr_wrmsr(msr, val);
250 *changed = true;
251 }
252 }
253
generic_get_free_region(unsigned long base,unsigned long size,int replace_reg)254 int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
255 /* [SUMMARY] Get a free MTRR.
256 <base> The starting (base) address of the region.
257 <size> The size (in bytes) of the region.
258 [RETURNS] The index of the region on success, else -1 on error.
259 */
260 {
261 int i, max;
262 mtrr_type ltype;
263 unsigned long lbase, lsize;
264
265 max = num_var_ranges;
266 if (replace_reg >= 0 && replace_reg < max)
267 return replace_reg;
268 for (i = 0; i < max; ++i) {
269 mtrr_if->get(i, &lbase, &lsize, <ype);
270 if (lsize == 0)
271 return i;
272 }
273 return -ENOSPC;
274 }
275
generic_get_mtrr(unsigned int reg,unsigned long * base,unsigned long * size,mtrr_type * type)276 static void generic_get_mtrr(unsigned int reg, unsigned long *base,
277 unsigned long *size, mtrr_type *type)
278 {
279 uint64_t _mask, _base;
280
281 rdmsrl(MSR_IA32_MTRR_PHYSMASK(reg), _mask);
282 if (!(_mask & MTRR_PHYSMASK_VALID)) {
283 /* Invalid (i.e. free) range */
284 *base = 0;
285 *size = 0;
286 *type = 0;
287 return;
288 }
289
290 rdmsrl(MSR_IA32_MTRR_PHYSBASE(reg), _base);
291
292 /* Work out the shifted address mask. */
293 _mask = size_or_mask | (_mask >> PAGE_SHIFT);
294
295 /* This works correctly if size is a power of two, i.e. a
296 contiguous range. */
297 *size = -(uint32_t)_mask;
298 *base = _base >> PAGE_SHIFT;
299 *type = _base & 0xff;
300 }
301
302 /**
303 * Checks and updates the fixed-range MTRRs if they differ from the saved set
304 * \param frs pointer to fixed-range MTRR values, saved by get_fixed_ranges()
305 */
set_fixed_ranges(mtrr_type * frs)306 static bool set_fixed_ranges(mtrr_type *frs)
307 {
308 unsigned long long *saved = (unsigned long long *) frs;
309 bool changed = false;
310 int block=-1, range;
311
312 while (fixed_range_blocks[++block].ranges)
313 for (range=0; range < fixed_range_blocks[block].ranges; range++)
314 set_fixed_range(fixed_range_blocks[block].base_msr + range,
315 &changed, (unsigned int *) saved++);
316
317 return changed;
318 }
319
320 /* Set the MSR pair relating to a var range. Returns true if
321 changes are made */
set_mtrr_var_ranges(unsigned int index,struct mtrr_var_range * vr)322 static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
323 {
324 uint32_t lo, hi, base_lo, base_hi, mask_lo, mask_hi;
325 uint64_t msr_content;
326 bool changed = false;
327
328 rdmsrl(MSR_IA32_MTRR_PHYSBASE(index), msr_content);
329 lo = (uint32_t)msr_content;
330 hi = (uint32_t)(msr_content >> 32);
331 base_lo = (uint32_t)vr->base;
332 base_hi = (uint32_t)(vr->base >> 32);
333
334 lo &= 0xfffff0ffUL;
335 base_lo &= 0xfffff0ffUL;
336 hi &= size_and_mask >> (32 - PAGE_SHIFT);
337 base_hi &= size_and_mask >> (32 - PAGE_SHIFT);
338
339 if ((base_lo != lo) || (base_hi != hi)) {
340 mtrr_wrmsr(MSR_IA32_MTRR_PHYSBASE(index), vr->base);
341 changed = true;
342 }
343
344 rdmsrl(MSR_IA32_MTRR_PHYSMASK(index), msr_content);
345 lo = (uint32_t)msr_content;
346 hi = (uint32_t)(msr_content >> 32);
347 mask_lo = (uint32_t)vr->mask;
348 mask_hi = (uint32_t)(vr->mask >> 32);
349
350 lo &= 0xfffff800UL;
351 mask_lo &= 0xfffff800UL;
352 hi &= size_and_mask >> (32 - PAGE_SHIFT);
353 mask_hi &= size_and_mask >> (32 - PAGE_SHIFT);
354
355 if ((mask_lo != lo) || (mask_hi != hi)) {
356 mtrr_wrmsr(MSR_IA32_MTRR_PHYSMASK(index), vr->mask);
357 changed = true;
358 }
359 return changed;
360 }
361
362 static uint64_t deftype;
363
set_mtrr_state(void)364 static unsigned long set_mtrr_state(void)
365 /* [SUMMARY] Set the MTRR state for this CPU.
366 <state> The MTRR state information to read.
367 <ctxt> Some relevant CPU context.
368 [NOTE] The CPU must already be in a safe state for MTRR changes.
369 [RETURNS] 0 if no changes made, else a mask indication what was changed.
370 */
371 {
372 unsigned int i;
373 unsigned long change_mask = 0;
374
375 for (i = 0; i < num_var_ranges; i++)
376 if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
377 change_mask |= MTRR_CHANGE_MASK_VARIABLE;
378
379 if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
380 change_mask |= MTRR_CHANGE_MASK_FIXED;
381
382 /* Set_mtrr_restore restores the old value of MTRRdefType,
383 so to set it we fiddle with the saved value */
384 if ((deftype & 0xff) != mtrr_state.def_type
385 || ((deftype & 0xc00) >> 10) != mtrr_state.enabled) {
386 deftype = (deftype & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10);
387 change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
388 }
389
390 return change_mask;
391 }
392
393
394 static DEFINE_SPINLOCK(set_atomicity_lock);
395
396 /*
397 * Since we are disabling the cache don't allow any interrupts - they
398 * would run extremely slow and would only increase the pain. The caller must
399 * ensure that local interrupts are disabled and are reenabled after post_set()
400 * has been called.
401 */
402
prepare_set(void)403 static void prepare_set(void)
404 {
405 /* Note that this is not ideal, since the cache is only flushed/disabled
406 for this CPU while the MTRRs are changed, but changing this requires
407 more invasive changes to the way the kernel boots */
408
409 spin_lock(&set_atomicity_lock);
410
411 /* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
412 write_cr0(read_cr0() | X86_CR0_CD);
413 wbinvd();
414
415 /* TLB flushing here relies on Xen always using CR4.PGE. */
416 BUILD_BUG_ON(!(XEN_MINIMAL_CR4 & X86_CR4_PGE));
417 write_cr4(read_cr4() & ~X86_CR4_PGE);
418
419 /* Save MTRR state */
420 rdmsrl(MSR_MTRRdefType, deftype);
421
422 /* Disable MTRRs, and set the default type to uncached */
423 mtrr_wrmsr(MSR_MTRRdefType, deftype & ~0xcff);
424 }
425
post_set(void)426 static void post_set(void)
427 {
428 /* Intel (P6) standard MTRRs */
429 mtrr_wrmsr(MSR_MTRRdefType, deftype);
430
431 /* Enable caches */
432 write_cr0(read_cr0() & ~X86_CR0_CD);
433
434 /* Reenable CR4.PGE (also flushes the TLB) */
435 write_cr4(read_cr4() | X86_CR4_PGE);
436
437 spin_unlock(&set_atomicity_lock);
438 }
439
generic_set_all(void)440 static void generic_set_all(void)
441 {
442 unsigned long mask, count;
443 unsigned long flags;
444
445 local_irq_save(flags);
446 prepare_set();
447
448 /* Actually set the state */
449 mask = set_mtrr_state();
450
451 post_set();
452 local_irq_restore(flags);
453
454 /* Use the atomic bitops to update the global mask */
455 for (count = 0; count < sizeof mask * 8; ++count) {
456 if (mask & 0x01)
457 set_bit(count, &smp_changes_mask);
458 mask >>= 1;
459 }
460
461 }
462
generic_set_mtrr(unsigned int reg,unsigned long base,unsigned long size,mtrr_type type)463 static void generic_set_mtrr(unsigned int reg, unsigned long base,
464 unsigned long size, mtrr_type type)
465 /* [SUMMARY] Set variable MTRR register on the local CPU.
466 <reg> The register to set.
467 <base> The base address of the region.
468 <size> The size of the region. If this is 0 the region is disabled.
469 <type> The type of the region.
470 <do_safe> If true, do the change safely. If false, safety measures should
471 be done externally.
472 [RETURNS] Nothing.
473 */
474 {
475 unsigned long flags;
476 struct mtrr_var_range *vr;
477
478 vr = &mtrr_state.var_ranges[reg];
479
480 local_irq_save(flags);
481 prepare_set();
482
483 if (size == 0) {
484 /* The invalid bit is kept in the mask, so we simply clear the
485 relevant mask register to disable a range. */
486 memset(vr, 0, sizeof(*vr));
487 mtrr_wrmsr(MSR_IA32_MTRR_PHYSMASK(reg), 0);
488 } else {
489 uint32_t base_lo, base_hi, mask_lo, mask_hi;
490
491 base_lo = base << PAGE_SHIFT | type;
492 base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
493 mask_lo = (-size << PAGE_SHIFT) | MTRR_PHYSMASK_VALID;
494 mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);
495 vr->base = ((uint64_t)base_hi << 32) | base_lo;
496 vr->mask = ((uint64_t)mask_hi << 32) | mask_lo;
497
498 mtrr_wrmsr(MSR_IA32_MTRR_PHYSBASE(reg), vr->base);
499 mtrr_wrmsr(MSR_IA32_MTRR_PHYSMASK(reg), vr->mask);
500 }
501
502 post_set();
503 local_irq_restore(flags);
504 }
505
generic_validate_add_page(unsigned long base,unsigned long size,unsigned int type)506 int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
507 {
508 unsigned long lbase, last;
509
510 /* For Intel PPro stepping <= 7, must be 4 MiB aligned
511 and not touch 0x70000000->0x7003FFFF */
512 if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
513 boot_cpu_data.x86_model == 1 &&
514 boot_cpu_data.x86_mask <= 7) {
515 if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
516 printk(KERN_WARNING "mtrr: base(%#lx000) is not 4 MiB aligned\n", base);
517 return -EINVAL;
518 }
519 if (!(base + size < 0x70000 || base > 0x7003F) &&
520 (type == MTRR_TYPE_WRCOMB
521 || type == MTRR_TYPE_WRBACK)) {
522 printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
523 return -EINVAL;
524 }
525 }
526
527 /* Check upper bits of base and last are equal and lower bits are 0
528 for base and 1 for last */
529 last = base + size - 1;
530 for (lbase = base; !(lbase & 1) && (last & 1);
531 lbase = lbase >> 1, last = last >> 1) ;
532 if (lbase != last) {
533 printk(KERN_WARNING "mtrr: base(%#lx000) is not aligned on a size(%#lx000) boundary\n",
534 base, size);
535 return -EINVAL;
536 }
537 return 0;
538 }
539
540
generic_have_wrcomb(void)541 static int generic_have_wrcomb(void)
542 {
543 unsigned long config;
544 rdmsrl(MSR_MTRRcap, config);
545 return (config & (1ULL << 10));
546 }
547
548 /* generic structure...
549 */
550 const struct mtrr_ops generic_mtrr_ops = {
551 .use_intel_if = true,
552 .set_all = generic_set_all,
553 .get = generic_get_mtrr,
554 .get_free_region = generic_get_free_region,
555 .set = generic_set_mtrr,
556 .validate_add_page = generic_validate_add_page,
557 .have_wrcomb = generic_have_wrcomb,
558 };
559