1 /*
2 * Copyright (c) 2010, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Author: Allen Kay <allen.m.kay@intel.com>
17 */
18
19 #include <xen/irq.h>
20 #include <xen/sched.h>
21 #include <xen/xmalloc.h>
22 #include <xen/domain_page.h>
23 #include <xen/iommu.h>
24 #include <xen/numa.h>
25 #include <xen/softirq.h>
26 #include <xen/time.h>
27 #include <xen/pci.h>
28 #include <xen/pci_ids.h>
29 #include <xen/pci_regs.h>
30 #include <xen/keyhandler.h>
31 #include <asm/msi.h>
32 #include <asm/irq.h>
33 #include <asm/pci.h>
34 #include <mach_apic.h>
35 #include "iommu.h"
36 #include "dmar.h"
37 #include "extern.h"
38 #include "vtd.h"
39
40 #define IOH_DEV 0
41 #define IGD_DEV 2
42
43 #define IGD_BAR_MASK 0xFFFFFFFFFFFF0000
44 #define GGC 0x52
45 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
46
47 #define IS_CTG(id) (id == 0x2a408086)
48 #define IS_ILK(id) (id == 0x00408086 || id == 0x00448086 || id== 0x00628086 || id == 0x006A8086)
49 #define IS_CPT(id) (id == 0x01008086 || id == 0x01048086)
50
51 /* SandyBridge IGD timeouts in milliseconds */
52 #define SNB_IGD_TIMEOUT_LEGACY 1000
53 #define SNB_IGD_TIMEOUT 670
54 static unsigned int snb_igd_timeout;
55
56 static u32 __read_mostly ioh_id;
57 static u32 __initdata igd_id;
58 bool_t __read_mostly rwbf_quirk;
59 static bool_t __read_mostly is_cantiga_b3;
60 static bool_t __read_mostly is_snb_gfx;
61 static u8 *__read_mostly igd_reg_va;
62 static spinlock_t igd_lock;
63
64 /*
65 * QUIRK to workaround Xen boot issue on Calpella/Ironlake OEM BIOS
66 * not enabling VT-d properly in IGD. The workaround is to not enabling
67 * IGD VT-d translation if VT is not enabled in IGD.
68 */
is_igd_vt_enabled_quirk(void)69 int is_igd_vt_enabled_quirk(void)
70 {
71 u16 ggc;
72
73 if ( !IS_ILK(ioh_id) )
74 return 1;
75
76 /* integrated graphics on Intel platforms is located at 0:2.0 */
77 ggc = pci_conf_read16(0, 0, IGD_DEV, 0, GGC);
78 return ( ggc & GGC_MEMORY_VT_ENABLED ? 1 : 0 );
79 }
80
81 /*
82 * QUIRK to workaround cantiga VT-d buffer flush issue.
83 * The workaround is to force write buffer flush even if
84 * VT-d capability indicates it is not required.
85 */
cantiga_b3_errata_init(void)86 static void __init cantiga_b3_errata_init(void)
87 {
88 u16 vid;
89 u8 did_hi, rid;
90
91 vid = pci_conf_read16(0, 0, IGD_DEV, 0, 0);
92 if ( vid != 0x8086 )
93 return;
94
95 did_hi = pci_conf_read8(0, 0, IGD_DEV, 0, 3);
96 rid = pci_conf_read8(0, 0, IGD_DEV, 0, 8);
97
98 if ( (did_hi == 0x2A) && (rid == 0x7) )
99 is_cantiga_b3 = 1;
100 }
101
102 /* check for Sandybridge IGD device ID's */
snb_errata_init(void)103 static void __init snb_errata_init(void)
104 {
105 is_snb_gfx = IS_SNB_GFX(igd_id);
106 spin_lock_init(&igd_lock);
107 }
108
109 /*
110 * QUIRK to workaround Cantiga IGD VT-d low power errata.
111 * This errata impacts IGD assignment on Cantiga systems
112 * and can potentially cause VT-d operations to hang.
113 * The workaround is to access an IGD PCI config register
114 * to get IGD out of low power state before VT-d translation
115 * enable/disable and IOTLB flushes.
116 */
117
118 /*
119 * map IGD MMIO+0x2000 page to allow Xen access to IGD 3D register.
120 */
map_igd_reg(void)121 static void __init map_igd_reg(void)
122 {
123 u64 igd_mmio;
124
125 if ( !is_cantiga_b3 && !is_snb_gfx )
126 return;
127
128 if ( igd_reg_va )
129 return;
130
131 igd_mmio = pci_conf_read32(0, 0, IGD_DEV, 0, PCI_BASE_ADDRESS_1);
132 igd_mmio <<= 32;
133 igd_mmio += pci_conf_read32(0, 0, IGD_DEV, 0, PCI_BASE_ADDRESS_0);
134 igd_reg_va = ioremap(igd_mmio & IGD_BAR_MASK, 0x3000);
135 }
136
137 /*
138 * force IGD to exit low power mode by accessing a IGD 3D regsiter.
139 */
cantiga_vtd_ops_preamble(struct iommu * iommu)140 static int cantiga_vtd_ops_preamble(struct iommu* iommu)
141 {
142 struct intel_iommu *intel = iommu->intel;
143 struct acpi_drhd_unit *drhd = intel ? intel->drhd : NULL;
144
145 if ( !is_igd_drhd(drhd) || !is_cantiga_b3 )
146 return 0;
147
148 if ( !igd_reg_va )
149 return 0;
150
151 /*
152 * Read IGD register at IGD MMIO + 0x20A4 to force IGD
153 * to exit low power state.
154 */
155 return *(volatile int *)(igd_reg_va + 0x20A4);
156 }
157
158 /*
159 * Sandybridge RC6 power management inhibit state erratum.
160 * This can cause power high power consumption.
161 * Workaround is to prevent graphics get into RC6
162 * state when doing VT-d IOTLB operations, do the VT-d
163 * IOTLB operation, and then re-enable RC6 state.
164 *
165 * This quirk is enabled with the snb_igd_quirk command
166 * line parameter. Specifying snb_igd_quirk with no value
167 * (or any of the standard boolean values) enables this
168 * quirk and sets the timeout to the legacy timeout of
169 * 1000 msec. Setting this parameter to the string
170 * "cap" enables this quirk and sets the timeout to
171 * the theoretical maximum of 670 msec. Setting this
172 * parameter to a numerical value enables the quirk and
173 * sets the timeout to that numerical number of msecs.
174 */
snb_vtd_ops_preamble(struct iommu * iommu)175 static void snb_vtd_ops_preamble(struct iommu* iommu)
176 {
177 struct intel_iommu *intel = iommu->intel;
178 struct acpi_drhd_unit *drhd = intel ? intel->drhd : NULL;
179 s_time_t start_time;
180
181 if ( !is_igd_drhd(drhd) || !is_snb_gfx )
182 return;
183
184 if ( !igd_reg_va )
185 return;
186
187 *(volatile u32 *)(igd_reg_va + 0x2054) = 0x000FFFFF;
188 *(volatile u32 *)(igd_reg_va + 0x2700) = 0;
189
190 start_time = NOW();
191 while ( (*(volatile u32 *)(igd_reg_va + 0x22AC) & 0xF) != 0 )
192 {
193 if ( NOW() > start_time + snb_igd_timeout )
194 {
195 dprintk(XENLOG_INFO VTDPREFIX,
196 "snb_vtd_ops_preamble: failed to disable idle handshake\n");
197 break;
198 }
199 cpu_relax();
200 }
201
202 *(volatile u32 *)(igd_reg_va + 0x2050) = 0x10001;
203 }
204
snb_vtd_ops_postamble(struct iommu * iommu)205 static void snb_vtd_ops_postamble(struct iommu* iommu)
206 {
207 struct intel_iommu *intel = iommu->intel;
208 struct acpi_drhd_unit *drhd = intel ? intel->drhd : NULL;
209
210 if ( !is_igd_drhd(drhd) || !is_snb_gfx )
211 return;
212
213 if ( !igd_reg_va )
214 return;
215
216 *(volatile u32 *)(igd_reg_va + 0x2054) = 0xA;
217 *(volatile u32 *)(igd_reg_va + 0x2050) = 0x10000;
218 }
219
220 /*
221 * call before VT-d translation enable and IOTLB flush operations.
222 */
223
vtd_ops_preamble_quirk(struct iommu * iommu)224 void vtd_ops_preamble_quirk(struct iommu* iommu)
225 {
226 cantiga_vtd_ops_preamble(iommu);
227 if ( snb_igd_timeout != 0 )
228 {
229 spin_lock(&igd_lock);
230
231 /* match unlock in postamble */
232 snb_vtd_ops_preamble(iommu);
233 }
234 }
235
236 /*
237 * call after VT-d translation enable and IOTLB flush operations.
238 */
vtd_ops_postamble_quirk(struct iommu * iommu)239 void vtd_ops_postamble_quirk(struct iommu* iommu)
240 {
241 if ( snb_igd_timeout != 0 )
242 {
243 snb_vtd_ops_postamble(iommu);
244
245 /* match the lock in preamble */
246 spin_unlock(&igd_lock);
247 }
248 }
249
parse_snb_timeout(const char * s)250 static int __init parse_snb_timeout(const char *s)
251 {
252 int t;
253 const char *q = NULL;
254
255 t = parse_bool(s, NULL);
256 if ( t < 0 )
257 {
258 if ( *s == '\0' )
259 t = SNB_IGD_TIMEOUT_LEGACY;
260 else if ( strcmp(s, "cap") == 0 )
261 t = SNB_IGD_TIMEOUT;
262 else
263 t = strtoul(s, &q, 0);
264 }
265 else
266 t = t ? SNB_IGD_TIMEOUT_LEGACY : 0;
267 snb_igd_timeout = MILLISECS(t);
268
269 return (q && *q) ? -EINVAL : 0;
270 }
271 custom_param("snb_igd_quirk", parse_snb_timeout);
272
273 /* 5500/5520/X58 Chipset Interrupt remapping errata, for stepping B-3.
274 * Fixed in stepping C-2. */
tylersburg_intremap_quirk(void)275 static void __init tylersburg_intremap_quirk(void)
276 {
277 uint32_t bus, device;
278 uint8_t rev;
279
280 for ( bus = 0; bus < 0x100; bus++ )
281 {
282 /* Match on System Management Registers on Device 20 Function 0 */
283 device = pci_conf_read32(0, bus, 20, 0, PCI_VENDOR_ID);
284 rev = pci_conf_read8(0, bus, 20, 0, PCI_REVISION_ID);
285
286 if ( rev == 0x13 && device == 0x342e8086 )
287 {
288 printk(XENLOG_WARNING VTDPREFIX
289 "Disabling IOMMU due to Intel 5500/5520/X58 Chipset errata #47, #53\n");
290 iommu_enable = 0;
291 break;
292 }
293 }
294 }
295
296 /* initialize platform identification flags */
platform_quirks_init(void)297 void __init platform_quirks_init(void)
298 {
299 ioh_id = pci_conf_read32(0, 0, IOH_DEV, 0, 0);
300 igd_id = pci_conf_read32(0, 0, IGD_DEV, 0, 0);
301
302 /* Mobile 4 Series Chipset neglects to set RWBF capability. */
303 if ( ioh_id == 0x2a408086 )
304 {
305 dprintk(XENLOG_INFO VTDPREFIX, "DMAR: Forcing write-buffer flush\n");
306 rwbf_quirk = 1;
307 }
308
309 /* initialize cantiga B3 identification */
310 cantiga_b3_errata_init();
311
312 snb_errata_init();
313
314 /* ioremap IGD MMIO+0x2000 page */
315 map_igd_reg();
316
317 /* Tylersburg interrupt remap quirk */
318 if ( iommu_intremap )
319 tylersburg_intremap_quirk();
320 }
321
322 /*
323 * QUIRK to workaround wifi direct assignment issue. This issue
324 * impacts only cases where Intel integrated wifi device is directly
325 * is directly assigned to a guest.
326 *
327 * The workaround is to map ME phantom device 0:3.7 or 0:22.7
328 * to the ME vt-d engine if detect the user is trying to directly
329 * assigning Intel integrated wifi device to a guest.
330 */
331
map_me_phantom_function(struct domain * domain,u32 dev,int map)332 static int __must_check map_me_phantom_function(struct domain *domain,
333 u32 dev, int map)
334 {
335 struct acpi_drhd_unit *drhd;
336 struct pci_dev *pdev;
337 int rc;
338
339 /* find ME VT-d engine base on a real ME device */
340 pdev = pci_get_pdev(0, 0, PCI_DEVFN(dev, 0));
341 drhd = acpi_find_matched_drhd_unit(pdev);
342
343 /* map or unmap ME phantom function */
344 if ( map )
345 rc = domain_context_mapping_one(domain, drhd->iommu, 0,
346 PCI_DEVFN(dev, 7), NULL);
347 else
348 rc = domain_context_unmap_one(domain, drhd->iommu, 0,
349 PCI_DEVFN(dev, 7));
350
351 return rc;
352 }
353
me_wifi_quirk(struct domain * domain,u8 bus,u8 devfn,int map)354 int me_wifi_quirk(struct domain *domain, u8 bus, u8 devfn, int map)
355 {
356 u32 id;
357 int rc = 0;
358
359 id = pci_conf_read32(0, 0, 0, 0, 0);
360 if ( IS_CTG(id) )
361 {
362 /* quit if ME does not exist */
363 if ( pci_conf_read32(0, 0, 3, 0, 0) == 0xffffffff )
364 return 0;
365
366 /* if device is WLAN device, map ME phantom device 0:3.7 */
367 id = pci_conf_read32(0, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), 0);
368 switch (id)
369 {
370 case 0x42328086:
371 case 0x42358086:
372 case 0x42368086:
373 case 0x42378086:
374 case 0x423a8086:
375 case 0x423b8086:
376 case 0x423c8086:
377 case 0x423d8086:
378 rc = map_me_phantom_function(domain, 3, map);
379 break;
380 default:
381 break;
382 }
383 }
384 else if ( IS_ILK(id) || IS_CPT(id) )
385 {
386 /* quit if ME does not exist */
387 if ( pci_conf_read32(0, 0, 22, 0, 0) == 0xffffffff )
388 return 0;
389
390 /* if device is WLAN device, map ME phantom device 0:22.7 */
391 id = pci_conf_read32(0, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), 0);
392 switch (id)
393 {
394 case 0x00878086: /* Kilmer Peak */
395 case 0x00898086:
396 case 0x00828086: /* Taylor Peak */
397 case 0x00858086:
398 case 0x008F8086: /* Rainbow Peak */
399 case 0x00908086:
400 case 0x00918086:
401 case 0x42388086: /* Puma Peak */
402 case 0x422b8086:
403 case 0x422c8086:
404 rc = map_me_phantom_function(domain, 22, map);
405 break;
406 default:
407 break;
408 }
409 }
410
411 return rc;
412 }
413
pci_vtd_quirk(const struct pci_dev * pdev)414 void pci_vtd_quirk(const struct pci_dev *pdev)
415 {
416 int seg = pdev->seg;
417 int bus = pdev->bus;
418 int dev = PCI_SLOT(pdev->devfn);
419 int func = PCI_FUNC(pdev->devfn);
420 int pos;
421 bool_t ff;
422 u32 val, val2;
423 u64 bar;
424 paddr_t pa;
425 const char *action;
426
427 if ( pci_conf_read16(seg, bus, dev, func, PCI_VENDOR_ID) !=
428 PCI_VENDOR_ID_INTEL )
429 return;
430
431 switch ( pci_conf_read16(seg, bus, dev, func, PCI_DEVICE_ID) )
432 {
433 /*
434 * Mask reporting Intel VT-d faults to IOH core logic:
435 * - Some platform escalates VT-d faults to platform errors.
436 * - This can cause system failure upon non-fatal VT-d faults.
437 * - Potential security issue if malicious guest trigger VT-d faults.
438 */
439 case 0x342e: /* Tylersburg chipset (Nehalem / Westmere systems) */
440 case 0x3728: /* Xeon C5500/C3500 (JasperForest) */
441 case 0x3c28: /* Sandybridge */
442 val = pci_conf_read32(seg, bus, dev, func, 0x1AC);
443 pci_conf_write32(seg, bus, dev, func, 0x1AC, val | (1 << 31));
444 printk(XENLOG_INFO "Masked VT-d error signaling on %04x:%02x:%02x.%u\n",
445 seg, bus, dev, func);
446 break;
447
448 /* Tylersburg (EP)/Boxboro (MP) chipsets (NHM-EP/EX, WSM-EP/EX) */
449 case 0x3400 ... 0x3407: /* host bridges */
450 case 0x3408 ... 0x3411: case 0x3420 ... 0x3421: /* root ports */
451 /* JasperForest (Intel Xeon Processor C5500/C3500 */
452 case 0x3700 ... 0x370f: /* host bridges */
453 case 0x3720 ... 0x3724: /* root ports */
454 /* Sandybridge-EP (Romley) */
455 case 0x3c00: /* host bridge */
456 case 0x3c01 ... 0x3c0b: /* root ports */
457 pos = pci_find_ext_capability(seg, bus, pdev->devfn,
458 PCI_EXT_CAP_ID_ERR);
459 if ( !pos )
460 {
461 pos = pci_find_ext_capability(seg, bus, pdev->devfn,
462 PCI_EXT_CAP_ID_VNDR);
463 while ( pos )
464 {
465 val = pci_conf_read32(seg, bus, dev, func, pos + PCI_VNDR_HEADER);
466 if ( PCI_VNDR_HEADER_ID(val) == 4 && PCI_VNDR_HEADER_REV(val) == 1 )
467 {
468 pos += PCI_VNDR_HEADER;
469 break;
470 }
471 pos = pci_find_next_ext_capability(seg, bus, pdev->devfn, pos,
472 PCI_EXT_CAP_ID_VNDR);
473 }
474 ff = 0;
475 }
476 else
477 ff = pcie_aer_get_firmware_first(pdev);
478 if ( !pos )
479 {
480 printk(XENLOG_WARNING "%04x:%02x:%02x.%u without AER capability?\n",
481 seg, bus, dev, func);
482 break;
483 }
484
485 val = pci_conf_read32(seg, bus, dev, func, pos + PCI_ERR_UNCOR_MASK);
486 val2 = pci_conf_read32(seg, bus, dev, func, pos + PCI_ERR_COR_MASK);
487 if ( (val & PCI_ERR_UNC_UNSUP) && (val2 & PCI_ERR_COR_ADV_NFAT) )
488 action = "Found masked";
489 else if ( !ff )
490 {
491 pci_conf_write32(seg, bus, dev, func, pos + PCI_ERR_UNCOR_MASK,
492 val | PCI_ERR_UNC_UNSUP);
493 pci_conf_write32(seg, bus, dev, func, pos + PCI_ERR_COR_MASK,
494 val2 | PCI_ERR_COR_ADV_NFAT);
495 action = "Masked";
496 }
497 else
498 action = "Must not mask";
499
500 /* XPUNCERRMSK Send Completion with Unsupported Request */
501 val = pci_conf_read32(seg, bus, dev, func, 0x20c);
502 pci_conf_write32(seg, bus, dev, func, 0x20c, val | (1 << 4));
503
504 printk(XENLOG_INFO "%s UR signaling on %04x:%02x:%02x.%u\n",
505 action, seg, bus, dev, func);
506 break;
507
508 case 0x0040: case 0x0044: case 0x0048: /* Nehalem/Westmere */
509 case 0x0100: case 0x0104: case 0x0108: /* Sandybridge */
510 case 0x0150: case 0x0154: case 0x0158: /* Ivybridge */
511 case 0x0a00: case 0x0a04: case 0x0a08: case 0x0a0f: /* Haswell ULT */
512 case 0x0c00: case 0x0c04: case 0x0c08: case 0x0c0f: /* Haswell */
513 case 0x0d00: case 0x0d04: case 0x0d08: case 0x0d0f: /* Haswell */
514 case 0x1600: case 0x1604: case 0x1608: case 0x160f: /* Broadwell */
515 case 0x1610: case 0x1614: case 0x1618: /* Broadwell */
516 case 0x1900: case 0x1904: case 0x1908: case 0x190c: case 0x190f: /* Skylake */
517 case 0x1910: case 0x1918: case 0x191f: /* Skylake */
518 bar = pci_conf_read32(seg, bus, dev, func, 0x6c);
519 bar = (bar << 32) | pci_conf_read32(seg, bus, dev, func, 0x68);
520 pa = bar & 0x7ffffff000UL; /* bits 12...38 */
521 if ( (bar & 1) && pa &&
522 page_is_ram_type(paddr_to_pfn(pa), RAM_TYPE_RESERVED) )
523 {
524 u32 __iomem *va = ioremap(pa, PAGE_SIZE);
525
526 if ( va )
527 {
528 __set_bit(0x1c8 * 8 + 20, va);
529 iounmap(va);
530 printk(XENLOG_INFO "Masked UR signaling on %04x:%02x:%02x.%u\n",
531 seg, bus, dev, func);
532 }
533 else
534 printk(XENLOG_ERR "Could not map %"PRIpaddr" for %04x:%02x:%02x.%u\n",
535 pa, seg, bus, dev, func);
536 }
537 else
538 printk(XENLOG_WARNING "Bogus DMIBAR %#"PRIx64" on %04x:%02x:%02x.%u\n",
539 bar, seg, bus, dev, func);
540 break;
541 }
542 }
543