1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * TPH (TLP Processing Hints) support
4 *
5 * Copyright (C) 2024 Advanced Micro Devices, Inc.
6 * Eric Van Tassell <Eric.VanTassell@amd.com>
7 * Wei Huang <wei.huang2@amd.com>
8 */
9 #include <linux/pci.h>
10 #include <linux/pci-acpi.h>
11 #include <linux/msi.h>
12 #include <linux/bitfield.h>
13 #include <linux/pci-tph.h>
14
15 #include "pci.h"
16
17 /* System-wide TPH disabled */
18 static bool pci_tph_disabled;
19
20 #ifdef CONFIG_ACPI
21 /*
22 * The st_info struct defines the Steering Tag (ST) info returned by the
23 * firmware PCI ACPI _DSM method (rev=0x7, func=0xF, "_DSM to Query Cache
24 * Locality TPH Features"), as specified in the approved ECN for PCI Firmware
25 * Spec and available at https://members.pcisig.com/wg/PCI-SIG/document/15470.
26 *
27 * @vm_st_valid: 8-bit ST for volatile memory is valid
28 * @vm_xst_valid: 16-bit extended ST for volatile memory is valid
29 * @vm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied
30 * @vm_st: 8-bit ST for volatile mem
31 * @vm_xst: 16-bit extended ST for volatile mem
32 * @pm_st_valid: 8-bit ST for persistent memory is valid
33 * @pm_xst_valid: 16-bit extended ST for persistent memory is valid
34 * @pm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied
35 * @pm_st: 8-bit ST for persistent mem
36 * @pm_xst: 16-bit extended ST for persistent mem
37 */
38 union st_info {
39 struct {
40 u64 vm_st_valid : 1;
41 u64 vm_xst_valid : 1;
42 u64 vm_ph_ignore : 1;
43 u64 rsvd1 : 5;
44 u64 vm_st : 8;
45 u64 vm_xst : 16;
46 u64 pm_st_valid : 1;
47 u64 pm_xst_valid : 1;
48 u64 pm_ph_ignore : 1;
49 u64 rsvd2 : 5;
50 u64 pm_st : 8;
51 u64 pm_xst : 16;
52 };
53 u64 value;
54 };
55
tph_extract_tag(enum tph_mem_type mem_type,u8 req_type,union st_info * info)56 static u16 tph_extract_tag(enum tph_mem_type mem_type, u8 req_type,
57 union st_info *info)
58 {
59 switch (req_type) {
60 case PCI_TPH_REQ_TPH_ONLY: /* 8-bit tag */
61 switch (mem_type) {
62 case TPH_MEM_TYPE_VM:
63 if (info->vm_st_valid)
64 return info->vm_st;
65 break;
66 case TPH_MEM_TYPE_PM:
67 if (info->pm_st_valid)
68 return info->pm_st;
69 break;
70 }
71 break;
72 case PCI_TPH_REQ_EXT_TPH: /* 16-bit tag */
73 switch (mem_type) {
74 case TPH_MEM_TYPE_VM:
75 if (info->vm_xst_valid)
76 return info->vm_xst;
77 break;
78 case TPH_MEM_TYPE_PM:
79 if (info->pm_xst_valid)
80 return info->pm_xst;
81 break;
82 }
83 break;
84 default:
85 return 0;
86 }
87
88 return 0;
89 }
90
91 #define TPH_ST_DSM_FUNC_INDEX 0xF
tph_invoke_dsm(acpi_handle handle,u32 cpu_uid,union st_info * st_out)92 static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid,
93 union st_info *st_out)
94 {
95 union acpi_object arg3[3], in_obj, *out_obj;
96
97 if (!acpi_check_dsm(handle, &pci_acpi_dsm_guid, 7,
98 BIT(TPH_ST_DSM_FUNC_INDEX)))
99 return AE_ERROR;
100
101 /* DWORD: feature ID (0 for processor cache ST query) */
102 arg3[0].integer.type = ACPI_TYPE_INTEGER;
103 arg3[0].integer.value = 0;
104
105 /* DWORD: target UID */
106 arg3[1].integer.type = ACPI_TYPE_INTEGER;
107 arg3[1].integer.value = cpu_uid;
108
109 /* QWORD: properties, all 0's */
110 arg3[2].integer.type = ACPI_TYPE_INTEGER;
111 arg3[2].integer.value = 0;
112
113 in_obj.type = ACPI_TYPE_PACKAGE;
114 in_obj.package.count = ARRAY_SIZE(arg3);
115 in_obj.package.elements = arg3;
116
117 out_obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 7,
118 TPH_ST_DSM_FUNC_INDEX, &in_obj);
119 if (!out_obj)
120 return AE_ERROR;
121
122 if (out_obj->type != ACPI_TYPE_BUFFER) {
123 ACPI_FREE(out_obj);
124 return AE_ERROR;
125 }
126
127 st_out->value = *((u64 *)(out_obj->buffer.pointer));
128
129 ACPI_FREE(out_obj);
130
131 return AE_OK;
132 }
133 #endif
134
135 /* Update the TPH Requester Enable field of TPH Control Register */
set_ctrl_reg_req_en(struct pci_dev * pdev,u8 req_type)136 static void set_ctrl_reg_req_en(struct pci_dev *pdev, u8 req_type)
137 {
138 u32 reg;
139
140 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, ®);
141
142 reg &= ~PCI_TPH_CTRL_REQ_EN_MASK;
143 reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, req_type);
144
145 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
146 }
147
get_st_modes(struct pci_dev * pdev)148 static u8 get_st_modes(struct pci_dev *pdev)
149 {
150 u32 reg;
151
152 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®);
153 reg &= PCI_TPH_CAP_ST_NS | PCI_TPH_CAP_ST_IV | PCI_TPH_CAP_ST_DS;
154
155 return reg;
156 }
157
get_st_table_loc(struct pci_dev * pdev)158 static u32 get_st_table_loc(struct pci_dev *pdev)
159 {
160 u32 reg;
161
162 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®);
163
164 return FIELD_GET(PCI_TPH_CAP_LOC_MASK, reg);
165 }
166
167 /*
168 * Return the size of ST table. If ST table is not in TPH Requester Extended
169 * Capability space, return 0. Otherwise return the ST Table Size + 1.
170 */
pcie_tph_get_st_table_size(struct pci_dev * pdev)171 u16 pcie_tph_get_st_table_size(struct pci_dev *pdev)
172 {
173 u32 reg;
174 u32 loc;
175
176 /* Check ST table location first */
177 loc = get_st_table_loc(pdev);
178
179 /* Convert loc to match with PCI_TPH_LOC_* defined in pci_regs.h */
180 loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc);
181 if (loc != PCI_TPH_LOC_CAP)
182 return 0;
183
184 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®);
185
186 return FIELD_GET(PCI_TPH_CAP_ST_MASK, reg) + 1;
187 }
188 EXPORT_SYMBOL(pcie_tph_get_st_table_size);
189
190 /* Return device's Root Port completer capability */
get_rp_completer_type(struct pci_dev * pdev)191 static u8 get_rp_completer_type(struct pci_dev *pdev)
192 {
193 struct pci_dev *rp;
194 u32 reg;
195 int ret;
196
197 rp = pcie_find_root_port(pdev);
198 if (!rp)
199 return 0;
200
201 ret = pcie_capability_read_dword(rp, PCI_EXP_DEVCAP2, ®);
202 if (ret)
203 return 0;
204
205 return FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg);
206 }
207
208 /* Write tag to ST table - Return 0 if OK, otherwise -errno */
write_tag_to_st_table(struct pci_dev * pdev,int index,u16 tag)209 static int write_tag_to_st_table(struct pci_dev *pdev, int index, u16 tag)
210 {
211 int st_table_size;
212 int offset;
213
214 /* Check if index is out of bound */
215 st_table_size = pcie_tph_get_st_table_size(pdev);
216 if (index >= st_table_size)
217 return -ENXIO;
218
219 offset = pdev->tph_cap + PCI_TPH_BASE_SIZEOF + index * sizeof(u16);
220
221 return pci_write_config_word(pdev, offset, tag);
222 }
223
224 /**
225 * pcie_tph_get_cpu_st() - Retrieve Steering Tag for a target memory associated
226 * with a specific CPU
227 * @pdev: PCI device
228 * @mem_type: target memory type (volatile or persistent RAM)
229 * @cpu_uid: associated CPU id
230 * @tag: Steering Tag to be returned
231 *
232 * Return the Steering Tag for a target memory that is associated with a
233 * specific CPU as indicated by cpu_uid.
234 *
235 * Return: 0 if success, otherwise negative value (-errno)
236 */
pcie_tph_get_cpu_st(struct pci_dev * pdev,enum tph_mem_type mem_type,unsigned int cpu_uid,u16 * tag)237 int pcie_tph_get_cpu_st(struct pci_dev *pdev, enum tph_mem_type mem_type,
238 unsigned int cpu_uid, u16 *tag)
239 {
240 #ifdef CONFIG_ACPI
241 struct pci_dev *rp;
242 acpi_handle rp_acpi_handle;
243 union st_info info;
244
245 rp = pcie_find_root_port(pdev);
246 if (!rp || !rp->bus || !rp->bus->bridge)
247 return -ENODEV;
248
249 rp_acpi_handle = ACPI_HANDLE(rp->bus->bridge);
250
251 if (tph_invoke_dsm(rp_acpi_handle, cpu_uid, &info) != AE_OK) {
252 *tag = 0;
253 return -EINVAL;
254 }
255
256 *tag = tph_extract_tag(mem_type, pdev->tph_req_type, &info);
257
258 pci_dbg(pdev, "get steering tag: mem_type=%s, cpu_uid=%d, tag=%#04x\n",
259 (mem_type == TPH_MEM_TYPE_VM) ? "volatile" : "persistent",
260 cpu_uid, *tag);
261
262 return 0;
263 #else
264 return -ENODEV;
265 #endif
266 }
267 EXPORT_SYMBOL(pcie_tph_get_cpu_st);
268
269 /**
270 * pcie_tph_set_st_entry() - Set Steering Tag in the ST table entry
271 * @pdev: PCI device
272 * @index: ST table entry index
273 * @tag: Steering Tag to be written
274 *
275 * Figure out the proper location of ST table, either in the MSI-X table or
276 * in the TPH Extended Capability space, and write the Steering Tag into
277 * the ST entry pointed by index.
278 *
279 * Return: 0 if success, otherwise negative value (-errno)
280 */
pcie_tph_set_st_entry(struct pci_dev * pdev,unsigned int index,u16 tag)281 int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, u16 tag)
282 {
283 u32 loc;
284 int err = 0;
285
286 if (!pdev->tph_cap)
287 return -EINVAL;
288
289 if (!pdev->tph_enabled)
290 return -EINVAL;
291
292 /* No need to write tag if device is in "No ST Mode" */
293 if (pdev->tph_mode == PCI_TPH_ST_NS_MODE)
294 return 0;
295
296 /*
297 * Disable TPH before updating ST to avoid potential instability as
298 * cautioned in PCIe r6.2, sec 6.17.3, "ST Modes of Operation"
299 */
300 set_ctrl_reg_req_en(pdev, PCI_TPH_REQ_DISABLE);
301
302 loc = get_st_table_loc(pdev);
303 /* Convert loc to match with PCI_TPH_LOC_* */
304 loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc);
305
306 switch (loc) {
307 case PCI_TPH_LOC_MSIX:
308 err = pci_msix_write_tph_tag(pdev, index, tag);
309 break;
310 case PCI_TPH_LOC_CAP:
311 err = write_tag_to_st_table(pdev, index, tag);
312 break;
313 default:
314 err = -EINVAL;
315 }
316
317 if (err) {
318 pcie_disable_tph(pdev);
319 return err;
320 }
321
322 set_ctrl_reg_req_en(pdev, pdev->tph_req_type);
323
324 pci_dbg(pdev, "set steering tag: %s table, index=%d, tag=%#04x\n",
325 (loc == PCI_TPH_LOC_MSIX) ? "MSI-X" : "ST", index, tag);
326
327 return 0;
328 }
329 EXPORT_SYMBOL(pcie_tph_set_st_entry);
330
331 /**
332 * pcie_disable_tph - Turn off TPH support for device
333 * @pdev: PCI device
334 *
335 * Return: none
336 */
pcie_disable_tph(struct pci_dev * pdev)337 void pcie_disable_tph(struct pci_dev *pdev)
338 {
339 if (!pdev->tph_cap)
340 return;
341
342 if (!pdev->tph_enabled)
343 return;
344
345 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, 0);
346
347 pdev->tph_mode = 0;
348 pdev->tph_req_type = 0;
349 pdev->tph_enabled = 0;
350 }
351 EXPORT_SYMBOL(pcie_disable_tph);
352
353 /**
354 * pcie_enable_tph - Enable TPH support for device using a specific ST mode
355 * @pdev: PCI device
356 * @mode: ST mode to enable. Current supported modes include:
357 *
358 * - PCI_TPH_ST_NS_MODE: NO ST Mode
359 * - PCI_TPH_ST_IV_MODE: Interrupt Vector Mode
360 * - PCI_TPH_ST_DS_MODE: Device Specific Mode
361 *
362 * Check whether the mode is actually supported by the device before enabling
363 * and return an error if not. Additionally determine what types of requests,
364 * TPH or extended TPH, can be issued by the device based on its TPH requester
365 * capability and the Root Port's completer capability.
366 *
367 * Return: 0 on success, otherwise negative value (-errno)
368 */
pcie_enable_tph(struct pci_dev * pdev,int mode)369 int pcie_enable_tph(struct pci_dev *pdev, int mode)
370 {
371 u32 reg;
372 u8 dev_modes;
373 u8 rp_req_type;
374
375 /* Honor "notph" kernel parameter */
376 if (pci_tph_disabled)
377 return -EINVAL;
378
379 if (!pdev->tph_cap)
380 return -EINVAL;
381
382 if (pdev->tph_enabled)
383 return -EBUSY;
384
385 /* Sanitize and check ST mode compatibility */
386 mode &= PCI_TPH_CTRL_MODE_SEL_MASK;
387 dev_modes = get_st_modes(pdev);
388 if (!((1 << mode) & dev_modes))
389 return -EINVAL;
390
391 pdev->tph_mode = mode;
392
393 /* Get req_type supported by device and its Root Port */
394 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®);
395 if (FIELD_GET(PCI_TPH_CAP_EXT_TPH, reg))
396 pdev->tph_req_type = PCI_TPH_REQ_EXT_TPH;
397 else
398 pdev->tph_req_type = PCI_TPH_REQ_TPH_ONLY;
399
400 rp_req_type = get_rp_completer_type(pdev);
401
402 /* Final req_type is the smallest value of two */
403 pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type);
404
405 if (pdev->tph_req_type == PCI_TPH_REQ_DISABLE)
406 return -EINVAL;
407
408 /* Write them into TPH control register */
409 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, ®);
410
411 reg &= ~PCI_TPH_CTRL_MODE_SEL_MASK;
412 reg |= FIELD_PREP(PCI_TPH_CTRL_MODE_SEL_MASK, pdev->tph_mode);
413
414 reg &= ~PCI_TPH_CTRL_REQ_EN_MASK;
415 reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, pdev->tph_req_type);
416
417 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
418
419 pdev->tph_enabled = 1;
420
421 return 0;
422 }
423 EXPORT_SYMBOL(pcie_enable_tph);
424
pci_restore_tph_state(struct pci_dev * pdev)425 void pci_restore_tph_state(struct pci_dev *pdev)
426 {
427 struct pci_cap_saved_state *save_state;
428 int num_entries, i, offset;
429 u16 *st_entry;
430 u32 *cap;
431
432 if (!pdev->tph_cap)
433 return;
434
435 if (!pdev->tph_enabled)
436 return;
437
438 save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH);
439 if (!save_state)
440 return;
441
442 /* Restore control register and all ST entries */
443 cap = &save_state->cap.data[0];
444 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, *cap++);
445 st_entry = (u16 *)cap;
446 offset = PCI_TPH_BASE_SIZEOF;
447 num_entries = pcie_tph_get_st_table_size(pdev);
448 for (i = 0; i < num_entries; i++) {
449 pci_write_config_word(pdev, pdev->tph_cap + offset,
450 *st_entry++);
451 offset += sizeof(u16);
452 }
453 }
454
pci_save_tph_state(struct pci_dev * pdev)455 void pci_save_tph_state(struct pci_dev *pdev)
456 {
457 struct pci_cap_saved_state *save_state;
458 int num_entries, i, offset;
459 u16 *st_entry;
460 u32 *cap;
461
462 if (!pdev->tph_cap)
463 return;
464
465 if (!pdev->tph_enabled)
466 return;
467
468 save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH);
469 if (!save_state)
470 return;
471
472 /* Save control register */
473 cap = &save_state->cap.data[0];
474 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, cap++);
475
476 /* Save all ST entries in extended capability structure */
477 st_entry = (u16 *)cap;
478 offset = PCI_TPH_BASE_SIZEOF;
479 num_entries = pcie_tph_get_st_table_size(pdev);
480 for (i = 0; i < num_entries; i++) {
481 pci_read_config_word(pdev, pdev->tph_cap + offset,
482 st_entry++);
483 offset += sizeof(u16);
484 }
485 }
486
pci_no_tph(void)487 void pci_no_tph(void)
488 {
489 pci_tph_disabled = true;
490
491 pr_info("PCIe TPH is disabled\n");
492 }
493
pci_tph_init(struct pci_dev * pdev)494 void pci_tph_init(struct pci_dev *pdev)
495 {
496 int num_entries;
497 u32 save_size;
498
499 pdev->tph_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_TPH);
500 if (!pdev->tph_cap)
501 return;
502
503 num_entries = pcie_tph_get_st_table_size(pdev);
504 save_size = sizeof(u32) + num_entries * sizeof(u16);
505 pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_TPH, save_size);
506 }
507