1 /*
2 * Arm SCP/MCP Software
3 * Copyright (c) 2020-2022, Arm Limited and Contributors. All rights reserved.
4 *
5 * SPDX-License-Identifier: BSD-3-Clause
6 */
7
8 /*
9 * During PCIe bus enumeration, when a device is not connected, accessing
10 * its configuration space should return 0xFFFFFFFF. However, due to
11 * a bug in the PCIe controller, bus fault is asserted instead of returning
12 * 0xFFFFFFFF. To avoid bus fault errors in AP cores, SCP will perform
13 * a bus enumeration, identifying valid endpoints connected and mask the bus
14 * fault when reading the configuration space.
15 */
16
17 #include "n1sdp_pcie.h"
18 #include "n1sdp_scp_mmap.h"
19
20 #include <mod_n1sdp_pcie.h>
21
22 #include <fwk_assert.h>
23 #include <fwk_attributes.h>
24 #include <fwk_macros.h>
25
26 #include <fmw_cmsis.h>
27
28 #include <stdbool.h>
29 #include <stddef.h>
30 #include <stdint.h>
31
32 /* PCIe configuration space offset definitions */
33 #define PCIE_HEADER_TYPE_OFFSET 0xE
34 #define PCIE_PRIMARY_BUS_NUM_OFFSET 0x18
35 #define PCIE_SECONDARY_BUS_NUM_OFFSET 0x19
36 #define PCIE_SUBORDINATE_BUS_NUM_OFFSET 0x1A
37
38 #define PCIE_HEADER_TYPE_MASK 0x7F
39 #define PCIE_HEADER_TYPE_ENDPOINT 0
40 #define PCIE_HEADER_TYPE_BRIDGE 1
41
42 /* BDF table offsets for PCIe & CCIX controllers */
43 #define PCIE_BDF_TABLE_OFFSET 0
44 #define CCIX_BDF_TABLE_OFFSET (16 * FWK_KIB)
45
46 /* PCIe standard definitions */
47 #define PCIE_BUS_NUM_MAX 0xFF
48 #define DEVICES_PER_BUS_MAX 32
49 #define FUNCTIONS_PER_DEVICE_MAX 8
50
51 #define BDF_ADDR_SHIFT_BUS 20
52 #define BDF_ADDR_SHIFT_DEVICE 15
53 #define BDF_ADDR_SHIFT_FUNCTION 12
54
55 /* Initial bus number definitions */
56 #define PCIE_PRIMARY_BUS_NUM_START 1
57 #define PCIE_SECONDARY_BUS_NUM_START 2
58
59 #define CCIX_PRIMARY_BUS_NUM_START 1
60 #define CCIX_SECONDARY_BUS_NUM_START 2
61
62 /* Structure defining the BDF table */
63 struct bdf_table {
64 /* Base address of Root Port's configuration space */
65 uint32_t rp_config_base_addr;
66 /* Total valid BDF entries found during bus enumeration */
67 uint32_t bdf_count;
68 /* BDF table entries */
69 uint32_t table_data[];
70 };
71
72 /*
73 * Global variables used during bus enumeration
74 * for BDF table generation
75 */
76 uint32_t *bdf_table_ptr;
77 uint32_t bdf_count;
78
79 /*!
80 * \brief Performs a "checked" 32-bit load from @src@.
81 *
82 * \details By default, SCPv2's exception handler will make the core enter into
83 * an indefinite while loop when an exception happens.
84 * The exception handler have special code that triggers if a load on
85 * @src@ when in this function trigger a bus fault.
86 * This function mask any bus faults triggered due to read of @src@.
87 *
88 * \retval true: Load was successful and @value@ is updated.
89 * \retval false: A bus fault was triggered during the load. @value@ is not
90 * updated.
91 */
92 static bool checked_read_u32(uint32_t *const value, const uint32_t *const src);
93
94 /*
95 * This function is called if link training failed to initialize the
96 * BDF table with RP base address and zero BDF count such that UEFI/Linux
97 * will not read garbage table values.
98 */
pcie_init_bdf_table(struct n1sdp_pcie_dev_config * config)99 void pcie_init_bdf_table(struct n1sdp_pcie_dev_config *config)
100 {
101 fwk_assert(config != NULL);
102 struct bdf_table *table;
103
104 /* Set BDF table pointer based on the root complex */
105 if (config->ccix_capable) {
106 table = (struct bdf_table *)(SCP_NONTRUSTED_RAM_BASE +
107 CCIX_BDF_TABLE_OFFSET);
108 } else {
109 table = (struct bdf_table *)(SCP_NONTRUSTED_RAM_BASE +
110 PCIE_BDF_TABLE_OFFSET);
111 }
112
113 table->rp_config_base_addr = config->global_config_base -
114 SCP_AP_AXI_OFFSET;
115 table->bdf_count = 0;
116 }
117
pcie_bus_scan(uint32_t ecam_addr,uint8_t pri_bnum,uint8_t sec_bnum)118 static uint8_t pcie_bus_scan(uint32_t ecam_addr,
119 uint8_t pri_bnum,
120 uint8_t sec_bnum)
121 {
122 int dev_num, fn_num;
123 uint32_t bdf_addr;
124 uint32_t vid;
125 uint32_t config_addr;
126 uint8_t header_type;
127 uint8_t sub_bnum = pri_bnum;
128
129 /* Loop over all devices on pri_bnum bus */
130 for (dev_num = 0; dev_num < DEVICES_PER_BUS_MAX; dev_num++) {
131 /*
132 * Special case:
133 * Ignore dev_num > 0 on bus 1 because the controller returns
134 * config space of device 0 for all other device numbers on bus 1
135 */
136 if (((pri_bnum == PCIE_PRIMARY_BUS_NUM_START) && (dev_num != 0)) ||
137 ((pri_bnum == CCIX_PRIMARY_BUS_NUM_START) && (dev_num != 0))) {
138 break;
139 }
140 /* Loop over all functions on dev_num device */
141 for (fn_num = 0; fn_num < FUNCTIONS_PER_DEVICE_MAX; fn_num++) {
142 bdf_addr = (pri_bnum << BDF_ADDR_SHIFT_BUS) |
143 (dev_num << BDF_ADDR_SHIFT_DEVICE) |
144 (fn_num << BDF_ADDR_SHIFT_FUNCTION);
145 config_addr = ecam_addr + bdf_addr;
146
147 if (!checked_read_u32(&vid, (uint32_t *)config_addr)) {
148 /*
149 * Exception occurred.
150 * Ignore this bus-device-function and move to next function.
151 */
152 continue;
153 }
154
155 /* Valid device is identified so fill the BDF table */
156 bdf_count++;
157 *bdf_table_ptr++ = bdf_addr;
158
159 /* If function 0 of any device has invalid VID break the loop */
160 if ((vid & 0xFFFF) == 0xFFFF) {
161 if (fn_num == 0) {
162 break;
163 } else {
164 continue;
165 }
166 }
167
168 /*
169 * Read the header type to identify if the device
170 * is an endpoint or a PCI-PCI bridge.
171 */
172 header_type = *(uint8_t *)(config_addr + PCIE_HEADER_TYPE_OFFSET);
173 if ((header_type &
174 PCIE_HEADER_TYPE_MASK) == PCIE_HEADER_TYPE_BRIDGE) {
175 /*
176 * PCI-PCI bridge is identified. Set primary and secondary bus
177 * numbers. Let subordinate bus number be max possible bus
178 * number as we need to further identify devices downstream.
179 */
180 *(uint8_t *)(config_addr +
181 PCIE_PRIMARY_BUS_NUM_OFFSET) = pri_bnum;
182 *(uint8_t *)(config_addr +
183 PCIE_SECONDARY_BUS_NUM_OFFSET) = sec_bnum;
184 *(uint8_t *)(config_addr +
185 PCIE_SUBORDINATE_BUS_NUM_OFFSET) = PCIE_BUS_NUM_MAX;
186 /*
187 * Recursively call the scan function with incremented
188 * primary and secondary bus numbers.
189 */
190 sub_bnum = pcie_bus_scan(ecam_addr, sec_bnum, sec_bnum + 1);
191 /*
192 * The recursive call has returned from an endpoint
193 * identification so use the returned bus number as the
194 * bridge's subordinate bus number.
195 */
196 *(uint8_t *)(config_addr +
197 PCIE_SUBORDINATE_BUS_NUM_OFFSET) = sub_bnum;
198 sec_bnum = sub_bnum + 1;
199 } else {
200 /*
201 * Endpoint is identified. Proceed to other functions &
202 * devices in this bus and return to previous recursive call.
203 */
204 sub_bnum = sec_bnum - 1;
205 }
206 }
207 }
208 /* Return the subordinate bus number to previous recursive call */
209 return sub_bnum;
210 }
211
pcie_bus_enumeration(struct n1sdp_pcie_dev_config * config)212 void pcie_bus_enumeration(struct n1sdp_pcie_dev_config *config)
213 {
214 fwk_assert(config != NULL);
215
216 uint32_t ecam_base_addr = config->axi_subordinate_base32;
217 uint8_t pri_bnum, sec_bnum, sub_bnum;
218 struct bdf_table *table;
219
220 /* Set BDF table pointer based on the root complex */
221 if (config->ccix_capable) {
222 table = (struct bdf_table *)(SCP_NONTRUSTED_RAM_BASE +
223 CCIX_BDF_TABLE_OFFSET);
224 } else {
225 table = (struct bdf_table *)(SCP_NONTRUSTED_RAM_BASE +
226 PCIE_BDF_TABLE_OFFSET);
227 }
228
229 pcie_init_bdf_table(config);
230
231 bdf_count = 0;
232 bdf_table_ptr = table->table_data;
233
234 /* Start with bus number 1 as bus 0 is root bus internal to the device */
235 if (config->ccix_capable) {
236 pri_bnum = CCIX_PRIMARY_BUS_NUM_START;
237 sec_bnum = CCIX_SECONDARY_BUS_NUM_START;
238 } else {
239 pri_bnum = PCIE_PRIMARY_BUS_NUM_START;
240 sec_bnum = PCIE_SECONDARY_BUS_NUM_START;
241 }
242
243 /*
244 * Configure primary & secondary bus numbers for root port.
245 * Let sub-ordinate bus number be maximum bus number initially.
246 */
247 *(volatile uint8_t *)(config->global_config_base +
248 PCIE_PRIMARY_BUS_NUM_OFFSET) = pri_bnum - 1;
249 *(volatile uint8_t *)(config->global_config_base +
250 PCIE_SECONDARY_BUS_NUM_OFFSET) = pri_bnum;
251 *(volatile uint8_t *)(config->global_config_base +
252 PCIE_SUBORDINATE_BUS_NUM_OFFSET) = PCIE_BUS_NUM_MAX;
253
254 sub_bnum = pcie_bus_scan(ecam_base_addr, pri_bnum, sec_bnum);
255
256 /*
257 * Update subordinate bus number with maximum bus number identified
258 * from bus scan for this bus hierarchy.
259 */
260 *(volatile uint8_t *)(config->global_config_base +
261 PCIE_SUBORDINATE_BUS_NUM_OFFSET) = sub_bnum;
262
263 /*
264 * Make table entries to be even count by adding a dummy entry
265 * for 64-bit alignment
266 */
267 if (bdf_count & 0x1) {
268 *bdf_table_ptr++ = 0xFFFFFFFF;
269 bdf_count++;
270 }
271
272 table->bdf_count = bdf_count;
273 }
274
275 /*!
276 * \brief Callee context at exception
277 */
278 struct FWK_PACKED context {
279 uint32_t r0;
280 uint32_t r1;
281 uint32_t r2;
282 uint32_t r3;
283 uint32_t r12;
284 uint32_t r14;
285 uint32_t PC;
286 uint32_t xPSR;
287 };
288
289 /* Interrupt Control and State Register */
290 #define ICSR ((FWK_R uint32_t *)(0xE000ED04UL))
291 #define ICSR_VECTACTIVE UINT32_C(0x000000FF)
292
293 /*
294 * Value to indicate a bus fault was masked.
295 * The definitions can not use the UINT32_C() or UL decorators to allow them to
296 * be used in inline assembly.
297 */
298 #define CHECKED_READ_U32__ERROR_VALUE 0xdeadbeef
299
300 /* Bus faults at this instruction should be ignored */
301 #define CHECKED_READ_U32__LOAD_LOC checked_read_u32_load_loc
302
303 /* Restore PC to this label if a bus fault is triggered and should be masked */
304 #define CHECKED_READ_U32__POSTLOAD_LOC checked_read_u32_postload_loc
305
306 #define EXCEPTION_BUSFAULT 5
307
308 FWK_NOINLINE /* only one location where the checked load happen */
309 static bool
checked_read_u32(uint32_t * const value,const uint32_t * const src)310 checked_read_u32(uint32_t *const value, const uint32_t *const src)
311 {
312 uint32_t err;
313 uint32_t dst;
314
315 /*
316 * If the instruction at CHECKED_READ_U32__LOAD_LOC trigger a bus fault
317 * exception the custom exception handler will:
318 * 1. Set register r0 to CHECKED_READ_U32__ERROR_VALUE.
319 * 2. Set PC to CHECKED_READ_U32__POSTLOAD_LOC.
320 * 3. Resume execution.
321 */
322 __asm__ volatile(
323 "ldr r0, =!(" FWK_STRINGIFY(CHECKED_READ_U32__ERROR_VALUE) ");"
324 FWK_STRINGIFY(CHECKED_READ_U32__LOAD_LOC) ": ldr %1, [%2, #0];"
325 FWK_STRINGIFY(CHECKED_READ_U32__POSTLOAD_LOC) ": mov %0, r0;"
326 : "=r"(err), "=r"(dst)
327 : "r"(src)
328 : "r0");
329 if (err == CHECKED_READ_U32__ERROR_VALUE) {
330 return false;
331 }
332 *value = dst;
333 return true;
334 }
335
exception_handler(const int exception,struct context * const context)336 static bool exception_handler(const int exception,
337 struct context *const context) {
338 extern void CHECKED_READ_U32__LOAD_LOC(void);
339 extern void CHECKED_READ_U32__POSTLOAD_LOC(void);
340
341 if (exception == EXCEPTION_BUSFAULT &&
342 (uintptr_t)&CHECKED_READ_U32__LOAD_LOC == (uintptr_t)context->PC) {
343 /*
344 * we got a bus fault in the "checked" read functions so lets jump to
345 * the safe place and notify that we masked the fault
346 */
347 context->r0 = CHECKED_READ_U32__ERROR_VALUE;
348 context->PC = (uintptr_t)&CHECKED_READ_U32__POSTLOAD_LOC;
349 return true;
350 }
351 return false;
352 }
353
arch_exception_invalid(void)354 void arch_exception_invalid(void)
355 {
356 struct context *context;
357 __asm__ volatile(
358 "tst lr, #4;"
359 "ite eq;"
360 "mrseq %0, msp;"
361 "mrsne %0, psp;"
362 : "=r"(context));
363
364 const int exception = *ICSR & ICSR_VECTACTIVE;
365 if (!exception_handler(exception, context)) {
366 while (true) {
367 __WFI();
368 }
369 }
370 }
371