1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Synopsys DesignWare PCIe PMU driver
4 *
5 * Copyright (C) 2021-2023 Alibaba Inc.
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/bitops.h>
10 #include <linux/cpuhotplug.h>
11 #include <linux/cpumask.h>
12 #include <linux/device.h>
13 #include <linux/errno.h>
14 #include <linux/kernel.h>
15 #include <linux/list.h>
16 #include <linux/pcie-dwc.h>
17 #include <linux/perf_event.h>
18 #include <linux/pci.h>
19 #include <linux/platform_device.h>
20 #include <linux/smp.h>
21 #include <linux/sysfs.h>
22 #include <linux/types.h>
23
24 #define DWC_PCIE_EVENT_CNT_CTL 0x8
25
26 /*
27 * Event Counter Data Select includes two parts:
28 * - 27-24: Group number(4-bit: 0..0x7)
29 * - 23-16: Event number(8-bit: 0..0x13) within the Group
30 *
31 * Put them together as in TRM.
32 */
33 #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16)
34 #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8)
35 #define DWC_PCIE_CNT_STATUS BIT(7)
36 #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2)
37 #define DWC_PCIE_PER_EVENT_OFF 0x1
38 #define DWC_PCIE_PER_EVENT_ON 0x3
39 #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0)
40 #define DWC_PCIE_EVENT_PER_CLEAR 0x1
41
42 #define DWC_PCIE_EVENT_CNT_DATA 0xC
43
44 #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10
45 #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24)
46 #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8)
47 #define DWC_PCIE_DURATION_MANUAL_CTL 0x0
48 #define DWC_PCIE_DURATION_1MS 0x1
49 #define DWC_PCIE_DURATION_10MS 0x2
50 #define DWC_PCIE_DURATION_100MS 0x3
51 #define DWC_PCIE_DURATION_1S 0x4
52 #define DWC_PCIE_DURATION_2S 0x5
53 #define DWC_PCIE_DURATION_4S 0x6
54 #define DWC_PCIE_DURATION_4US 0xFF
55 #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0)
56 #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1
57
58 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14
59 #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18
60
61 /* Event attributes */
62 #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0)
63 #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16)
64 #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20)
65
66 #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config)
67 #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config)
68 #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config)
69
70 enum dwc_pcie_event_type {
71 DWC_PCIE_TIME_BASE_EVENT,
72 DWC_PCIE_LANE_EVENT,
73 DWC_PCIE_EVENT_TYPE_MAX,
74 };
75
76 #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0)
77 #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0)
78
79 struct dwc_pcie_pmu {
80 struct pmu pmu;
81 struct pci_dev *pdev; /* Root Port device */
82 u16 ras_des_offset;
83 u32 nr_lanes;
84
85 struct hlist_node cpuhp_node;
86 struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX];
87 int on_cpu;
88 };
89
90 #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu))
91
92 static int dwc_pcie_pmu_hp_state;
93 static struct list_head dwc_pcie_dev_info_head =
94 LIST_HEAD_INIT(dwc_pcie_dev_info_head);
95 static bool notify;
96
97 struct dwc_pcie_dev_info {
98 struct platform_device *plat_dev;
99 struct pci_dev *pdev;
100 struct list_head dev_node;
101 };
102
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)103 static ssize_t cpumask_show(struct device *dev,
104 struct device_attribute *attr,
105 char *buf)
106 {
107 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
108
109 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
110 }
111 static DEVICE_ATTR_RO(cpumask);
112
113 static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = {
114 &dev_attr_cpumask.attr,
115 NULL
116 };
117
118 static struct attribute_group dwc_pcie_cpumask_attr_group = {
119 .attrs = dwc_pcie_pmu_cpumask_attrs,
120 };
121
122 struct dwc_pcie_format_attr {
123 struct device_attribute attr;
124 u64 field;
125 int config;
126 };
127
128 PMU_FORMAT_ATTR(eventid, "config:0-15");
129 PMU_FORMAT_ATTR(type, "config:16-19");
130 PMU_FORMAT_ATTR(lane, "config:20-27");
131
132 static struct attribute *dwc_pcie_format_attrs[] = {
133 &format_attr_type.attr,
134 &format_attr_eventid.attr,
135 &format_attr_lane.attr,
136 NULL,
137 };
138
139 static struct attribute_group dwc_pcie_format_attrs_group = {
140 .name = "format",
141 .attrs = dwc_pcie_format_attrs,
142 };
143
144 struct dwc_pcie_event_attr {
145 struct device_attribute attr;
146 enum dwc_pcie_event_type type;
147 u16 eventid;
148 u8 lane;
149 };
150
dwc_pcie_event_show(struct device * dev,struct device_attribute * attr,char * buf)151 static ssize_t dwc_pcie_event_show(struct device *dev,
152 struct device_attribute *attr, char *buf)
153 {
154 struct dwc_pcie_event_attr *eattr;
155
156 eattr = container_of(attr, typeof(*eattr), attr);
157
158 if (eattr->type == DWC_PCIE_LANE_EVENT)
159 return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n",
160 eattr->eventid, eattr->type);
161 else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT)
162 return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n",
163 eattr->eventid, eattr->type);
164
165 return 0;
166 }
167
168 #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \
169 (&((struct dwc_pcie_event_attr[]) {{ \
170 .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \
171 .type = _type, \
172 .eventid = _eventid, \
173 .lane = _lane, \
174 }})[0].attr.attr)
175
176 #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \
177 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0)
178 #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \
179 DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0)
180
181 static struct attribute *dwc_pcie_pmu_time_event_attrs[] = {
182 /* Group #0 */
183 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00),
184 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01),
185 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02),
186 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03),
187 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04),
188 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05),
189 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06),
190 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07),
191 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08),
192 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09),
193
194 /* Group #1 */
195 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20),
196 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21),
197 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22),
198 DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23),
199
200 /*
201 * Leave it to the user to specify the lane ID to avoid generating
202 * a list of hundreds of events.
203 */
204 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600),
205 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601),
206 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602),
207 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603),
208 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604),
209 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605),
210 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606),
211 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700),
212 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701),
213 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702),
214 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703),
215 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704),
216 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705),
217 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706),
218 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707),
219 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708),
220 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709),
221 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A),
222 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B),
223 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C),
224 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F),
225 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710),
226 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711),
227 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712),
228 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713),
229 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714),
230 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715),
231 DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716),
232 DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717),
233 NULL
234 };
235
236 static const struct attribute_group dwc_pcie_event_attrs_group = {
237 .name = "events",
238 .attrs = dwc_pcie_pmu_time_event_attrs,
239 };
240
241 static const struct attribute_group *dwc_pcie_attr_groups[] = {
242 &dwc_pcie_event_attrs_group,
243 &dwc_pcie_format_attrs_group,
244 &dwc_pcie_cpumask_attr_group,
245 NULL
246 };
247
dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu * pcie_pmu,bool enable)248 static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,
249 bool enable)
250 {
251 struct pci_dev *pdev = pcie_pmu->pdev;
252 u16 ras_des_offset = pcie_pmu->ras_des_offset;
253
254 if (enable)
255 pci_clear_and_set_config_dword(pdev,
256 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
257 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
258 else
259 pci_clear_and_set_config_dword(pdev,
260 ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
261 DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);
262 }
263
dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu * pcie_pmu,bool enable)264 static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
265 bool enable)
266 {
267 struct pci_dev *pdev = pcie_pmu->pdev;
268 u16 ras_des_offset = pcie_pmu->ras_des_offset;
269
270 pci_clear_and_set_config_dword(pdev,
271 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL,
272 DWC_PCIE_TIME_BASED_TIMER_START, enable);
273 }
274
dwc_pcie_pmu_read_lane_event_counter(struct perf_event * event)275 static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event)
276 {
277 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
278 struct pci_dev *pdev = pcie_pmu->pdev;
279 u16 ras_des_offset = pcie_pmu->ras_des_offset;
280 u32 val;
281
282 pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);
283
284 return val;
285 }
286
dwc_pcie_pmu_read_time_based_counter(struct perf_event * event)287 static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event)
288 {
289 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
290 struct pci_dev *pdev = pcie_pmu->pdev;
291 int event_id = DWC_PCIE_EVENT_ID(event);
292 u16 ras_des_offset = pcie_pmu->ras_des_offset;
293 u32 lo, hi, ss;
294 u64 val;
295
296 /*
297 * The 64-bit value of the data counter is spread across two
298 * registers that are not synchronized. In order to read them
299 * atomically, ensure that the high 32 bits match before and after
300 * reading the low 32 bits.
301 */
302 pci_read_config_dword(pdev,
303 ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi);
304 do {
305 /* snapshot the high 32 bits */
306 ss = hi;
307
308 pci_read_config_dword(
309 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW,
310 &lo);
311 pci_read_config_dword(
312 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH,
313 &hi);
314 } while (hi != ss);
315
316 val = ((u64)hi << 32) | lo;
317 /*
318 * The Group#1 event measures the amount of data processed in 16-byte
319 * units. Simplify the end-user interface by multiplying the counter
320 * at the point of read.
321 */
322 if (event_id >= 0x20 && event_id <= 0x23)
323 val *= 16;
324
325 return val;
326 }
327
dwc_pcie_pmu_event_update(struct perf_event * event)328 static void dwc_pcie_pmu_event_update(struct perf_event *event)
329 {
330 struct hw_perf_event *hwc = &event->hw;
331 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
332 u64 delta, prev, now = 0;
333
334 do {
335 prev = local64_read(&hwc->prev_count);
336
337 if (type == DWC_PCIE_LANE_EVENT)
338 now = dwc_pcie_pmu_read_lane_event_counter(event);
339 else if (type == DWC_PCIE_TIME_BASE_EVENT)
340 now = dwc_pcie_pmu_read_time_based_counter(event);
341
342 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
343
344 delta = (now - prev) & DWC_PCIE_MAX_PERIOD;
345 /* 32-bit counter for Lane Event Counting */
346 if (type == DWC_PCIE_LANE_EVENT)
347 delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD;
348
349 local64_add(delta, &event->count);
350 }
351
dwc_pcie_pmu_event_init(struct perf_event * event)352 static int dwc_pcie_pmu_event_init(struct perf_event *event)
353 {
354 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
355 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
356 struct perf_event *sibling;
357 u32 lane;
358
359 if (event->attr.type != event->pmu->type)
360 return -ENOENT;
361
362 /* We don't support sampling */
363 if (is_sampling_event(event))
364 return -EINVAL;
365
366 /* We cannot support task bound events */
367 if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
368 return -EINVAL;
369
370 if (event->group_leader != event &&
371 !is_software_event(event->group_leader))
372 return -EINVAL;
373
374 for_each_sibling_event(sibling, event->group_leader) {
375 if (sibling->pmu != event->pmu && !is_software_event(sibling))
376 return -EINVAL;
377 }
378
379 if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX)
380 return -EINVAL;
381
382 if (type == DWC_PCIE_LANE_EVENT) {
383 lane = DWC_PCIE_EVENT_LANE(event);
384 if (lane < 0 || lane >= pcie_pmu->nr_lanes)
385 return -EINVAL;
386 }
387
388 event->cpu = pcie_pmu->on_cpu;
389
390 return 0;
391 }
392
dwc_pcie_pmu_event_start(struct perf_event * event,int flags)393 static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags)
394 {
395 struct hw_perf_event *hwc = &event->hw;
396 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
397 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
398
399 hwc->state = 0;
400 local64_set(&hwc->prev_count, 0);
401
402 if (type == DWC_PCIE_LANE_EVENT)
403 dwc_pcie_pmu_lane_event_enable(pcie_pmu, true);
404 else if (type == DWC_PCIE_TIME_BASE_EVENT)
405 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
406 }
407
dwc_pcie_pmu_event_stop(struct perf_event * event,int flags)408 static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags)
409 {
410 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
411 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
412 struct hw_perf_event *hwc = &event->hw;
413
414 if (event->hw.state & PERF_HES_STOPPED)
415 return;
416
417 if (type == DWC_PCIE_LANE_EVENT)
418 dwc_pcie_pmu_lane_event_enable(pcie_pmu, false);
419 else if (type == DWC_PCIE_TIME_BASE_EVENT)
420 dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
421
422 dwc_pcie_pmu_event_update(event);
423 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
424 }
425
dwc_pcie_pmu_event_add(struct perf_event * event,int flags)426 static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags)
427 {
428 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
429 struct pci_dev *pdev = pcie_pmu->pdev;
430 struct hw_perf_event *hwc = &event->hw;
431 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
432 int event_id = DWC_PCIE_EVENT_ID(event);
433 int lane = DWC_PCIE_EVENT_LANE(event);
434 u16 ras_des_offset = pcie_pmu->ras_des_offset;
435 u32 ctrl;
436
437 /* one counter for each type and it is in use */
438 if (pcie_pmu->event[type])
439 return -ENOSPC;
440
441 pcie_pmu->event[type] = event;
442 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
443
444 if (type == DWC_PCIE_LANE_EVENT) {
445 /* EVENT_COUNTER_DATA_REG needs clear manually */
446 ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
447 FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
448 FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) |
449 FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
450 pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
451 ctrl);
452 } else if (type == DWC_PCIE_TIME_BASE_EVENT) {
453 /*
454 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely
455 * use it with any manually controlled duration. And it is
456 * cleared when next measurement starts.
457 */
458 ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) |
459 FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL,
460 DWC_PCIE_DURATION_MANUAL_CTL) |
461 DWC_PCIE_TIME_BASED_CNT_ENABLE;
462 pci_write_config_dword(
463 pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl);
464 }
465
466 if (flags & PERF_EF_START)
467 dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD);
468
469 perf_event_update_userpage(event);
470
471 return 0;
472 }
473
dwc_pcie_pmu_event_del(struct perf_event * event,int flags)474 static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags)
475 {
476 struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
477 enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
478
479 dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);
480 perf_event_update_userpage(event);
481 pcie_pmu->event[type] = NULL;
482 }
483
dwc_pcie_pmu_remove_cpuhp_instance(void * hotplug_node)484 static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)
485 {
486 cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node);
487 }
488
489 /*
490 * Find the binded DES capability device info of a PCI device.
491 * @pdev: The PCI device.
492 */
dwc_pcie_find_dev_info(struct pci_dev * pdev)493 static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev)
494 {
495 struct dwc_pcie_dev_info *dev_info;
496
497 list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node)
498 if (dev_info->pdev == pdev)
499 return dev_info;
500
501 return NULL;
502 }
503
dwc_pcie_unregister_pmu(void * data)504 static void dwc_pcie_unregister_pmu(void *data)
505 {
506 struct dwc_pcie_pmu *pcie_pmu = data;
507
508 perf_pmu_unregister(&pcie_pmu->pmu);
509 }
510
dwc_pcie_des_cap(struct pci_dev * pdev)511 static u16 dwc_pcie_des_cap(struct pci_dev *pdev)
512 {
513 const struct dwc_pcie_vsec_id *vid;
514 u16 vsec;
515 u32 val;
516
517 if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT))
518 return 0;
519
520 for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) {
521 vsec = pci_find_vsec_capability(pdev, vid->vendor_id,
522 vid->vsec_id);
523 if (vsec) {
524 pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER,
525 &val);
526 if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) {
527 pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n");
528 return vsec;
529 }
530 }
531 }
532 return 0;
533 }
534
dwc_pcie_unregister_dev(struct dwc_pcie_dev_info * dev_info)535 static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info)
536 {
537 platform_device_unregister(dev_info->plat_dev);
538 list_del(&dev_info->dev_node);
539 kfree(dev_info);
540 }
541
dwc_pcie_register_dev(struct pci_dev * pdev)542 static int dwc_pcie_register_dev(struct pci_dev *pdev)
543 {
544 struct platform_device *plat_dev;
545 struct dwc_pcie_dev_info *dev_info;
546 u32 sbdf;
547
548 sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn);
549 plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0);
550 if (IS_ERR(plat_dev))
551 return PTR_ERR(plat_dev);
552
553 dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
554 if (!dev_info) {
555 platform_device_unregister(plat_dev);
556 return -ENOMEM;
557 }
558
559 /* Cache platform device to handle pci device hotplug */
560 dev_info->plat_dev = plat_dev;
561 dev_info->pdev = pdev;
562 list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head);
563
564 return 0;
565 }
566
dwc_pcie_pmu_notifier(struct notifier_block * nb,unsigned long action,void * data)567 static int dwc_pcie_pmu_notifier(struct notifier_block *nb,
568 unsigned long action, void *data)
569 {
570 struct device *dev = data;
571 struct pci_dev *pdev = to_pci_dev(dev);
572 struct dwc_pcie_dev_info *dev_info;
573
574 switch (action) {
575 case BUS_NOTIFY_ADD_DEVICE:
576 if (!dwc_pcie_des_cap(pdev))
577 return NOTIFY_DONE;
578 if (dwc_pcie_register_dev(pdev))
579 return NOTIFY_BAD;
580 break;
581 case BUS_NOTIFY_DEL_DEVICE:
582 dev_info = dwc_pcie_find_dev_info(pdev);
583 if (!dev_info)
584 return NOTIFY_DONE;
585 dwc_pcie_unregister_dev(dev_info);
586 break;
587 }
588
589 return NOTIFY_OK;
590 }
591
592 static struct notifier_block dwc_pcie_pmu_nb = {
593 .notifier_call = dwc_pcie_pmu_notifier,
594 };
595
dwc_pcie_pmu_probe(struct platform_device * plat_dev)596 static int dwc_pcie_pmu_probe(struct platform_device *plat_dev)
597 {
598 struct pci_dev *pdev;
599 struct dwc_pcie_pmu *pcie_pmu;
600 char *name;
601 u32 sbdf;
602 u16 vsec;
603 int ret;
604
605 sbdf = plat_dev->id;
606 pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff),
607 sbdf & 0xff);
608 if (!pdev) {
609 pr_err("No pdev found for the sbdf 0x%x\n", sbdf);
610 return -ENODEV;
611 }
612
613 vsec = dwc_pcie_des_cap(pdev);
614 if (!vsec)
615 return -ENODEV;
616
617 pci_dev_put(pdev);
618 name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf);
619 if (!name)
620 return -ENOMEM;
621
622 pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
623 if (!pcie_pmu)
624 return -ENOMEM;
625
626 pcie_pmu->pdev = pdev;
627 pcie_pmu->ras_des_offset = vsec;
628 pcie_pmu->nr_lanes = pcie_get_width_cap(pdev);
629 pcie_pmu->on_cpu = -1;
630 pcie_pmu->pmu = (struct pmu){
631 .name = name,
632 .parent = &plat_dev->dev,
633 .module = THIS_MODULE,
634 .attr_groups = dwc_pcie_attr_groups,
635 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
636 .task_ctx_nr = perf_invalid_context,
637 .event_init = dwc_pcie_pmu_event_init,
638 .add = dwc_pcie_pmu_event_add,
639 .del = dwc_pcie_pmu_event_del,
640 .start = dwc_pcie_pmu_event_start,
641 .stop = dwc_pcie_pmu_event_stop,
642 .read = dwc_pcie_pmu_event_update,
643 };
644
645 /* Add this instance to the list used by the offline callback */
646 ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state,
647 &pcie_pmu->cpuhp_node);
648 if (ret) {
649 pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf);
650 return ret;
651 }
652
653 /* Unwind when platform driver removes */
654 ret = devm_add_action_or_reset(&plat_dev->dev,
655 dwc_pcie_pmu_remove_cpuhp_instance,
656 &pcie_pmu->cpuhp_node);
657 if (ret)
658 return ret;
659
660 ret = perf_pmu_register(&pcie_pmu->pmu, name, -1);
661 if (ret) {
662 pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf);
663 return ret;
664 }
665 ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu,
666 pcie_pmu);
667 if (ret)
668 return ret;
669
670 return 0;
671 }
672
dwc_pcie_pmu_online_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)673 static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
674 {
675 struct dwc_pcie_pmu *pcie_pmu;
676
677 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
678 if (pcie_pmu->on_cpu == -1)
679 pcie_pmu->on_cpu = cpumask_local_spread(
680 0, dev_to_node(&pcie_pmu->pdev->dev));
681
682 return 0;
683 }
684
dwc_pcie_pmu_offline_cpu(unsigned int cpu,struct hlist_node * cpuhp_node)685 static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
686 {
687 struct dwc_pcie_pmu *pcie_pmu;
688 struct pci_dev *pdev;
689 unsigned int target;
690 int node;
691
692 pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
693 /* Nothing to do if this CPU doesn't own the PMU */
694 if (cpu != pcie_pmu->on_cpu)
695 return 0;
696
697 pcie_pmu->on_cpu = -1;
698 pdev = pcie_pmu->pdev;
699 node = dev_to_node(&pdev->dev);
700
701 target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
702 if (target >= nr_cpu_ids)
703 target = cpumask_any_but(cpu_online_mask, cpu);
704
705 if (target >= nr_cpu_ids) {
706 pci_err(pdev, "There is no CPU to set\n");
707 return 0;
708 }
709
710 /* This PMU does NOT support interrupt, just migrate context. */
711 perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
712 pcie_pmu->on_cpu = target;
713
714 return 0;
715 }
716
717 static struct platform_driver dwc_pcie_pmu_driver = {
718 .probe = dwc_pcie_pmu_probe,
719 .driver = {.name = "dwc_pcie_pmu",},
720 };
721
dwc_pcie_cleanup_devices(void)722 static void dwc_pcie_cleanup_devices(void)
723 {
724 struct dwc_pcie_dev_info *dev_info, *tmp;
725
726 list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) {
727 dwc_pcie_unregister_dev(dev_info);
728 }
729 }
730
dwc_pcie_pmu_init(void)731 static int __init dwc_pcie_pmu_init(void)
732 {
733 struct pci_dev *pdev = NULL;
734 int ret;
735
736 for_each_pci_dev(pdev) {
737 if (!dwc_pcie_des_cap(pdev))
738 continue;
739
740 ret = dwc_pcie_register_dev(pdev);
741 if (ret) {
742 pci_dev_put(pdev);
743 goto err_cleanup;
744 }
745 }
746
747 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
748 "perf/dwc_pcie_pmu:online",
749 dwc_pcie_pmu_online_cpu,
750 dwc_pcie_pmu_offline_cpu);
751 if (ret < 0)
752 goto err_cleanup;
753
754 dwc_pcie_pmu_hp_state = ret;
755
756 ret = platform_driver_register(&dwc_pcie_pmu_driver);
757 if (ret)
758 goto err_remove_cpuhp;
759
760 ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
761 if (ret)
762 goto err_unregister_driver;
763 notify = true;
764
765 return 0;
766
767 err_unregister_driver:
768 platform_driver_unregister(&dwc_pcie_pmu_driver);
769 err_remove_cpuhp:
770 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
771 err_cleanup:
772 dwc_pcie_cleanup_devices();
773 return ret;
774 }
775
dwc_pcie_pmu_exit(void)776 static void __exit dwc_pcie_pmu_exit(void)
777 {
778 if (notify)
779 bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
780 dwc_pcie_cleanup_devices();
781 platform_driver_unregister(&dwc_pcie_pmu_driver);
782 cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
783 }
784
785 module_init(dwc_pcie_pmu_init);
786 module_exit(dwc_pcie_pmu_exit);
787
788 MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller");
789 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
790 MODULE_LICENSE("GPL v2");
791