1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver
3 *
4 * Copyright (C) 2021 Marvell.
5 */
6
7 #include <linux/init.h>
8 #include <linux/io.h>
9 #include <linux/module.h>
10 #include <linux/of.h>
11 #include <linux/of_address.h>
12 #include <linux/of_device.h>
13 #include <linux/perf_event.h>
14 #include <linux/hrtimer.h>
15 #include <linux/acpi.h>
16
17 /* Performance Counters Operating Mode Control Registers */
18 #define DDRC_PERF_CNT_OP_MODE_CTRL 0x8020
19 #define OP_MODE_CTRL_VAL_MANNUAL 0x1
20
21 /* Performance Counters Start Operation Control Registers */
22 #define DDRC_PERF_CNT_START_OP_CTRL 0x8028
23 #define START_OP_CTRL_VAL_START 0x1ULL
24 #define START_OP_CTRL_VAL_ACTIVE 0x2
25
26 /* Performance Counters End Operation Control Registers */
27 #define DDRC_PERF_CNT_END_OP_CTRL 0x8030
28 #define END_OP_CTRL_VAL_END 0x1ULL
29
30 /* Performance Counters End Status Registers */
31 #define DDRC_PERF_CNT_END_STATUS 0x8038
32 #define END_STATUS_VAL_END_TIMER_MODE_END 0x1
33
34 /* Performance Counters Configuration Registers */
35 #define DDRC_PERF_CFG_BASE 0x8040
36
37 /* 8 Generic event counter + 2 fixed event counters */
38 #define DDRC_PERF_NUM_GEN_COUNTERS 8
39 #define DDRC_PERF_NUM_FIX_COUNTERS 2
40 #define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS
41 #define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1)
42 #define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \
43 DDRC_PERF_NUM_FIX_COUNTERS)
44
45 /* Generic event counter registers */
46 #define DDRC_PERF_CFG(n) (DDRC_PERF_CFG_BASE + 8 * (n))
47 #define EVENT_ENABLE BIT_ULL(63)
48
49 /* Two dedicated event counters for DDR reads and writes */
50 #define EVENT_DDR_READS 101
51 #define EVENT_DDR_WRITES 100
52
53 /*
54 * programmable events IDs in programmable event counters.
55 * DO NOT change these event-id numbers, they are used to
56 * program event bitmap in h/w.
57 */
58 #define EVENT_OP_IS_ZQLATCH 55
59 #define EVENT_OP_IS_ZQSTART 54
60 #define EVENT_OP_IS_TCR_MRR 53
61 #define EVENT_OP_IS_DQSOSC_MRR 52
62 #define EVENT_OP_IS_DQSOSC_MPC 51
63 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 50
64 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 49
65 #define EVENT_BSM_STARVATION 48
66 #define EVENT_BSM_ALLOC 47
67 #define EVENT_LPR_REQ_WITH_NOCREDIT 46
68 #define EVENT_HPR_REQ_WITH_NOCREDIT 45
69 #define EVENT_OP_IS_ZQCS 44
70 #define EVENT_OP_IS_ZQCL 43
71 #define EVENT_OP_IS_LOAD_MODE 42
72 #define EVENT_OP_IS_SPEC_REF 41
73 #define EVENT_OP_IS_CRIT_REF 40
74 #define EVENT_OP_IS_REFRESH 39
75 #define EVENT_OP_IS_ENTER_MPSM 35
76 #define EVENT_OP_IS_ENTER_POWERDOWN 31
77 #define EVENT_OP_IS_ENTER_SELFREF 27
78 #define EVENT_WAW_HAZARD 26
79 #define EVENT_RAW_HAZARD 25
80 #define EVENT_WAR_HAZARD 24
81 #define EVENT_WRITE_COMBINE 23
82 #define EVENT_RDWR_TRANSITIONS 22
83 #define EVENT_PRECHARGE_FOR_OTHER 21
84 #define EVENT_PRECHARGE_FOR_RDWR 20
85 #define EVENT_OP_IS_PRECHARGE 19
86 #define EVENT_OP_IS_MWR 18
87 #define EVENT_OP_IS_WR 17
88 #define EVENT_OP_IS_RD 16
89 #define EVENT_OP_IS_RD_ACTIVATE 15
90 #define EVENT_OP_IS_RD_OR_WR 14
91 #define EVENT_OP_IS_ACTIVATE 13
92 #define EVENT_WR_XACT_WHEN_CRITICAL 12
93 #define EVENT_LPR_XACT_WHEN_CRITICAL 11
94 #define EVENT_HPR_XACT_WHEN_CRITICAL 10
95 #define EVENT_DFI_RD_DATA_CYCLES 9
96 #define EVENT_DFI_WR_DATA_CYCLES 8
97 #define EVENT_ACT_BYPASS 7
98 #define EVENT_READ_BYPASS 6
99 #define EVENT_HIF_HI_PRI_RD 5
100 #define EVENT_HIF_RMW 4
101 #define EVENT_HIF_RD 3
102 #define EVENT_HIF_WR 2
103 #define EVENT_HIF_RD_OR_WR 1
104
105 /* Event counter value registers */
106 #define DDRC_PERF_CNT_VALUE_BASE 0x8080
107 #define DDRC_PERF_CNT_VALUE(n) (DDRC_PERF_CNT_VALUE_BASE + 8 * (n))
108
109 /* Fixed event counter enable/disable register */
110 #define DDRC_PERF_CNT_FREERUN_EN 0x80C0
111 #define DDRC_PERF_FREERUN_WRITE_EN 0x1
112 #define DDRC_PERF_FREERUN_READ_EN 0x2
113
114 /* Fixed event counter control register */
115 #define DDRC_PERF_CNT_FREERUN_CTRL 0x80C8
116 #define DDRC_FREERUN_WRITE_CNT_CLR 0x1
117 #define DDRC_FREERUN_READ_CNT_CLR 0x2
118
119 /* Fixed event counter value register */
120 #define DDRC_PERF_CNT_VALUE_WR_OP 0x80D0
121 #define DDRC_PERF_CNT_VALUE_RD_OP 0x80D8
122 #define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48)
123 #define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0)
124
125 struct cn10k_ddr_pmu {
126 struct pmu pmu;
127 void __iomem *base;
128 unsigned int cpu;
129 struct device *dev;
130 int active_events;
131 struct perf_event *events[DDRC_PERF_NUM_COUNTERS];
132 struct hrtimer hrtimer;
133 struct hlist_node node;
134 };
135
136 #define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu)
137
cn10k_ddr_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)138 static ssize_t cn10k_ddr_pmu_event_show(struct device *dev,
139 struct device_attribute *attr,
140 char *page)
141 {
142 struct perf_pmu_events_attr *pmu_attr;
143
144 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
145 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
146
147 }
148
149 #define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \
150 PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id)
151
152 static struct attribute *cn10k_ddr_perf_events_attrs[] = {
153 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
154 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
155 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
156 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
157 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
158 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
159 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
160 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES),
161 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES),
162 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
163 EVENT_HPR_XACT_WHEN_CRITICAL),
164 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
165 EVENT_LPR_XACT_WHEN_CRITICAL),
166 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
167 EVENT_WR_XACT_WHEN_CRITICAL),
168 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
169 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR),
170 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE),
171 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
172 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
173 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
174 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
175 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR),
176 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
177 EVENT_PRECHARGE_FOR_OTHER),
178 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
179 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
180 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
181 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
182 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
183 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
184 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN),
185 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
186 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
187 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
188 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
189 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
190 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
191 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
192 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit,
193 EVENT_HPR_REQ_WITH_NOCREDIT),
194 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit,
195 EVENT_LPR_REQ_WITH_NOCREDIT),
196 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
197 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
198 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
199 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
200 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
201 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
202 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
203 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
204 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
205 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
206 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
207 /* Free run event counters */
208 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
209 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
210 NULL
211 };
212
213 static struct attribute_group cn10k_ddr_perf_events_attr_group = {
214 .name = "events",
215 .attrs = cn10k_ddr_perf_events_attrs,
216 };
217
218 PMU_FORMAT_ATTR(event, "config:0-8");
219
220 static struct attribute *cn10k_ddr_perf_format_attrs[] = {
221 &format_attr_event.attr,
222 NULL,
223 };
224
225 static struct attribute_group cn10k_ddr_perf_format_attr_group = {
226 .name = "format",
227 .attrs = cn10k_ddr_perf_format_attrs,
228 };
229
cn10k_ddr_perf_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)230 static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,
231 struct device_attribute *attr,
232 char *buf)
233 {
234 struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);
235
236 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
237 }
238
239 static struct device_attribute cn10k_ddr_perf_cpumask_attr =
240 __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL);
241
242 static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = {
243 &cn10k_ddr_perf_cpumask_attr.attr,
244 NULL,
245 };
246
247 static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = {
248 .attrs = cn10k_ddr_perf_cpumask_attrs,
249 };
250
251 static const struct attribute_group *cn10k_attr_groups[] = {
252 &cn10k_ddr_perf_events_attr_group,
253 &cn10k_ddr_perf_format_attr_group,
254 &cn10k_ddr_perf_cpumask_attr_group,
255 NULL,
256 };
257
258 /* Default poll timeout is 100 sec, which is very sufficient for
259 * 48 bit counter incremented max at 5.6 GT/s, which may take many
260 * hours to overflow.
261 */
262 static unsigned long cn10k_ddr_pmu_poll_period_sec = 100;
263 module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644);
264
cn10k_ddr_pmu_timer_period(void)265 static ktime_t cn10k_ddr_pmu_timer_period(void)
266 {
267 return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC);
268 }
269
ddr_perf_get_event_bitmap(int eventid,u64 * event_bitmap)270 static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap)
271 {
272 switch (eventid) {
273 case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD:
274 case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH:
275 *event_bitmap = (1ULL << (eventid - 1));
276 break;
277 case EVENT_OP_IS_ENTER_SELFREF:
278 case EVENT_OP_IS_ENTER_POWERDOWN:
279 case EVENT_OP_IS_ENTER_MPSM:
280 *event_bitmap = (0xFULL << (eventid - 1));
281 break;
282 default:
283 pr_err("%s Invalid eventid %d\n", __func__, eventid);
284 return -EINVAL;
285 }
286
287 return 0;
288 }
289
cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu * pmu,struct perf_event * event)290 static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu,
291 struct perf_event *event)
292 {
293 u8 config = event->attr.config;
294 int i;
295
296 /* DDR read free-run counter index */
297 if (config == EVENT_DDR_READS) {
298 pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event;
299 return DDRC_PERF_READ_COUNTER_IDX;
300 }
301
302 /* DDR write free-run counter index */
303 if (config == EVENT_DDR_WRITES) {
304 pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event;
305 return DDRC_PERF_WRITE_COUNTER_IDX;
306 }
307
308 /* Allocate DDR generic counters */
309 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
310 if (pmu->events[i] == NULL) {
311 pmu->events[i] = event;
312 return i;
313 }
314 }
315
316 return -ENOENT;
317 }
318
cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu * pmu,int counter)319 static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter)
320 {
321 pmu->events[counter] = NULL;
322 }
323
cn10k_ddr_perf_event_init(struct perf_event * event)324 static int cn10k_ddr_perf_event_init(struct perf_event *event)
325 {
326 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
327 struct hw_perf_event *hwc = &event->hw;
328
329 if (event->attr.type != event->pmu->type)
330 return -ENOENT;
331
332 if (is_sampling_event(event)) {
333 dev_info(pmu->dev, "Sampling not supported!\n");
334 return -EOPNOTSUPP;
335 }
336
337 if (event->cpu < 0) {
338 dev_warn(pmu->dev, "Can't provide per-task data!\n");
339 return -EOPNOTSUPP;
340 }
341
342 /* We must NOT create groups containing mixed PMUs */
343 if (event->group_leader->pmu != event->pmu &&
344 !is_software_event(event->group_leader))
345 return -EINVAL;
346
347 /* Set ownership of event to one CPU, same event can not be observed
348 * on multiple cpus at same time.
349 */
350 event->cpu = pmu->cpu;
351 hwc->idx = -1;
352 return 0;
353 }
354
cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu * pmu,int counter,bool enable)355 static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu,
356 int counter, bool enable)
357 {
358 u32 reg;
359 u64 val;
360
361 if (counter > DDRC_PERF_NUM_COUNTERS) {
362 pr_err("Error: unsupported counter %d\n", counter);
363 return;
364 }
365
366 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
367 reg = DDRC_PERF_CFG(counter);
368 val = readq_relaxed(pmu->base + reg);
369
370 if (enable)
371 val |= EVENT_ENABLE;
372 else
373 val &= ~EVENT_ENABLE;
374
375 writeq_relaxed(val, pmu->base + reg);
376 } else {
377 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_FREERUN_EN);
378 if (enable) {
379 if (counter == DDRC_PERF_READ_COUNTER_IDX)
380 val |= DDRC_PERF_FREERUN_READ_EN;
381 else
382 val |= DDRC_PERF_FREERUN_WRITE_EN;
383 } else {
384 if (counter == DDRC_PERF_READ_COUNTER_IDX)
385 val &= ~DDRC_PERF_FREERUN_READ_EN;
386 else
387 val &= ~DDRC_PERF_FREERUN_WRITE_EN;
388 }
389 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_EN);
390 }
391 }
392
cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu * pmu,int counter)393 static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter)
394 {
395 u64 val;
396
397 if (counter == DDRC_PERF_READ_COUNTER_IDX)
398 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_RD_OP);
399
400 if (counter == DDRC_PERF_WRITE_COUNTER_IDX)
401 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_WR_OP);
402
403 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE(counter));
404 return val;
405 }
406
cn10k_ddr_perf_event_update(struct perf_event * event)407 static void cn10k_ddr_perf_event_update(struct perf_event *event)
408 {
409 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
410 struct hw_perf_event *hwc = &event->hw;
411 u64 prev_count, new_count, mask;
412
413 do {
414 prev_count = local64_read(&hwc->prev_count);
415 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
416 } while (local64_xchg(&hwc->prev_count, new_count) != prev_count);
417
418 mask = DDRC_PERF_CNT_MAX_VALUE;
419
420 local64_add((new_count - prev_count) & mask, &event->count);
421 }
422
cn10k_ddr_perf_event_start(struct perf_event * event,int flags)423 static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags)
424 {
425 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
426 struct hw_perf_event *hwc = &event->hw;
427 int counter = hwc->idx;
428
429 local64_set(&hwc->prev_count, 0);
430
431 cn10k_ddr_perf_counter_enable(pmu, counter, true);
432
433 hwc->state = 0;
434 }
435
cn10k_ddr_perf_event_add(struct perf_event * event,int flags)436 static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags)
437 {
438 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
439 struct hw_perf_event *hwc = &event->hw;
440 u8 config = event->attr.config;
441 int counter, ret;
442 u32 reg_offset;
443 u64 val;
444
445 counter = cn10k_ddr_perf_alloc_counter(pmu, event);
446 if (counter < 0)
447 return -EAGAIN;
448
449 pmu->active_events++;
450 hwc->idx = counter;
451
452 if (pmu->active_events == 1)
453 hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(),
454 HRTIMER_MODE_REL_PINNED);
455
456 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
457 /* Generic counters, configure event id */
458 reg_offset = DDRC_PERF_CFG(counter);
459 ret = ddr_perf_get_event_bitmap(config, &val);
460 if (ret)
461 return ret;
462
463 writeq_relaxed(val, pmu->base + reg_offset);
464 } else {
465 /* fixed event counter, clear counter value */
466 if (counter == DDRC_PERF_READ_COUNTER_IDX)
467 val = DDRC_FREERUN_READ_CNT_CLR;
468 else
469 val = DDRC_FREERUN_WRITE_CNT_CLR;
470
471 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_CTRL);
472 }
473
474 hwc->state |= PERF_HES_STOPPED;
475
476 if (flags & PERF_EF_START)
477 cn10k_ddr_perf_event_start(event, flags);
478
479 return 0;
480 }
481
cn10k_ddr_perf_event_stop(struct perf_event * event,int flags)482 static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags)
483 {
484 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
485 struct hw_perf_event *hwc = &event->hw;
486 int counter = hwc->idx;
487
488 cn10k_ddr_perf_counter_enable(pmu, counter, false);
489
490 if (flags & PERF_EF_UPDATE)
491 cn10k_ddr_perf_event_update(event);
492
493 hwc->state |= PERF_HES_STOPPED;
494 }
495
cn10k_ddr_perf_event_del(struct perf_event * event,int flags)496 static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags)
497 {
498 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
499 struct hw_perf_event *hwc = &event->hw;
500 int counter = hwc->idx;
501
502 cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE);
503
504 cn10k_ddr_perf_free_counter(pmu, counter);
505 pmu->active_events--;
506 hwc->idx = -1;
507
508 /* Cancel timer when no events to capture */
509 if (pmu->active_events == 0)
510 hrtimer_cancel(&pmu->hrtimer);
511 }
512
cn10k_ddr_perf_pmu_enable(struct pmu * pmu)513 static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu)
514 {
515 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
516
517 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
518 DDRC_PERF_CNT_START_OP_CTRL);
519 }
520
cn10k_ddr_perf_pmu_disable(struct pmu * pmu)521 static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu)
522 {
523 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
524
525 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
526 DDRC_PERF_CNT_END_OP_CTRL);
527 }
528
cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu * pmu)529 static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu)
530 {
531 struct hw_perf_event *hwc;
532 int i;
533
534 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
535 if (pmu->events[i] == NULL)
536 continue;
537
538 cn10k_ddr_perf_event_update(pmu->events[i]);
539 }
540
541 /* Reset previous count as h/w counter are reset */
542 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
543 if (pmu->events[i] == NULL)
544 continue;
545
546 hwc = &pmu->events[i]->hw;
547 local64_set(&hwc->prev_count, 0);
548 }
549 }
550
cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu * pmu)551 static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu)
552 {
553 struct perf_event *event;
554 struct hw_perf_event *hwc;
555 u64 prev_count, new_count;
556 u64 value;
557 int i;
558
559 event = pmu->events[DDRC_PERF_READ_COUNTER_IDX];
560 if (event) {
561 hwc = &event->hw;
562 prev_count = local64_read(&hwc->prev_count);
563 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
564
565 /* Overflow condition is when new count less than
566 * previous count
567 */
568 if (new_count < prev_count)
569 cn10k_ddr_perf_event_update(event);
570 }
571
572 event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX];
573 if (event) {
574 hwc = &event->hw;
575 prev_count = local64_read(&hwc->prev_count);
576 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
577
578 /* Overflow condition is when new count less than
579 * previous count
580 */
581 if (new_count < prev_count)
582 cn10k_ddr_perf_event_update(event);
583 }
584
585 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
586 if (pmu->events[i] == NULL)
587 continue;
588
589 value = cn10k_ddr_perf_read_counter(pmu, i);
590 if (value == DDRC_PERF_CNT_MAX_VALUE) {
591 pr_info("Counter-(%d) reached max value\n", i);
592 cn10k_ddr_perf_event_update_all(pmu);
593 cn10k_ddr_perf_pmu_disable(&pmu->pmu);
594 cn10k_ddr_perf_pmu_enable(&pmu->pmu);
595 }
596 }
597
598 return IRQ_HANDLED;
599 }
600
cn10k_ddr_pmu_timer_handler(struct hrtimer * hrtimer)601 static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer)
602 {
603 struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu,
604 hrtimer);
605 unsigned long flags;
606
607 local_irq_save(flags);
608 cn10k_ddr_pmu_overflow_handler(pmu);
609 local_irq_restore(flags);
610
611 hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period());
612 return HRTIMER_RESTART;
613 }
614
cn10k_ddr_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)615 static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
616 {
617 struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu,
618 node);
619 unsigned int target;
620
621 if (cpu != pmu->cpu)
622 return 0;
623
624 target = cpumask_any_but(cpu_online_mask, cpu);
625 if (target >= nr_cpu_ids)
626 return 0;
627
628 perf_pmu_migrate_context(&pmu->pmu, cpu, target);
629 pmu->cpu = target;
630 return 0;
631 }
632
cn10k_ddr_perf_probe(struct platform_device * pdev)633 static int cn10k_ddr_perf_probe(struct platform_device *pdev)
634 {
635 struct cn10k_ddr_pmu *ddr_pmu;
636 struct resource *res;
637 void __iomem *base;
638 char *name;
639 int ret;
640
641 ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL);
642 if (!ddr_pmu)
643 return -ENOMEM;
644
645 ddr_pmu->dev = &pdev->dev;
646 platform_set_drvdata(pdev, ddr_pmu);
647
648 base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
649 if (IS_ERR(base))
650 return PTR_ERR(base);
651
652 ddr_pmu->base = base;
653
654 /* Setup the PMU counter to work in manual mode */
655 writeq_relaxed(OP_MODE_CTRL_VAL_MANNUAL, ddr_pmu->base +
656 DDRC_PERF_CNT_OP_MODE_CTRL);
657
658 ddr_pmu->pmu = (struct pmu) {
659 .module = THIS_MODULE,
660 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
661 .task_ctx_nr = perf_invalid_context,
662 .attr_groups = cn10k_attr_groups,
663 .event_init = cn10k_ddr_perf_event_init,
664 .add = cn10k_ddr_perf_event_add,
665 .del = cn10k_ddr_perf_event_del,
666 .start = cn10k_ddr_perf_event_start,
667 .stop = cn10k_ddr_perf_event_stop,
668 .read = cn10k_ddr_perf_event_update,
669 .pmu_enable = cn10k_ddr_perf_pmu_enable,
670 .pmu_disable = cn10k_ddr_perf_pmu_disable,
671 };
672
673 /* Choose this cpu to collect perf data */
674 ddr_pmu->cpu = raw_smp_processor_id();
675
676 name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx",
677 res->start);
678 if (!name)
679 return -ENOMEM;
680
681 hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
682 ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler;
683
684 cpuhp_state_add_instance_nocalls(
685 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
686 &ddr_pmu->node);
687
688 ret = perf_pmu_register(&ddr_pmu->pmu, name, -1);
689 if (ret)
690 goto error;
691
692 pr_info("CN10K DDR PMU Driver for ddrc@%llx\n", res->start);
693 return 0;
694 error:
695 cpuhp_state_remove_instance_nocalls(
696 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
697 &ddr_pmu->node);
698 return ret;
699 }
700
cn10k_ddr_perf_remove(struct platform_device * pdev)701 static int cn10k_ddr_perf_remove(struct platform_device *pdev)
702 {
703 struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev);
704
705 cpuhp_state_remove_instance_nocalls(
706 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
707 &ddr_pmu->node);
708
709 perf_pmu_unregister(&ddr_pmu->pmu);
710 return 0;
711 }
712
713 #ifdef CONFIG_OF
714 static const struct of_device_id cn10k_ddr_pmu_of_match[] = {
715 { .compatible = "marvell,cn10k-ddr-pmu", },
716 { },
717 };
718 MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match);
719 #endif
720
721 #ifdef CONFIG_ACPI
722 static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = {
723 {"MRVL000A", 0},
724 {},
725 };
726 MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match);
727 #endif
728
729 static struct platform_driver cn10k_ddr_pmu_driver = {
730 .driver = {
731 .name = "cn10k-ddr-pmu",
732 .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match),
733 .acpi_match_table = ACPI_PTR(cn10k_ddr_pmu_acpi_match),
734 .suppress_bind_attrs = true,
735 },
736 .probe = cn10k_ddr_perf_probe,
737 .remove = cn10k_ddr_perf_remove,
738 };
739
cn10k_ddr_pmu_init(void)740 static int __init cn10k_ddr_pmu_init(void)
741 {
742 int ret;
743
744 ret = cpuhp_setup_state_multi(
745 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
746 "perf/marvell/cn10k/ddr:online", NULL,
747 cn10k_ddr_pmu_offline_cpu);
748 if (ret)
749 return ret;
750
751 ret = platform_driver_register(&cn10k_ddr_pmu_driver);
752 if (ret)
753 cpuhp_remove_multi_state(
754 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
755 return ret;
756 }
757
cn10k_ddr_pmu_exit(void)758 static void __exit cn10k_ddr_pmu_exit(void)
759 {
760 platform_driver_unregister(&cn10k_ddr_pmu_driver);
761 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
762 }
763
764 module_init(cn10k_ddr_pmu_init);
765 module_exit(cn10k_ddr_pmu_exit);
766
767 MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>");
768 MODULE_LICENSE("GPL v2");
769