1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2022 Amlogic, Inc. All rights reserved.
4 */
5
6 #include <linux/bitfield.h>
7 #include <linux/init.h>
8 #include <linux/irqreturn.h>
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/of_device.h>
13 #include <linux/of_irq.h>
14 #include <linux/perf_event.h>
15 #include <linux/platform_device.h>
16 #include <linux/printk.h>
17 #include <linux/sysfs.h>
18 #include <linux/types.h>
19
20 #include <soc/amlogic/meson_ddr_pmu.h>
21
22 struct ddr_pmu {
23 struct pmu pmu;
24 struct dmc_info info;
25 struct dmc_counter counters; /* save counters from hw */
26 bool pmu_enabled;
27 struct device *dev;
28 char *name;
29 struct hlist_node node;
30 enum cpuhp_state cpuhp_state;
31 int cpu; /* for cpu hotplug */
32 };
33
34 #define DDR_PERF_DEV_NAME "meson_ddr_bw"
35 #define MAX_AXI_PORTS_OF_CHANNEL 4 /* A DMC channel can monitor max 4 axi ports */
36
37 #define to_ddr_pmu(p) container_of(p, struct ddr_pmu, pmu)
38 #define dmc_info_to_pmu(p) container_of(p, struct ddr_pmu, info)
39
dmc_pmu_enable(struct ddr_pmu * pmu)40 static void dmc_pmu_enable(struct ddr_pmu *pmu)
41 {
42 if (!pmu->pmu_enabled)
43 pmu->info.hw_info->enable(&pmu->info);
44
45 pmu->pmu_enabled = true;
46 }
47
dmc_pmu_disable(struct ddr_pmu * pmu)48 static void dmc_pmu_disable(struct ddr_pmu *pmu)
49 {
50 if (pmu->pmu_enabled)
51 pmu->info.hw_info->disable(&pmu->info);
52
53 pmu->pmu_enabled = false;
54 }
55
meson_ddr_set_axi_filter(struct perf_event * event,u8 axi_id)56 static void meson_ddr_set_axi_filter(struct perf_event *event, u8 axi_id)
57 {
58 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
59 int chann;
60
61 if (event->attr.config > ALL_CHAN_COUNTER_ID &&
62 event->attr.config < COUNTER_MAX_ID) {
63 chann = event->attr.config - CHAN1_COUNTER_ID;
64
65 pmu->info.hw_info->set_axi_filter(&pmu->info, axi_id, chann);
66 }
67 }
68
ddr_cnt_addition(struct dmc_counter * sum,struct dmc_counter * add1,struct dmc_counter * add2,int chann_nr)69 static void ddr_cnt_addition(struct dmc_counter *sum,
70 struct dmc_counter *add1,
71 struct dmc_counter *add2,
72 int chann_nr)
73 {
74 int i;
75 u64 cnt1, cnt2;
76
77 sum->all_cnt = add1->all_cnt + add2->all_cnt;
78 sum->all_req = add1->all_req + add2->all_req;
79 for (i = 0; i < chann_nr; i++) {
80 cnt1 = add1->channel_cnt[i];
81 cnt2 = add2->channel_cnt[i];
82
83 sum->channel_cnt[i] = cnt1 + cnt2;
84 }
85 }
86
meson_ddr_perf_event_update(struct perf_event * event)87 static void meson_ddr_perf_event_update(struct perf_event *event)
88 {
89 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
90 u64 new_raw_count = 0;
91 struct dmc_counter dc = {0}, sum_dc = {0};
92 int idx;
93 int chann_nr = pmu->info.hw_info->chann_nr;
94
95 /* get the remain counters in register. */
96 pmu->info.hw_info->get_counters(&pmu->info, &dc);
97
98 ddr_cnt_addition(&sum_dc, &pmu->counters, &dc, chann_nr);
99
100 switch (event->attr.config) {
101 case ALL_CHAN_COUNTER_ID:
102 new_raw_count = sum_dc.all_cnt;
103 break;
104 case CHAN1_COUNTER_ID:
105 case CHAN2_COUNTER_ID:
106 case CHAN3_COUNTER_ID:
107 case CHAN4_COUNTER_ID:
108 case CHAN5_COUNTER_ID:
109 case CHAN6_COUNTER_ID:
110 case CHAN7_COUNTER_ID:
111 case CHAN8_COUNTER_ID:
112 idx = event->attr.config - CHAN1_COUNTER_ID;
113 new_raw_count = sum_dc.channel_cnt[idx];
114 break;
115 }
116
117 local64_set(&event->count, new_raw_count);
118 }
119
meson_ddr_perf_event_init(struct perf_event * event)120 static int meson_ddr_perf_event_init(struct perf_event *event)
121 {
122 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
123 u64 config1 = event->attr.config1;
124 u64 config2 = event->attr.config2;
125
126 if (event->attr.type != event->pmu->type)
127 return -ENOENT;
128
129 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
130 return -EOPNOTSUPP;
131
132 if (event->cpu < 0)
133 return -EOPNOTSUPP;
134
135 /* check if the number of parameters is too much */
136 if (event->attr.config != ALL_CHAN_COUNTER_ID &&
137 hweight64(config1) + hweight64(config2) > MAX_AXI_PORTS_OF_CHANNEL)
138 return -EOPNOTSUPP;
139
140 event->cpu = pmu->cpu;
141
142 return 0;
143 }
144
meson_ddr_perf_event_start(struct perf_event * event,int flags)145 static void meson_ddr_perf_event_start(struct perf_event *event, int flags)
146 {
147 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
148
149 memset(&pmu->counters, 0, sizeof(pmu->counters));
150 dmc_pmu_enable(pmu);
151 }
152
meson_ddr_perf_event_add(struct perf_event * event,int flags)153 static int meson_ddr_perf_event_add(struct perf_event *event, int flags)
154 {
155 u64 config1 = event->attr.config1;
156 u64 config2 = event->attr.config2;
157 int i;
158
159 for_each_set_bit(i, (const unsigned long *)&config1, sizeof(config1))
160 meson_ddr_set_axi_filter(event, i);
161
162 for_each_set_bit(i, (const unsigned long *)&config2, sizeof(config2))
163 meson_ddr_set_axi_filter(event, i + 64);
164
165 if (flags & PERF_EF_START)
166 meson_ddr_perf_event_start(event, flags);
167
168 return 0;
169 }
170
meson_ddr_perf_event_stop(struct perf_event * event,int flags)171 static void meson_ddr_perf_event_stop(struct perf_event *event, int flags)
172 {
173 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
174
175 if (flags & PERF_EF_UPDATE)
176 meson_ddr_perf_event_update(event);
177
178 dmc_pmu_disable(pmu);
179 }
180
meson_ddr_perf_event_del(struct perf_event * event,int flags)181 static void meson_ddr_perf_event_del(struct perf_event *event, int flags)
182 {
183 meson_ddr_perf_event_stop(event, PERF_EF_UPDATE);
184 }
185
meson_ddr_perf_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)186 static ssize_t meson_ddr_perf_cpumask_show(struct device *dev,
187 struct device_attribute *attr,
188 char *buf)
189 {
190 struct ddr_pmu *pmu = dev_get_drvdata(dev);
191
192 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
193 }
194
195 static struct device_attribute meson_ddr_perf_cpumask_attr =
196 __ATTR(cpumask, 0444, meson_ddr_perf_cpumask_show, NULL);
197
198 static struct attribute *meson_ddr_perf_cpumask_attrs[] = {
199 &meson_ddr_perf_cpumask_attr.attr,
200 NULL,
201 };
202
203 static const struct attribute_group ddr_perf_cpumask_attr_group = {
204 .attrs = meson_ddr_perf_cpumask_attrs,
205 };
206
207 static ssize_t
pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)208 pmu_event_show(struct device *dev, struct device_attribute *attr,
209 char *page)
210 {
211 struct perf_pmu_events_attr *pmu_attr;
212
213 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
214 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
215 }
216
217 static ssize_t
event_show_unit(struct device * dev,struct device_attribute * attr,char * page)218 event_show_unit(struct device *dev, struct device_attribute *attr,
219 char *page)
220 {
221 return sysfs_emit(page, "MB\n");
222 }
223
224 static ssize_t
event_show_scale(struct device * dev,struct device_attribute * attr,char * page)225 event_show_scale(struct device *dev, struct device_attribute *attr,
226 char *page)
227 {
228 /* one count = 16byte = 1.52587890625e-05 MB */
229 return sysfs_emit(page, "1.52587890625e-05\n");
230 }
231
232 #define AML_DDR_PMU_EVENT_ATTR(_name, _id) \
233 { \
234 .attr = __ATTR(_name, 0444, pmu_event_show, NULL), \
235 .id = _id, \
236 }
237
238 #define AML_DDR_PMU_EVENT_UNIT_ATTR(_name) \
239 __ATTR(_name.unit, 0444, event_show_unit, NULL)
240
241 #define AML_DDR_PMU_EVENT_SCALE_ATTR(_name) \
242 __ATTR(_name.scale, 0444, event_show_scale, NULL)
243
244 static struct device_attribute event_unit_attrs[] = {
245 AML_DDR_PMU_EVENT_UNIT_ATTR(total_rw_bytes),
246 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_1_rw_bytes),
247 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_2_rw_bytes),
248 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_3_rw_bytes),
249 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_4_rw_bytes),
250 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_5_rw_bytes),
251 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_6_rw_bytes),
252 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_7_rw_bytes),
253 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_8_rw_bytes),
254 };
255
256 static struct device_attribute event_scale_attrs[] = {
257 AML_DDR_PMU_EVENT_SCALE_ATTR(total_rw_bytes),
258 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_1_rw_bytes),
259 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_2_rw_bytes),
260 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_3_rw_bytes),
261 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_4_rw_bytes),
262 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_5_rw_bytes),
263 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_6_rw_bytes),
264 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_7_rw_bytes),
265 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_8_rw_bytes),
266 };
267
268 static struct perf_pmu_events_attr event_attrs[] = {
269 AML_DDR_PMU_EVENT_ATTR(total_rw_bytes, ALL_CHAN_COUNTER_ID),
270 AML_DDR_PMU_EVENT_ATTR(chan_1_rw_bytes, CHAN1_COUNTER_ID),
271 AML_DDR_PMU_EVENT_ATTR(chan_2_rw_bytes, CHAN2_COUNTER_ID),
272 AML_DDR_PMU_EVENT_ATTR(chan_3_rw_bytes, CHAN3_COUNTER_ID),
273 AML_DDR_PMU_EVENT_ATTR(chan_4_rw_bytes, CHAN4_COUNTER_ID),
274 AML_DDR_PMU_EVENT_ATTR(chan_5_rw_bytes, CHAN5_COUNTER_ID),
275 AML_DDR_PMU_EVENT_ATTR(chan_6_rw_bytes, CHAN6_COUNTER_ID),
276 AML_DDR_PMU_EVENT_ATTR(chan_7_rw_bytes, CHAN7_COUNTER_ID),
277 AML_DDR_PMU_EVENT_ATTR(chan_8_rw_bytes, CHAN8_COUNTER_ID),
278 };
279
280 /* three attrs are combined an event */
281 static struct attribute *ddr_perf_events_attrs[COUNTER_MAX_ID * 3];
282
283 static struct attribute_group ddr_perf_events_attr_group = {
284 .name = "events",
285 .attrs = ddr_perf_events_attrs,
286 };
287
meson_ddr_perf_format_attr_visible(struct kobject * kobj,struct attribute * attr,int n)288 static umode_t meson_ddr_perf_format_attr_visible(struct kobject *kobj,
289 struct attribute *attr,
290 int n)
291 {
292 struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj));
293 struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
294 const u64 *capability = ddr_pmu->info.hw_info->capability;
295 struct device_attribute *dev_attr;
296 int id;
297 char value[20]; // config1:xxx, 20 is enough
298
299 dev_attr = container_of(attr, struct device_attribute, attr);
300 dev_attr->show(NULL, NULL, value);
301
302 if (sscanf(value, "config1:%d", &id) == 1)
303 return capability[0] & (1ULL << id) ? attr->mode : 0;
304
305 if (sscanf(value, "config2:%d", &id) == 1)
306 return capability[1] & (1ULL << id) ? attr->mode : 0;
307
308 return attr->mode;
309 }
310
311 static struct attribute_group ddr_perf_format_attr_group = {
312 .name = "format",
313 .is_visible = meson_ddr_perf_format_attr_visible,
314 };
315
meson_ddr_perf_identifier_show(struct device * dev,struct device_attribute * attr,char * page)316 static ssize_t meson_ddr_perf_identifier_show(struct device *dev,
317 struct device_attribute *attr,
318 char *page)
319 {
320 struct ddr_pmu *pmu = dev_get_drvdata(dev);
321
322 return sysfs_emit(page, "%s\n", pmu->name);
323 }
324
325 static struct device_attribute meson_ddr_perf_identifier_attr =
326 __ATTR(identifier, 0444, meson_ddr_perf_identifier_show, NULL);
327
328 static struct attribute *meson_ddr_perf_identifier_attrs[] = {
329 &meson_ddr_perf_identifier_attr.attr,
330 NULL,
331 };
332
333 static const struct attribute_group ddr_perf_identifier_attr_group = {
334 .attrs = meson_ddr_perf_identifier_attrs,
335 };
336
337 static const struct attribute_group *attr_groups[] = {
338 &ddr_perf_events_attr_group,
339 &ddr_perf_format_attr_group,
340 &ddr_perf_cpumask_attr_group,
341 &ddr_perf_identifier_attr_group,
342 NULL,
343 };
344
dmc_irq_handler(int irq,void * dev_id)345 static irqreturn_t dmc_irq_handler(int irq, void *dev_id)
346 {
347 struct dmc_info *info = dev_id;
348 struct ddr_pmu *pmu;
349 struct dmc_counter counters, *sum_cnter;
350 int i;
351
352 pmu = dmc_info_to_pmu(info);
353
354 if (info->hw_info->irq_handler(info, &counters) != 0)
355 goto out;
356
357 sum_cnter = &pmu->counters;
358 sum_cnter->all_cnt += counters.all_cnt;
359 sum_cnter->all_req += counters.all_req;
360
361 for (i = 0; i < pmu->info.hw_info->chann_nr; i++)
362 sum_cnter->channel_cnt[i] += counters.channel_cnt[i];
363
364 if (pmu->pmu_enabled)
365 /*
366 * the timer interrupt only supprt
367 * one shot mode, we have to re-enable
368 * it in ISR to support continue mode.
369 */
370 info->hw_info->enable(info);
371
372 dev_dbg(pmu->dev, "counts: %llu %llu %llu, %llu, %llu, %llu\t\t"
373 "sum: %llu %llu %llu, %llu, %llu, %llu\n",
374 counters.all_req,
375 counters.all_cnt,
376 counters.channel_cnt[0],
377 counters.channel_cnt[1],
378 counters.channel_cnt[2],
379 counters.channel_cnt[3],
380
381 pmu->counters.all_req,
382 pmu->counters.all_cnt,
383 pmu->counters.channel_cnt[0],
384 pmu->counters.channel_cnt[1],
385 pmu->counters.channel_cnt[2],
386 pmu->counters.channel_cnt[3]);
387 out:
388 return IRQ_HANDLED;
389 }
390
ddr_perf_offline_cpu(unsigned int cpu,struct hlist_node * node)391 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
392 {
393 struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
394 int target;
395
396 if (cpu != pmu->cpu)
397 return 0;
398
399 target = cpumask_any_but(cpu_online_mask, cpu);
400 if (target >= nr_cpu_ids)
401 return 0;
402
403 perf_pmu_migrate_context(&pmu->pmu, cpu, target);
404 pmu->cpu = target;
405
406 WARN_ON(irq_set_affinity(pmu->info.irq_num, cpumask_of(pmu->cpu)));
407
408 return 0;
409 }
410
fill_event_attr(struct ddr_pmu * pmu)411 static void fill_event_attr(struct ddr_pmu *pmu)
412 {
413 int i, j, k;
414 struct attribute **dst = ddr_perf_events_attrs;
415
416 j = 0;
417 k = 0;
418
419 /* fill ALL_CHAN_COUNTER_ID event */
420 dst[j++] = &event_attrs[k].attr.attr;
421 dst[j++] = &event_unit_attrs[k].attr;
422 dst[j++] = &event_scale_attrs[k].attr;
423
424 k++;
425
426 /* fill each channel event */
427 for (i = 0; i < pmu->info.hw_info->chann_nr; i++, k++) {
428 dst[j++] = &event_attrs[k].attr.attr;
429 dst[j++] = &event_unit_attrs[k].attr;
430 dst[j++] = &event_scale_attrs[k].attr;
431 }
432
433 dst[j] = NULL; /* mark end */
434 }
435
fmt_attr_fill(struct attribute ** fmt_attr)436 static void fmt_attr_fill(struct attribute **fmt_attr)
437 {
438 ddr_perf_format_attr_group.attrs = fmt_attr;
439 }
440
ddr_pmu_parse_dt(struct platform_device * pdev,struct dmc_info * info)441 static int ddr_pmu_parse_dt(struct platform_device *pdev,
442 struct dmc_info *info)
443 {
444 void __iomem *base;
445 int i, ret;
446
447 info->hw_info = of_device_get_match_data(&pdev->dev);
448
449 for (i = 0; i < info->hw_info->dmc_nr; i++) {
450 /* resource 0 for ddr register base */
451 base = devm_platform_ioremap_resource(pdev, i);
452 if (IS_ERR(base))
453 return PTR_ERR(base);
454
455 info->ddr_reg[i] = base;
456 }
457
458 /* resource i for pll register base */
459 base = devm_platform_ioremap_resource(pdev, i);
460 if (IS_ERR(base))
461 return PTR_ERR(base);
462
463 info->pll_reg = base;
464
465 ret = platform_get_irq(pdev, 0);
466 if (ret < 0)
467 return ret;
468
469 info->irq_num = ret;
470
471 ret = devm_request_irq(&pdev->dev, info->irq_num, dmc_irq_handler,
472 IRQF_NOBALANCING, dev_name(&pdev->dev),
473 (void *)info);
474 if (ret < 0)
475 return ret;
476
477 return 0;
478 }
479
meson_ddr_pmu_create(struct platform_device * pdev)480 int meson_ddr_pmu_create(struct platform_device *pdev)
481 {
482 int ret;
483 char *name;
484 struct ddr_pmu *pmu;
485
486 pmu = devm_kzalloc(&pdev->dev, sizeof(struct ddr_pmu), GFP_KERNEL);
487 if (!pmu)
488 return -ENOMEM;
489
490 *pmu = (struct ddr_pmu) {
491 .pmu = {
492 .module = THIS_MODULE,
493 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
494 .task_ctx_nr = perf_invalid_context,
495 .attr_groups = attr_groups,
496 .event_init = meson_ddr_perf_event_init,
497 .add = meson_ddr_perf_event_add,
498 .del = meson_ddr_perf_event_del,
499 .start = meson_ddr_perf_event_start,
500 .stop = meson_ddr_perf_event_stop,
501 .read = meson_ddr_perf_event_update,
502 },
503 };
504
505 ret = ddr_pmu_parse_dt(pdev, &pmu->info);
506 if (ret < 0)
507 return ret;
508
509 fmt_attr_fill(pmu->info.hw_info->fmt_attr);
510
511 pmu->cpu = smp_processor_id();
512
513 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME);
514 if (!name)
515 return -ENOMEM;
516
517 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, name, NULL,
518 ddr_perf_offline_cpu);
519 if (ret < 0)
520 return ret;
521
522 pmu->cpuhp_state = ret;
523
524 /* Register the pmu instance for cpu hotplug */
525 ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
526 if (ret)
527 goto cpuhp_instance_err;
528
529 fill_event_attr(pmu);
530
531 ret = perf_pmu_register(&pmu->pmu, name, -1);
532 if (ret)
533 goto pmu_register_err;
534
535 pmu->name = name;
536 pmu->dev = &pdev->dev;
537 pmu->pmu_enabled = false;
538
539 platform_set_drvdata(pdev, pmu);
540
541 return 0;
542
543 pmu_register_err:
544 cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
545
546 cpuhp_instance_err:
547 cpuhp_remove_state(pmu->cpuhp_state);
548
549 return ret;
550 }
551
meson_ddr_pmu_remove(struct platform_device * pdev)552 int meson_ddr_pmu_remove(struct platform_device *pdev)
553 {
554 struct ddr_pmu *pmu = platform_get_drvdata(pdev);
555
556 perf_pmu_unregister(&pmu->pmu);
557 cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
558 cpuhp_remove_state(pmu->cpuhp_state);
559
560 return 0;
561 }
562