1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3 /* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
4 /* Copyright 2019 Collabora ltd. */
5
6 #ifdef CONFIG_ARM_ARCH_TIMER
7 #include <asm/arch_timer.h>
8 #endif
9
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/pagemap.h>
13 #include <linux/platform_device.h>
14 #include <linux/pm_runtime.h>
15 #include <drm/panfrost_drm.h>
16 #include <drm/drm_debugfs.h>
17 #include <drm/drm_drv.h>
18 #include <drm/drm_ioctl.h>
19 #include <drm/drm_syncobj.h>
20 #include <drm/drm_utils.h>
21
22 #include "panfrost_device.h"
23 #include "panfrost_gem.h"
24 #include "panfrost_mmu.h"
25 #include "panfrost_job.h"
26 #include "panfrost_gpu.h"
27 #include "panfrost_perfcnt.h"
28
29 #define JOB_REQUIREMENTS (PANFROST_JD_REQ_FS | PANFROST_JD_REQ_CYCLE_COUNT)
30
31 static bool unstable_ioctls;
32 module_param_unsafe(unstable_ioctls, bool, 0600);
33
panfrost_ioctl_query_timestamp(struct panfrost_device * pfdev,u64 * arg)34 static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev,
35 u64 *arg)
36 {
37 int ret;
38
39 ret = pm_runtime_resume_and_get(pfdev->dev);
40 if (ret)
41 return ret;
42
43 panfrost_cycle_counter_get(pfdev);
44 *arg = panfrost_timestamp_read(pfdev);
45 panfrost_cycle_counter_put(pfdev);
46
47 pm_runtime_put(pfdev->dev);
48 return 0;
49 }
50
panfrost_ioctl_get_param(struct drm_device * ddev,void * data,struct drm_file * file)51 static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file)
52 {
53 struct drm_panfrost_get_param *param = data;
54 struct panfrost_device *pfdev = ddev->dev_private;
55 int ret;
56
57 if (param->pad != 0)
58 return -EINVAL;
59
60 #define PANFROST_FEATURE(name, member) \
61 case DRM_PANFROST_PARAM_ ## name: \
62 param->value = pfdev->features.member; \
63 break
64 #define PANFROST_FEATURE_ARRAY(name, member, max) \
65 case DRM_PANFROST_PARAM_ ## name ## 0 ... \
66 DRM_PANFROST_PARAM_ ## name ## max: \
67 param->value = pfdev->features.member[param->param - \
68 DRM_PANFROST_PARAM_ ## name ## 0]; \
69 break
70
71 switch (param->param) {
72 PANFROST_FEATURE(GPU_PROD_ID, id);
73 PANFROST_FEATURE(GPU_REVISION, revision);
74 PANFROST_FEATURE(SHADER_PRESENT, shader_present);
75 PANFROST_FEATURE(TILER_PRESENT, tiler_present);
76 PANFROST_FEATURE(L2_PRESENT, l2_present);
77 PANFROST_FEATURE(STACK_PRESENT, stack_present);
78 PANFROST_FEATURE(AS_PRESENT, as_present);
79 PANFROST_FEATURE(JS_PRESENT, js_present);
80 PANFROST_FEATURE(L2_FEATURES, l2_features);
81 PANFROST_FEATURE(CORE_FEATURES, core_features);
82 PANFROST_FEATURE(TILER_FEATURES, tiler_features);
83 PANFROST_FEATURE(MEM_FEATURES, mem_features);
84 PANFROST_FEATURE(MMU_FEATURES, mmu_features);
85 PANFROST_FEATURE(THREAD_FEATURES, thread_features);
86 PANFROST_FEATURE(MAX_THREADS, max_threads);
87 PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ,
88 thread_max_workgroup_sz);
89 PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ,
90 thread_max_barrier_sz);
91 PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features);
92 PANFROST_FEATURE(AFBC_FEATURES, afbc_features);
93 PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3);
94 PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15);
95 PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups);
96 PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc);
97
98 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP:
99 ret = panfrost_ioctl_query_timestamp(pfdev, ¶m->value);
100 if (ret)
101 return ret;
102 break;
103
104 case DRM_PANFROST_PARAM_SYSTEM_TIMESTAMP_FREQUENCY:
105 #ifdef CONFIG_ARM_ARCH_TIMER
106 param->value = arch_timer_get_cntfrq();
107 #else
108 param->value = 0;
109 #endif
110 break;
111
112 default:
113 return -EINVAL;
114 }
115
116 return 0;
117 }
118
panfrost_ioctl_create_bo(struct drm_device * dev,void * data,struct drm_file * file)119 static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data,
120 struct drm_file *file)
121 {
122 struct panfrost_file_priv *priv = file->driver_priv;
123 struct panfrost_gem_object *bo;
124 struct drm_panfrost_create_bo *args = data;
125 struct panfrost_gem_mapping *mapping;
126 int ret;
127
128 if (!args->size || args->pad ||
129 (args->flags & ~(PANFROST_BO_NOEXEC | PANFROST_BO_HEAP)))
130 return -EINVAL;
131
132 /* Heaps should never be executable */
133 if ((args->flags & PANFROST_BO_HEAP) &&
134 !(args->flags & PANFROST_BO_NOEXEC))
135 return -EINVAL;
136
137 bo = panfrost_gem_create(dev, args->size, args->flags);
138 if (IS_ERR(bo))
139 return PTR_ERR(bo);
140
141 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
142 if (ret)
143 goto out;
144
145 mapping = panfrost_gem_mapping_get(bo, priv);
146 if (mapping) {
147 args->offset = mapping->mmnode.start << PAGE_SHIFT;
148 panfrost_gem_mapping_put(mapping);
149 } else {
150 /* This can only happen if the handle from
151 * drm_gem_handle_create() has already been guessed and freed
152 * by user space
153 */
154 ret = -EINVAL;
155 }
156
157 out:
158 drm_gem_object_put(&bo->base.base);
159 return ret;
160 }
161
162 /**
163 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects
164 * referenced by the job.
165 * @dev: DRM device
166 * @file_priv: DRM file for this fd
167 * @args: IOCTL args
168 * @job: job being set up
169 *
170 * Resolve handles from userspace to BOs and attach them to job.
171 *
172 * Note that this function doesn't need to unreference the BOs on
173 * failure, because that will happen at panfrost_job_cleanup() time.
174 */
175 static int
panfrost_lookup_bos(struct drm_device * dev,struct drm_file * file_priv,struct drm_panfrost_submit * args,struct panfrost_job * job)176 panfrost_lookup_bos(struct drm_device *dev,
177 struct drm_file *file_priv,
178 struct drm_panfrost_submit *args,
179 struct panfrost_job *job)
180 {
181 struct panfrost_file_priv *priv = file_priv->driver_priv;
182 struct panfrost_gem_object *bo;
183 unsigned int i;
184 int ret;
185
186 job->bo_count = args->bo_handle_count;
187
188 if (!job->bo_count)
189 return 0;
190
191 ret = drm_gem_objects_lookup(file_priv,
192 (void __user *)(uintptr_t)args->bo_handles,
193 job->bo_count, &job->bos);
194 if (ret)
195 return ret;
196
197 job->mappings = kvmalloc_array(job->bo_count,
198 sizeof(struct panfrost_gem_mapping *),
199 GFP_KERNEL | __GFP_ZERO);
200 if (!job->mappings)
201 return -ENOMEM;
202
203 for (i = 0; i < job->bo_count; i++) {
204 struct panfrost_gem_mapping *mapping;
205
206 bo = to_panfrost_bo(job->bos[i]);
207 mapping = panfrost_gem_mapping_get(bo, priv);
208 if (!mapping) {
209 ret = -EINVAL;
210 break;
211 }
212
213 atomic_inc(&bo->gpu_usecount);
214 job->mappings[i] = mapping;
215 }
216
217 return ret;
218 }
219
220 /**
221 * panfrost_copy_in_sync() - Sets up job->deps with the sync objects
222 * referenced by the job.
223 * @dev: DRM device
224 * @file_priv: DRM file for this fd
225 * @args: IOCTL args
226 * @job: job being set up
227 *
228 * Resolve syncobjs from userspace to fences and attach them to job.
229 *
230 * Note that this function doesn't need to unreference the fences on
231 * failure, because that will happen at panfrost_job_cleanup() time.
232 */
233 static int
panfrost_copy_in_sync(struct drm_device * dev,struct drm_file * file_priv,struct drm_panfrost_submit * args,struct panfrost_job * job)234 panfrost_copy_in_sync(struct drm_device *dev,
235 struct drm_file *file_priv,
236 struct drm_panfrost_submit *args,
237 struct panfrost_job *job)
238 {
239 u32 *handles;
240 int ret = 0;
241 int i, in_fence_count;
242
243 in_fence_count = args->in_sync_count;
244
245 if (!in_fence_count)
246 return 0;
247
248 handles = kvmalloc_array(in_fence_count, sizeof(u32), GFP_KERNEL);
249 if (!handles) {
250 ret = -ENOMEM;
251 DRM_DEBUG("Failed to allocate incoming syncobj handles\n");
252 goto fail;
253 }
254
255 if (copy_from_user(handles,
256 (void __user *)(uintptr_t)args->in_syncs,
257 in_fence_count * sizeof(u32))) {
258 ret = -EFAULT;
259 DRM_DEBUG("Failed to copy in syncobj handles\n");
260 goto fail;
261 }
262
263 for (i = 0; i < in_fence_count; i++) {
264 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv,
265 handles[i], 0);
266 if (ret)
267 goto fail;
268 }
269
270 fail:
271 kvfree(handles);
272 return ret;
273 }
274
panfrost_ioctl_submit(struct drm_device * dev,void * data,struct drm_file * file)275 static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
276 struct drm_file *file)
277 {
278 struct panfrost_device *pfdev = dev->dev_private;
279 struct panfrost_file_priv *file_priv = file->driver_priv;
280 struct drm_panfrost_submit *args = data;
281 struct drm_syncobj *sync_out = NULL;
282 struct panfrost_job *job;
283 int ret = 0, slot;
284
285 if (!args->jc)
286 return -EINVAL;
287
288 if (args->requirements & ~JOB_REQUIREMENTS)
289 return -EINVAL;
290
291 if (args->out_sync > 0) {
292 sync_out = drm_syncobj_find(file, args->out_sync);
293 if (!sync_out)
294 return -ENODEV;
295 }
296
297 job = kzalloc(sizeof(*job), GFP_KERNEL);
298 if (!job) {
299 ret = -ENOMEM;
300 goto out_put_syncout;
301 }
302
303 kref_init(&job->refcount);
304
305 job->pfdev = pfdev;
306 job->jc = args->jc;
307 job->requirements = args->requirements;
308 job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev);
309 job->mmu = file_priv->mmu;
310 job->engine_usage = &file_priv->engine_usage;
311
312 slot = panfrost_job_get_slot(job);
313
314 ret = drm_sched_job_init(&job->base,
315 &file_priv->sched_entity[slot],
316 1, NULL, file->client_id);
317 if (ret)
318 goto out_put_job;
319
320 ret = panfrost_copy_in_sync(dev, file, args, job);
321 if (ret)
322 goto out_cleanup_job;
323
324 ret = panfrost_lookup_bos(dev, file, args, job);
325 if (ret)
326 goto out_cleanup_job;
327
328 ret = panfrost_job_push(job);
329 if (ret)
330 goto out_cleanup_job;
331
332 /* Update the return sync object for the job */
333 if (sync_out)
334 drm_syncobj_replace_fence(sync_out, job->render_done_fence);
335
336 out_cleanup_job:
337 if (ret)
338 drm_sched_job_cleanup(&job->base);
339 out_put_job:
340 panfrost_job_put(job);
341 out_put_syncout:
342 if (sync_out)
343 drm_syncobj_put(sync_out);
344
345 return ret;
346 }
347
348 static int
panfrost_ioctl_wait_bo(struct drm_device * dev,void * data,struct drm_file * file_priv)349 panfrost_ioctl_wait_bo(struct drm_device *dev, void *data,
350 struct drm_file *file_priv)
351 {
352 long ret;
353 struct drm_panfrost_wait_bo *args = data;
354 struct drm_gem_object *gem_obj;
355 unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
356
357 if (args->pad)
358 return -EINVAL;
359
360 gem_obj = drm_gem_object_lookup(file_priv, args->handle);
361 if (!gem_obj)
362 return -ENOENT;
363
364 ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_READ,
365 true, timeout);
366 if (!ret)
367 ret = timeout ? -ETIMEDOUT : -EBUSY;
368
369 drm_gem_object_put(gem_obj);
370
371 return ret;
372 }
373
panfrost_ioctl_mmap_bo(struct drm_device * dev,void * data,struct drm_file * file_priv)374 static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data,
375 struct drm_file *file_priv)
376 {
377 struct drm_panfrost_mmap_bo *args = data;
378 struct drm_gem_object *gem_obj;
379 int ret;
380
381 if (args->flags != 0) {
382 DRM_INFO("unknown mmap_bo flags: %d\n", args->flags);
383 return -EINVAL;
384 }
385
386 gem_obj = drm_gem_object_lookup(file_priv, args->handle);
387 if (!gem_obj) {
388 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
389 return -ENOENT;
390 }
391
392 /* Don't allow mmapping of heap objects as pages are not pinned. */
393 if (to_panfrost_bo(gem_obj)->is_heap) {
394 ret = -EINVAL;
395 goto out;
396 }
397
398 ret = drm_gem_create_mmap_offset(gem_obj);
399 if (ret == 0)
400 args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
401
402 out:
403 drm_gem_object_put(gem_obj);
404 return ret;
405 }
406
panfrost_ioctl_get_bo_offset(struct drm_device * dev,void * data,struct drm_file * file_priv)407 static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data,
408 struct drm_file *file_priv)
409 {
410 struct panfrost_file_priv *priv = file_priv->driver_priv;
411 struct drm_panfrost_get_bo_offset *args = data;
412 struct panfrost_gem_mapping *mapping;
413 struct drm_gem_object *gem_obj;
414 struct panfrost_gem_object *bo;
415
416 gem_obj = drm_gem_object_lookup(file_priv, args->handle);
417 if (!gem_obj) {
418 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
419 return -ENOENT;
420 }
421 bo = to_panfrost_bo(gem_obj);
422
423 mapping = panfrost_gem_mapping_get(bo, priv);
424 drm_gem_object_put(gem_obj);
425
426 if (!mapping)
427 return -EINVAL;
428
429 args->offset = mapping->mmnode.start << PAGE_SHIFT;
430 panfrost_gem_mapping_put(mapping);
431 return 0;
432 }
433
panfrost_ioctl_madvise(struct drm_device * dev,void * data,struct drm_file * file_priv)434 static int panfrost_ioctl_madvise(struct drm_device *dev, void *data,
435 struct drm_file *file_priv)
436 {
437 struct panfrost_file_priv *priv = file_priv->driver_priv;
438 struct drm_panfrost_madvise *args = data;
439 struct panfrost_device *pfdev = dev->dev_private;
440 struct drm_gem_object *gem_obj;
441 struct panfrost_gem_object *bo;
442 int ret = 0;
443
444 gem_obj = drm_gem_object_lookup(file_priv, args->handle);
445 if (!gem_obj) {
446 DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
447 return -ENOENT;
448 }
449
450 bo = to_panfrost_bo(gem_obj);
451
452 ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL);
453 if (ret)
454 goto out_put_object;
455
456 mutex_lock(&pfdev->shrinker_lock);
457 mutex_lock(&bo->mappings.lock);
458 if (args->madv == PANFROST_MADV_DONTNEED) {
459 struct panfrost_gem_mapping *first;
460
461 first = list_first_entry(&bo->mappings.list,
462 struct panfrost_gem_mapping,
463 node);
464
465 /*
466 * If we want to mark the BO purgeable, there must be only one
467 * user: the caller FD.
468 * We could do something smarter and mark the BO purgeable only
469 * when all its users have marked it purgeable, but globally
470 * visible/shared BOs are likely to never be marked purgeable
471 * anyway, so let's not bother.
472 */
473 if (!list_is_singular(&bo->mappings.list) ||
474 WARN_ON_ONCE(first->mmu != priv->mmu)) {
475 ret = -EINVAL;
476 goto out_unlock_mappings;
477 }
478 }
479
480 args->retained = drm_gem_shmem_madvise_locked(&bo->base, args->madv);
481
482 if (args->retained) {
483 if (args->madv == PANFROST_MADV_DONTNEED)
484 list_move_tail(&bo->base.madv_list,
485 &pfdev->shrinker_list);
486 else if (args->madv == PANFROST_MADV_WILLNEED)
487 list_del_init(&bo->base.madv_list);
488 }
489
490 out_unlock_mappings:
491 mutex_unlock(&bo->mappings.lock);
492 mutex_unlock(&pfdev->shrinker_lock);
493 dma_resv_unlock(bo->base.base.resv);
494 out_put_object:
495 drm_gem_object_put(gem_obj);
496 return ret;
497 }
498
panfrost_ioctl_set_label_bo(struct drm_device * ddev,void * data,struct drm_file * file)499 static int panfrost_ioctl_set_label_bo(struct drm_device *ddev, void *data,
500 struct drm_file *file)
501 {
502 struct drm_panfrost_set_label_bo *args = data;
503 struct drm_gem_object *obj;
504 const char *label = NULL;
505 int ret = 0;
506
507 if (args->pad)
508 return -EINVAL;
509
510 obj = drm_gem_object_lookup(file, args->handle);
511 if (!obj)
512 return -ENOENT;
513
514 if (args->label) {
515 label = strndup_user(u64_to_user_ptr(args->label),
516 PANFROST_BO_LABEL_MAXLEN);
517 if (IS_ERR(label)) {
518 ret = PTR_ERR(label);
519 if (ret == -EINVAL)
520 ret = -E2BIG;
521 goto err_put_obj;
522 }
523 }
524
525 /*
526 * We treat passing a label of length 0 and passing a NULL label
527 * differently, because even though they might seem conceptually
528 * similar, future uses of the BO label might expect a different
529 * behaviour in each case.
530 */
531 panfrost_gem_set_label(obj, label);
532
533 err_put_obj:
534 drm_gem_object_put(obj);
535
536 return ret;
537 }
538
panfrost_unstable_ioctl_check(void)539 int panfrost_unstable_ioctl_check(void)
540 {
541 if (!unstable_ioctls)
542 return -ENOSYS;
543
544 return 0;
545 }
546
547 static int
panfrost_open(struct drm_device * dev,struct drm_file * file)548 panfrost_open(struct drm_device *dev, struct drm_file *file)
549 {
550 int ret;
551 struct panfrost_device *pfdev = dev->dev_private;
552 struct panfrost_file_priv *panfrost_priv;
553
554 panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL);
555 if (!panfrost_priv)
556 return -ENOMEM;
557
558 panfrost_priv->pfdev = pfdev;
559 file->driver_priv = panfrost_priv;
560
561 panfrost_priv->mmu = panfrost_mmu_ctx_create(pfdev);
562 if (IS_ERR(panfrost_priv->mmu)) {
563 ret = PTR_ERR(panfrost_priv->mmu);
564 goto err_free;
565 }
566
567 ret = panfrost_job_open(panfrost_priv);
568 if (ret)
569 goto err_job;
570
571 return 0;
572
573 err_job:
574 panfrost_mmu_ctx_put(panfrost_priv->mmu);
575 err_free:
576 kfree(panfrost_priv);
577 return ret;
578 }
579
580 static void
panfrost_postclose(struct drm_device * dev,struct drm_file * file)581 panfrost_postclose(struct drm_device *dev, struct drm_file *file)
582 {
583 struct panfrost_file_priv *panfrost_priv = file->driver_priv;
584
585 panfrost_perfcnt_close(file);
586 panfrost_job_close(panfrost_priv);
587
588 panfrost_mmu_ctx_put(panfrost_priv->mmu);
589 kfree(panfrost_priv);
590 }
591
592 static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = {
593 #define PANFROST_IOCTL(n, func, flags) \
594 DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags)
595
596 PANFROST_IOCTL(SUBMIT, submit, DRM_RENDER_ALLOW),
597 PANFROST_IOCTL(WAIT_BO, wait_bo, DRM_RENDER_ALLOW),
598 PANFROST_IOCTL(CREATE_BO, create_bo, DRM_RENDER_ALLOW),
599 PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW),
600 PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW),
601 PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW),
602 PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW),
603 PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW),
604 PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW),
605 PANFROST_IOCTL(SET_LABEL_BO, set_label_bo, DRM_RENDER_ALLOW),
606 };
607
panfrost_gpu_show_fdinfo(struct panfrost_device * pfdev,struct panfrost_file_priv * panfrost_priv,struct drm_printer * p)608 static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev,
609 struct panfrost_file_priv *panfrost_priv,
610 struct drm_printer *p)
611 {
612 int i;
613
614 /*
615 * IMPORTANT NOTE: drm-cycles and drm-engine measurements are not
616 * accurate, as they only provide a rough estimation of the number of
617 * GPU cycles and CPU time spent in a given context. This is due to two
618 * different factors:
619 * - Firstly, we must consider the time the CPU and then the kernel
620 * takes to process the GPU interrupt, which means additional time and
621 * GPU cycles will be added in excess to the real figure.
622 * - Secondly, the pipelining done by the Job Manager (2 job slots per
623 * engine) implies there is no way to know exactly how much time each
624 * job spent on the GPU.
625 */
626
627 static const char * const engine_names[] = {
628 "fragment", "vertex-tiler", "compute-only"
629 };
630
631 BUILD_BUG_ON(ARRAY_SIZE(engine_names) != NUM_JOB_SLOTS);
632
633 for (i = 0; i < NUM_JOB_SLOTS - 1; i++) {
634 if (pfdev->profile_mode) {
635 drm_printf(p, "drm-engine-%s:\t%llu ns\n",
636 engine_names[i], panfrost_priv->engine_usage.elapsed_ns[i]);
637 drm_printf(p, "drm-cycles-%s:\t%llu\n",
638 engine_names[i], panfrost_priv->engine_usage.cycles[i]);
639 }
640 drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n",
641 engine_names[i], pfdev->pfdevfreq.fast_rate);
642 drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n",
643 engine_names[i], pfdev->pfdevfreq.current_frequency);
644 }
645 }
646
panfrost_show_fdinfo(struct drm_printer * p,struct drm_file * file)647 static void panfrost_show_fdinfo(struct drm_printer *p, struct drm_file *file)
648 {
649 struct drm_device *dev = file->minor->dev;
650 struct panfrost_device *pfdev = dev->dev_private;
651
652 panfrost_gpu_show_fdinfo(pfdev, file->driver_priv, p);
653
654 drm_show_memory_stats(p, file);
655 }
656
657 static const struct file_operations panfrost_drm_driver_fops = {
658 .owner = THIS_MODULE,
659 DRM_GEM_FOPS,
660 .show_fdinfo = drm_show_fdinfo,
661 };
662
663 #ifdef CONFIG_DEBUG_FS
panthor_gems_show(struct seq_file * m,void * data)664 static int panthor_gems_show(struct seq_file *m, void *data)
665 {
666 struct drm_info_node *node = m->private;
667 struct drm_device *dev = node->minor->dev;
668 struct panfrost_device *pfdev = dev->dev_private;
669
670 panfrost_gem_debugfs_print_bos(pfdev, m);
671
672 return 0;
673 }
674
675 static struct drm_info_list panthor_debugfs_list[] = {
676 {"gems", panthor_gems_show, 0, NULL},
677 };
678
panthor_gems_debugfs_init(struct drm_minor * minor)679 static int panthor_gems_debugfs_init(struct drm_minor *minor)
680 {
681 drm_debugfs_create_files(panthor_debugfs_list,
682 ARRAY_SIZE(panthor_debugfs_list),
683 minor->debugfs_root, minor);
684
685 return 0;
686 }
687
panfrost_debugfs_init(struct drm_minor * minor)688 static void panfrost_debugfs_init(struct drm_minor *minor)
689 {
690 panthor_gems_debugfs_init(minor);
691 }
692 #endif
693
694 /*
695 * Panfrost driver version:
696 * - 1.0 - initial interface
697 * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO
698 * - 1.2 - adds AFBC_FEATURES query
699 * - 1.3 - adds JD_REQ_CYCLE_COUNT job requirement for SUBMIT
700 * - adds SYSTEM_TIMESTAMP and SYSTEM_TIMESTAMP_FREQUENCY queries
701 * - 1.4 - adds SET_LABEL_BO
702 */
703 static const struct drm_driver panfrost_drm_driver = {
704 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ,
705 .open = panfrost_open,
706 .postclose = panfrost_postclose,
707 .show_fdinfo = panfrost_show_fdinfo,
708 .ioctls = panfrost_drm_driver_ioctls,
709 .num_ioctls = ARRAY_SIZE(panfrost_drm_driver_ioctls),
710 .fops = &panfrost_drm_driver_fops,
711 .name = "panfrost",
712 .desc = "panfrost DRM",
713 .major = 1,
714 .minor = 4,
715
716 .gem_create_object = panfrost_gem_create_object,
717 .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
718 #ifdef CONFIG_DEBUG_FS
719 .debugfs_init = panfrost_debugfs_init,
720 #endif
721 };
722
panfrost_probe(struct platform_device * pdev)723 static int panfrost_probe(struct platform_device *pdev)
724 {
725 struct panfrost_device *pfdev;
726 struct drm_device *ddev;
727 int err;
728
729 pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL);
730 if (!pfdev)
731 return -ENOMEM;
732
733 pfdev->pdev = pdev;
734 pfdev->dev = &pdev->dev;
735
736 platform_set_drvdata(pdev, pfdev);
737
738 pfdev->comp = of_device_get_match_data(&pdev->dev);
739 if (!pfdev->comp)
740 return -ENODEV;
741
742 pfdev->coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT;
743
744 /* Allocate and initialize the DRM device. */
745 ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev);
746 if (IS_ERR(ddev))
747 return PTR_ERR(ddev);
748
749 ddev->dev_private = pfdev;
750 pfdev->ddev = ddev;
751
752 mutex_init(&pfdev->shrinker_lock);
753 INIT_LIST_HEAD(&pfdev->shrinker_list);
754
755 err = panfrost_device_init(pfdev);
756 if (err) {
757 if (err != -EPROBE_DEFER)
758 dev_err(&pdev->dev, "Fatal error during GPU init\n");
759 goto err_out0;
760 }
761
762 pm_runtime_set_active(pfdev->dev);
763 pm_runtime_mark_last_busy(pfdev->dev);
764 pm_runtime_enable(pfdev->dev);
765 pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */
766 pm_runtime_use_autosuspend(pfdev->dev);
767
768 /*
769 * Register the DRM device with the core and the connectors with
770 * sysfs
771 */
772 err = drm_dev_register(ddev, 0);
773 if (err < 0)
774 goto err_out1;
775
776 err = panfrost_gem_shrinker_init(ddev);
777 if (err)
778 goto err_out2;
779
780 return 0;
781
782 err_out2:
783 drm_dev_unregister(ddev);
784 err_out1:
785 pm_runtime_disable(pfdev->dev);
786 panfrost_device_fini(pfdev);
787 pm_runtime_set_suspended(pfdev->dev);
788 err_out0:
789 drm_dev_put(ddev);
790 return err;
791 }
792
panfrost_remove(struct platform_device * pdev)793 static void panfrost_remove(struct platform_device *pdev)
794 {
795 struct panfrost_device *pfdev = platform_get_drvdata(pdev);
796 struct drm_device *ddev = pfdev->ddev;
797
798 drm_dev_unregister(ddev);
799 panfrost_gem_shrinker_cleanup(ddev);
800
801 pm_runtime_get_sync(pfdev->dev);
802 pm_runtime_disable(pfdev->dev);
803 panfrost_device_fini(pfdev);
804 pm_runtime_set_suspended(pfdev->dev);
805
806 drm_dev_put(ddev);
807 }
808
profiling_show(struct device * dev,struct device_attribute * attr,char * buf)809 static ssize_t profiling_show(struct device *dev,
810 struct device_attribute *attr, char *buf)
811 {
812 struct panfrost_device *pfdev = dev_get_drvdata(dev);
813
814 return sysfs_emit(buf, "%d\n", pfdev->profile_mode);
815 }
816
profiling_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)817 static ssize_t profiling_store(struct device *dev,
818 struct device_attribute *attr,
819 const char *buf, size_t len)
820 {
821 struct panfrost_device *pfdev = dev_get_drvdata(dev);
822 bool value;
823 int err;
824
825 err = kstrtobool(buf, &value);
826 if (err)
827 return err;
828
829 pfdev->profile_mode = value;
830
831 return len;
832 }
833
834 static DEVICE_ATTR_RW(profiling);
835
836 static struct attribute *panfrost_attrs[] = {
837 &dev_attr_profiling.attr,
838 NULL,
839 };
840
841 ATTRIBUTE_GROUPS(panfrost);
842
843 /*
844 * The OPP core wants the supply names to be NULL terminated, but we need the
845 * correct num_supplies value for regulator core. Hence, we NULL terminate here
846 * and then initialize num_supplies with ARRAY_SIZE - 1.
847 */
848 static const char * const default_supplies[] = { "mali", NULL };
849 static const struct panfrost_compatible default_data = {
850 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
851 .supply_names = default_supplies,
852 .num_pm_domains = 1, /* optional */
853 .pm_domain_names = NULL,
854 };
855
856 static const struct panfrost_compatible allwinner_h616_data = {
857 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
858 .supply_names = default_supplies,
859 .num_pm_domains = 1,
860 .pm_features = BIT(GPU_PM_RT),
861 };
862
863 static const struct panfrost_compatible amlogic_data = {
864 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
865 .supply_names = default_supplies,
866 .vendor_quirk = panfrost_gpu_amlogic_quirk,
867 };
868
869 static const char * const mediatek_pm_domains[] = { "core0", "core1", "core2",
870 "core3", "core4" };
871 /*
872 * The old data with two power supplies for MT8183 is here only to
873 * keep retro-compatibility with older devicetrees, as DVFS will
874 * not work with this one.
875 *
876 * On new devicetrees please use the _b variant with a single and
877 * coupled regulators instead.
878 */
879 static const char * const legacy_supplies[] = { "mali", "sram", NULL };
880 static const struct panfrost_compatible mediatek_mt8183_data = {
881 .num_supplies = ARRAY_SIZE(legacy_supplies) - 1,
882 .supply_names = legacy_supplies,
883 .num_pm_domains = 3,
884 .pm_domain_names = mediatek_pm_domains,
885 };
886
887 static const struct panfrost_compatible mediatek_mt8183_b_data = {
888 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
889 .supply_names = default_supplies,
890 .num_pm_domains = 3,
891 .pm_domain_names = mediatek_pm_domains,
892 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
893 };
894
895 static const struct panfrost_compatible mediatek_mt8186_data = {
896 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
897 .supply_names = default_supplies,
898 .num_pm_domains = 2,
899 .pm_domain_names = mediatek_pm_domains,
900 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
901 };
902
903 static const struct panfrost_compatible mediatek_mt8188_data = {
904 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
905 .supply_names = default_supplies,
906 .num_pm_domains = 3,
907 .pm_domain_names = mediatek_pm_domains,
908 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
909 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE),
910 };
911
912 static const struct panfrost_compatible mediatek_mt8192_data = {
913 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
914 .supply_names = default_supplies,
915 .num_pm_domains = 5,
916 .pm_domain_names = mediatek_pm_domains,
917 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
918 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE),
919 };
920
921 static const struct panfrost_compatible mediatek_mt8370_data = {
922 .num_supplies = ARRAY_SIZE(default_supplies) - 1,
923 .supply_names = default_supplies,
924 .num_pm_domains = 2,
925 .pm_domain_names = mediatek_pm_domains,
926 .pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
927 .gpu_quirks = BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE),
928 };
929
930 static const struct of_device_id dt_match[] = {
931 /* Set first to probe before the generic compatibles */
932 { .compatible = "amlogic,meson-gxm-mali",
933 .data = &amlogic_data, },
934 { .compatible = "amlogic,meson-g12a-mali",
935 .data = &amlogic_data, },
936 { .compatible = "arm,mali-t604", .data = &default_data, },
937 { .compatible = "arm,mali-t624", .data = &default_data, },
938 { .compatible = "arm,mali-t628", .data = &default_data, },
939 { .compatible = "arm,mali-t720", .data = &default_data, },
940 { .compatible = "arm,mali-t760", .data = &default_data, },
941 { .compatible = "arm,mali-t820", .data = &default_data, },
942 { .compatible = "arm,mali-t830", .data = &default_data, },
943 { .compatible = "arm,mali-t860", .data = &default_data, },
944 { .compatible = "arm,mali-t880", .data = &default_data, },
945 { .compatible = "arm,mali-bifrost", .data = &default_data, },
946 { .compatible = "arm,mali-valhall-jm", .data = &default_data, },
947 { .compatible = "mediatek,mt8183-mali", .data = &mediatek_mt8183_data },
948 { .compatible = "mediatek,mt8183b-mali", .data = &mediatek_mt8183_b_data },
949 { .compatible = "mediatek,mt8186-mali", .data = &mediatek_mt8186_data },
950 { .compatible = "mediatek,mt8188-mali", .data = &mediatek_mt8188_data },
951 { .compatible = "mediatek,mt8192-mali", .data = &mediatek_mt8192_data },
952 { .compatible = "mediatek,mt8370-mali", .data = &mediatek_mt8370_data },
953 { .compatible = "allwinner,sun50i-h616-mali", .data = &allwinner_h616_data },
954 {}
955 };
956 MODULE_DEVICE_TABLE(of, dt_match);
957
958 static struct platform_driver panfrost_driver = {
959 .probe = panfrost_probe,
960 .remove = panfrost_remove,
961 .driver = {
962 .name = "panfrost",
963 .pm = pm_ptr(&panfrost_pm_ops),
964 .of_match_table = dt_match,
965 .dev_groups = panfrost_groups,
966 },
967 };
968 module_platform_driver(panfrost_driver);
969
970 MODULE_AUTHOR("Panfrost Project Developers");
971 MODULE_DESCRIPTION("Panfrost DRM Driver");
972 MODULE_LICENSE("GPL v2");
973 MODULE_SOFTDEP("pre: governor_simpleondemand");
974