Lines Matching refs:stream
146 static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head) in xe_oa_circ_diff() argument
149 tail + stream->oa_buffer.circ_size - head; in xe_oa_circ_diff()
152 static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n) in xe_oa_circ_incr() argument
154 return ptr + n >= stream->oa_buffer.circ_size ? in xe_oa_circ_incr()
155 ptr + n - stream->oa_buffer.circ_size : ptr + n; in xe_oa_circ_incr()
201 static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream) in __oa_regs() argument
203 return &stream->oa_unit->regs; in __oa_regs()
206 static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream) in xe_oa_hw_tail_read() argument
208 return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) & in xe_oa_hw_tail_read()
215 static u64 oa_report_id(struct xe_oa_stream *stream, void *report) in oa_report_id() argument
217 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report; in oa_report_id()
220 static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report) in oa_report_id_clear() argument
222 if (oa_report_header_64bit(stream)) in oa_report_id_clear()
228 static u64 oa_timestamp(struct xe_oa_stream *stream, void *report) in oa_timestamp() argument
230 return oa_report_header_64bit(stream) ? in oa_timestamp()
235 static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report) in oa_timestamp_clear() argument
237 if (oa_report_header_64bit(stream)) in oa_timestamp_clear()
243 static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) in xe_oa_buffer_check_unlocked() argument
245 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_buffer_check_unlocked()
247 int report_size = stream->oa_buffer.format->size; in xe_oa_buffer_check_unlocked()
250 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_buffer_check_unlocked()
252 hw_tail = xe_oa_hw_tail_read(stream); in xe_oa_buffer_check_unlocked()
260 partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail); in xe_oa_buffer_check_unlocked()
264 hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size); in xe_oa_buffer_check_unlocked()
276 while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { in xe_oa_buffer_check_unlocked()
277 void *report = stream->oa_buffer.vaddr + tail; in xe_oa_buffer_check_unlocked()
279 if (oa_report_id(stream, report) || oa_timestamp(stream, report)) in xe_oa_buffer_check_unlocked()
282 tail = xe_oa_circ_diff(stream, tail, report_size); in xe_oa_buffer_check_unlocked()
285 if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size) in xe_oa_buffer_check_unlocked()
286 drm_dbg(&stream->oa->xe->drm, in xe_oa_buffer_check_unlocked()
288 stream->oa_buffer.head, tail, hw_tail); in xe_oa_buffer_check_unlocked()
290 stream->oa_buffer.tail = tail; in xe_oa_buffer_check_unlocked()
292 available = xe_oa_circ_diff(stream, stream->oa_buffer.tail, stream->oa_buffer.head); in xe_oa_buffer_check_unlocked()
293 stream->pollin = available >= stream->wait_num_reports * report_size; in xe_oa_buffer_check_unlocked()
295 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_buffer_check_unlocked()
297 return stream->pollin; in xe_oa_buffer_check_unlocked()
302 struct xe_oa_stream *stream = in xe_oa_poll_check_timer_cb() local
303 container_of(hrtimer, typeof(*stream), poll_check_timer); in xe_oa_poll_check_timer_cb()
305 if (xe_oa_buffer_check_unlocked(stream)) in xe_oa_poll_check_timer_cb()
306 wake_up(&stream->poll_wq); in xe_oa_poll_check_timer_cb()
308 hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns)); in xe_oa_poll_check_timer_cb()
313 static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf, in xe_oa_append_report() argument
316 int report_size = stream->oa_buffer.format->size; in xe_oa_append_report()
325 oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; in xe_oa_append_report()
333 if (copy_to_user(buf, stream->oa_buffer.vaddr, in xe_oa_append_report()
345 static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, in xe_oa_append_reports() argument
348 int report_size = stream->oa_buffer.format->size; in xe_oa_append_reports()
349 u8 *oa_buf_base = stream->oa_buffer.vaddr; in xe_oa_append_reports()
350 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_append_reports()
356 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
357 head = stream->oa_buffer.head; in xe_oa_append_reports()
358 tail = stream->oa_buffer.tail; in xe_oa_append_reports()
359 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
361 xe_assert(stream->oa->xe, in xe_oa_append_reports()
362 head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size); in xe_oa_append_reports()
364 for (; xe_oa_circ_diff(stream, tail, head); in xe_oa_append_reports()
365 head = xe_oa_circ_incr(stream, head, report_size)) { in xe_oa_append_reports()
368 ret = xe_oa_append_report(stream, buf, count, offset, report); in xe_oa_append_reports()
372 if (!(stream->oa_buffer.circ_size % report_size)) { in xe_oa_append_reports()
374 oa_report_id_clear(stream, (void *)report); in xe_oa_append_reports()
375 oa_timestamp_clear(stream, (void *)report); in xe_oa_append_reports()
377 u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; in xe_oa_append_reports()
391 struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr; in xe_oa_append_reports()
393 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
394 xe_mmio_write32(&stream->gt->mmio, oaheadptr, in xe_oa_append_reports()
396 stream->oa_buffer.head = head; in xe_oa_append_reports()
397 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
403 static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream) in xe_oa_init_oa_buffer() argument
405 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_init_oa_buffer()
406 int size_exponent = __ffs(xe_bo_size(stream->oa_buffer.bo)); in xe_oa_init_oa_buffer()
408 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_init_oa_buffer()
418 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_init_oa_buffer()
420 xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0); in xe_oa_init_oa_buffer()
421 xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr, in xe_oa_init_oa_buffer()
423 stream->oa_buffer.head = 0; in xe_oa_init_oa_buffer()
428 xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf); in xe_oa_init_oa_buffer()
429 xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr, in xe_oa_init_oa_buffer()
433 stream->oa_buffer.tail = 0; in xe_oa_init_oa_buffer()
435 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_init_oa_buffer()
438 memset(stream->oa_buffer.vaddr, 0, xe_bo_size(stream->oa_buffer.bo)); in xe_oa_init_oa_buffer()
448 static u32 __oa_ccs_select(struct xe_oa_stream *stream) in __oa_ccs_select() argument
452 if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE) in __oa_ccs_select()
455 val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance); in __oa_ccs_select()
456 xe_assert(stream->oa->xe, in __oa_ccs_select()
457 REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance); in __oa_ccs_select()
461 static u32 __oactrl_used_bits(struct xe_oa_stream *stream) in __oactrl_used_bits() argument
463 return stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG ? in __oactrl_used_bits()
467 static void xe_oa_enable(struct xe_oa_stream *stream) in xe_oa_enable() argument
469 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_enable()
477 xe_oa_init_oa_buffer(stream); in xe_oa_enable()
479 regs = __oa_regs(stream); in xe_oa_enable()
481 __oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE; in xe_oa_enable()
483 if (GRAPHICS_VER(stream->oa->xe) >= 20 && in xe_oa_enable()
484 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG) in xe_oa_enable()
487 xe_mmio_rmw32(&stream->gt->mmio, regs->oa_ctrl, __oactrl_used_bits(stream), val); in xe_oa_enable()
490 static void xe_oa_disable(struct xe_oa_stream *stream) in xe_oa_disable() argument
492 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_disable()
494 xe_mmio_rmw32(mmio, __oa_regs(stream)->oa_ctrl, __oactrl_used_bits(stream), 0); in xe_oa_disable()
495 if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl, in xe_oa_disable()
497 drm_err(&stream->oa->xe->drm, in xe_oa_disable()
500 if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) { in xe_oa_disable()
504 drm_err(&stream->oa->xe->drm, in xe_oa_disable()
509 static int xe_oa_wait_unlocked(struct xe_oa_stream *stream) in xe_oa_wait_unlocked() argument
512 if (!stream->periodic) in xe_oa_wait_unlocked()
515 return wait_event_interruptible(stream->poll_wq, in xe_oa_wait_unlocked()
516 xe_oa_buffer_check_unlocked(stream)); in xe_oa_wait_unlocked()
522 static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf, in __xe_oa_read() argument
526 stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status, in __xe_oa_read()
532 if (stream->oa_status & OASTATUS_RELEVANT_BITS) in __xe_oa_read()
535 return xe_oa_append_reports(stream, buf, count, offset); in __xe_oa_read()
541 struct xe_oa_stream *stream = file->private_data; in xe_oa_read() local
546 if (!stream->enabled || !stream->sample) in xe_oa_read()
551 ret = xe_oa_wait_unlocked(stream); in xe_oa_read()
555 mutex_lock(&stream->stream_lock); in xe_oa_read()
556 ret = __xe_oa_read(stream, buf, count, &offset); in xe_oa_read()
557 mutex_unlock(&stream->stream_lock); in xe_oa_read()
560 xe_oa_buffer_check_unlocked(stream); in xe_oa_read()
561 mutex_lock(&stream->stream_lock); in xe_oa_read()
562 ret = __xe_oa_read(stream, buf, count, &offset); in xe_oa_read()
563 mutex_unlock(&stream->stream_lock); in xe_oa_read()
576 stream->pollin = false; in xe_oa_read()
582 static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream, in xe_oa_poll_locked() argument
587 poll_wait(file, &stream->poll_wq, wait); in xe_oa_poll_locked()
595 if (stream->pollin) in xe_oa_poll_locked()
603 struct xe_oa_stream *stream = file->private_data; in xe_oa_poll() local
606 mutex_lock(&stream->stream_lock); in xe_oa_poll()
607 ret = xe_oa_poll_locked(stream, file, wait); in xe_oa_poll()
608 mutex_unlock(&stream->stream_lock); in xe_oa_poll()
629 static struct dma_fence *xe_oa_submit_bb(struct xe_oa_stream *stream, enum xe_oa_submit_deps deps, in xe_oa_submit_bb() argument
632 struct xe_exec_queue *q = stream->exec_q ?: stream->k_exec_q; in xe_oa_submit_bb()
647 for (int i = 0; i < stream->num_syncs && !err; i++) in xe_oa_submit_bb()
648 err = xe_sync_entry_add_deps(&stream->syncs[i], job); in xe_oa_submit_bb()
650 drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err); in xe_oa_submit_bb()
699 static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream) in xe_oa_free_oa_buffer() argument
701 xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); in xe_oa_free_oa_buffer()
704 static void xe_oa_free_configs(struct xe_oa_stream *stream) in xe_oa_free_configs() argument
708 xe_oa_config_put(stream->oa_config); in xe_oa_free_configs()
709 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in xe_oa_free_configs()
710 free_oa_config_bo(oa_bo, stream->last_fence); in xe_oa_free_configs()
711 dma_fence_put(stream->last_fence); in xe_oa_free_configs()
714 static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri, u32 count) in xe_oa_load_with_lri() argument
720 bb = xe_bb_new(stream->gt, 2 * count + 1, false); in xe_oa_load_with_lri()
728 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb); in xe_oa_load_with_lri()
743 static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable) in xe_oa_configure_oar_context() argument
745 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_configure_oar_context()
751 OACTXCONTROL(stream->hwe->mmio_base), in xe_oa_configure_oar_context()
759 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), in xe_oa_configure_oar_context()
765 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); in xe_oa_configure_oar_context()
768 static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable) in xe_oa_configure_oac_context() argument
770 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_configure_oac_context()
775 OACTXCONTROL(stream->hwe->mmio_base), in xe_oa_configure_oac_context()
783 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), in xe_oa_configure_oac_context()
791 xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl, in xe_oa_configure_oac_context()
792 __oa_ccs_select(stream)); in xe_oa_configure_oac_context()
794 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); in xe_oa_configure_oac_context()
797 static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable) in xe_oa_configure_oa_context() argument
799 switch (stream->hwe->class) { in xe_oa_configure_oa_context()
801 return xe_oa_configure_oar_context(stream, enable); in xe_oa_configure_oa_context()
803 return xe_oa_configure_oac_context(stream, enable); in xe_oa_configure_oa_context()
812 static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable) in oag_configure_mmio_trigger() argument
815 enable && stream && stream->sample ? in oag_configure_mmio_trigger()
819 static void xe_oa_disable_metric_set(struct xe_oa_stream *stream) in xe_oa_disable_metric_set() argument
821 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_disable_metric_set()
825 if (XE_WA(stream->gt, 1508761755)) { in xe_oa_disable_metric_set()
826 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, in xe_oa_disable_metric_set()
828 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, in xe_oa_disable_metric_set()
832 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, in xe_oa_disable_metric_set()
833 oag_configure_mmio_trigger(stream, false)); in xe_oa_disable_metric_set()
836 if (stream->exec_q) in xe_oa_disable_metric_set()
837 xe_oa_configure_oa_context(stream, false); in xe_oa_disable_metric_set()
843 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); in xe_oa_disable_metric_set()
848 if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || in xe_oa_disable_metric_set()
849 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && in xe_oa_disable_metric_set()
850 GRAPHICS_VER(stream->oa->xe) >= 30) in xe_oa_disable_metric_set()
854 static void xe_oa_stream_destroy(struct xe_oa_stream *stream) in xe_oa_stream_destroy() argument
856 struct xe_oa_unit *u = stream->oa_unit; in xe_oa_stream_destroy()
857 struct xe_gt *gt = stream->hwe->gt; in xe_oa_stream_destroy()
859 if (WARN_ON(stream != u->exclusive_stream)) in xe_oa_stream_destroy()
864 mutex_destroy(&stream->stream_lock); in xe_oa_stream_destroy()
866 xe_oa_disable_metric_set(stream); in xe_oa_stream_destroy()
867 xe_exec_queue_put(stream->k_exec_q); in xe_oa_stream_destroy()
869 xe_oa_free_oa_buffer(stream); in xe_oa_stream_destroy()
872 xe_pm_runtime_put(stream->oa->xe); in xe_oa_stream_destroy()
875 if (stream->override_gucrc) in xe_oa_stream_destroy()
878 xe_oa_free_configs(stream); in xe_oa_stream_destroy()
879 xe_file_put(stream->xef); in xe_oa_stream_destroy()
882 static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size) in xe_oa_alloc_oa_buffer() argument
886 bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL, in xe_oa_alloc_oa_buffer()
892 stream->oa_buffer.bo = bo; in xe_oa_alloc_oa_buffer()
894 xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0); in xe_oa_alloc_oa_buffer()
895 stream->oa_buffer.vaddr = bo->vmap.vaddr; in xe_oa_alloc_oa_buffer()
900 __xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) in __xe_oa_alloc_config_buffer() argument
913 bb = xe_bb_new(stream->gt, config_length, false); in __xe_oa_alloc_config_buffer()
921 llist_add(&oa_bo->node, &stream->oa_config_bos); in __xe_oa_alloc_config_buffer()
930 xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) in xe_oa_alloc_config_buffer() argument
935 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in xe_oa_alloc_config_buffer()
942 oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config); in xe_oa_alloc_config_buffer()
947 static void xe_oa_update_last_fence(struct xe_oa_stream *stream, struct dma_fence *fence) in xe_oa_update_last_fence() argument
949 dma_fence_put(stream->last_fence); in xe_oa_update_last_fence()
950 stream->last_fence = dma_fence_get(fence); in xe_oa_update_last_fence()
990 static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config) in xe_oa_emit_oa_config() argument
1004 oa_bo = xe_oa_alloc_config_buffer(stream, config); in xe_oa_emit_oa_config()
1011 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb); in xe_oa_emit_oa_config()
1021 for (i = 0; i < stream->num_syncs; i++) { in xe_oa_emit_oa_config()
1022 if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL) in xe_oa_emit_oa_config()
1024 xe_sync_entry_signal(&stream->syncs[i], &ofence->base); in xe_oa_emit_oa_config()
1032 xe_oa_update_last_fence(stream, fence); in xe_oa_emit_oa_config()
1036 xe_gt_assert(stream->gt, !err || err == -ENOENT); in xe_oa_emit_oa_config()
1047 for (i = 0; i < stream->num_syncs; i++) in xe_oa_emit_oa_config()
1048 xe_sync_entry_cleanup(&stream->syncs[i]); in xe_oa_emit_oa_config()
1049 kfree(stream->syncs); in xe_oa_emit_oa_config()
1057 static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream) in oag_report_ctx_switches() argument
1061 stream->sample ? in oag_report_ctx_switches()
1065 static u32 oag_buf_size_select(const struct xe_oa_stream *stream) in oag_buf_size_select() argument
1068 xe_bo_size(stream->oa_buffer.bo) > SZ_16M ? in oag_buf_size_select()
1072 static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) in xe_oa_enable_metric_set() argument
1074 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_enable_metric_set()
1082 if (XE_WA(stream->gt, 1508761755)) { in xe_oa_enable_metric_set()
1083 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, in xe_oa_enable_metric_set()
1085 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, in xe_oa_enable_metric_set()
1093 if (GRAPHICS_VER(stream->oa->xe) >= 20) in xe_oa_enable_metric_set()
1100 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, in xe_oa_enable_metric_set()
1102 oag_report_ctx_switches(stream) | in xe_oa_enable_metric_set()
1103 oag_buf_size_select(stream) | in xe_oa_enable_metric_set()
1104 oag_configure_mmio_trigger(stream, true)); in xe_oa_enable_metric_set()
1106 xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ? in xe_oa_enable_metric_set()
1110 stream->period_exponent)) : 0); in xe_oa_enable_metric_set()
1118 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); in xe_oa_enable_metric_set()
1121 if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || in xe_oa_enable_metric_set()
1122 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && in xe_oa_enable_metric_set()
1123 GRAPHICS_VER(stream->oa->xe) >= 30) in xe_oa_enable_metric_set()
1127 if (stream->exec_q) { in xe_oa_enable_metric_set()
1128 ret = xe_oa_configure_oa_context(stream, true); in xe_oa_enable_metric_set()
1133 return xe_oa_emit_oa_config(stream, stream->oa_config); in xe_oa_enable_metric_set()
1435 static void xe_oa_stream_enable(struct xe_oa_stream *stream) in xe_oa_stream_enable() argument
1437 stream->pollin = false; in xe_oa_stream_enable()
1439 xe_oa_enable(stream); in xe_oa_stream_enable()
1441 if (stream->sample) in xe_oa_stream_enable()
1442 hrtimer_start(&stream->poll_check_timer, in xe_oa_stream_enable()
1443 ns_to_ktime(stream->poll_period_ns), in xe_oa_stream_enable()
1447 static void xe_oa_stream_disable(struct xe_oa_stream *stream) in xe_oa_stream_disable() argument
1449 xe_oa_disable(stream); in xe_oa_stream_disable()
1451 if (stream->sample) in xe_oa_stream_disable()
1452 hrtimer_cancel(&stream->poll_check_timer); in xe_oa_stream_disable()
1455 static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) in xe_oa_enable_preempt_timeslice() argument
1457 struct xe_exec_queue *q = stream->exec_q; in xe_oa_enable_preempt_timeslice()
1461 ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us); in xe_oa_enable_preempt_timeslice()
1462 ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us); in xe_oa_enable_preempt_timeslice()
1467 drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2); in xe_oa_enable_preempt_timeslice()
1471 static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream) in xe_oa_disable_preempt_timeslice() argument
1473 struct xe_exec_queue *q = stream->exec_q; in xe_oa_disable_preempt_timeslice()
1487 xe_oa_enable_preempt_timeslice(stream); in xe_oa_disable_preempt_timeslice()
1488 drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret); in xe_oa_disable_preempt_timeslice()
1492 static int xe_oa_enable_locked(struct xe_oa_stream *stream) in xe_oa_enable_locked() argument
1494 if (stream->enabled) in xe_oa_enable_locked()
1497 if (stream->no_preempt) { in xe_oa_enable_locked()
1498 int ret = xe_oa_disable_preempt_timeslice(stream); in xe_oa_enable_locked()
1504 xe_oa_stream_enable(stream); in xe_oa_enable_locked()
1506 stream->enabled = true; in xe_oa_enable_locked()
1510 static int xe_oa_disable_locked(struct xe_oa_stream *stream) in xe_oa_disable_locked() argument
1514 if (!stream->enabled) in xe_oa_disable_locked()
1517 xe_oa_stream_disable(stream); in xe_oa_disable_locked()
1519 if (stream->no_preempt) in xe_oa_disable_locked()
1520 ret = xe_oa_enable_preempt_timeslice(stream); in xe_oa_disable_locked()
1522 stream->enabled = false; in xe_oa_disable_locked()
1526 static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg) in xe_oa_config_locked() argument
1529 long ret = stream->oa_config->id; in xe_oa_config_locked()
1533 err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, ¶m); in xe_oa_config_locked()
1537 config = xe_oa_get_oa_config(stream->oa, param.metric_set); in xe_oa_config_locked()
1541 param.xef = stream->xef; in xe_oa_config_locked()
1542 err = xe_oa_parse_syncs(stream->oa, ¶m); in xe_oa_config_locked()
1546 stream->num_syncs = param.num_syncs; in xe_oa_config_locked()
1547 stream->syncs = param.syncs; in xe_oa_config_locked()
1549 err = xe_oa_emit_oa_config(stream, config); in xe_oa_config_locked()
1551 config = xchg(&stream->oa_config, config); in xe_oa_config_locked()
1552 drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n", in xe_oa_config_locked()
1553 stream->oa_config->uuid); in xe_oa_config_locked()
1562 static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg) in xe_oa_status_locked() argument
1568 if (stream->oa_status & OASTATUS_REPORT_LOST) in xe_oa_status_locked()
1570 if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW) in xe_oa_status_locked()
1572 if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW) in xe_oa_status_locked()
1574 if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL) in xe_oa_status_locked()
1583 static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg) in xe_oa_info_locked() argument
1585 struct drm_xe_oa_stream_info info = { .oa_buf_size = xe_bo_size(stream->oa_buffer.bo), }; in xe_oa_info_locked()
1594 static long xe_oa_ioctl_locked(struct xe_oa_stream *stream, in xe_oa_ioctl_locked() argument
1600 return xe_oa_enable_locked(stream); in xe_oa_ioctl_locked()
1602 return xe_oa_disable_locked(stream); in xe_oa_ioctl_locked()
1604 return xe_oa_config_locked(stream, arg); in xe_oa_ioctl_locked()
1606 return xe_oa_status_locked(stream, arg); in xe_oa_ioctl_locked()
1608 return xe_oa_info_locked(stream, arg); in xe_oa_ioctl_locked()
1618 struct xe_oa_stream *stream = file->private_data; in xe_oa_ioctl() local
1621 mutex_lock(&stream->stream_lock); in xe_oa_ioctl()
1622 ret = xe_oa_ioctl_locked(stream, cmd, arg); in xe_oa_ioctl()
1623 mutex_unlock(&stream->stream_lock); in xe_oa_ioctl()
1628 static void xe_oa_destroy_locked(struct xe_oa_stream *stream) in xe_oa_destroy_locked() argument
1630 if (stream->enabled) in xe_oa_destroy_locked()
1631 xe_oa_disable_locked(stream); in xe_oa_destroy_locked()
1633 xe_oa_stream_destroy(stream); in xe_oa_destroy_locked()
1635 if (stream->exec_q) in xe_oa_destroy_locked()
1636 xe_exec_queue_put(stream->exec_q); in xe_oa_destroy_locked()
1638 kfree(stream); in xe_oa_destroy_locked()
1643 struct xe_oa_stream *stream = file->private_data; in xe_oa_release() local
1644 struct xe_gt *gt = stream->gt; in xe_oa_release()
1648 xe_oa_destroy_locked(stream); in xe_oa_release()
1660 struct xe_oa_stream *stream = file->private_data; in xe_oa_mmap() local
1661 struct xe_bo *bo = stream->oa_buffer.bo; in xe_oa_mmap()
1666 drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); in xe_oa_mmap()
1671 if (vma->vm_end - vma->vm_start != xe_bo_size(stream->oa_buffer.bo)) { in xe_oa_mmap()
1672 drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); in xe_oa_mmap()
1681 drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n"); in xe_oa_mmap()
1687 xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma)); in xe_oa_mmap()
1709 static int xe_oa_stream_init(struct xe_oa_stream *stream, in xe_oa_stream_init() argument
1716 stream->exec_q = param->exec_q; in xe_oa_stream_init()
1717 stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS; in xe_oa_stream_init()
1718 stream->oa_unit = param->oa_unit; in xe_oa_stream_init()
1719 stream->hwe = param->hwe; in xe_oa_stream_init()
1720 stream->gt = stream->hwe->gt; in xe_oa_stream_init()
1721 stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format]; in xe_oa_stream_init()
1723 stream->sample = param->sample; in xe_oa_stream_init()
1724 stream->periodic = param->period_exponent >= 0; in xe_oa_stream_init()
1725 stream->period_exponent = param->period_exponent; in xe_oa_stream_init()
1726 stream->no_preempt = param->no_preempt; in xe_oa_stream_init()
1727 stream->wait_num_reports = param->wait_num_reports; in xe_oa_stream_init()
1729 stream->xef = xe_file_get(param->xef); in xe_oa_stream_init()
1730 stream->num_syncs = param->num_syncs; in xe_oa_stream_init()
1731 stream->syncs = param->syncs; in xe_oa_stream_init()
1738 if (GRAPHICS_VER(stream->oa->xe) >= 20 && in xe_oa_stream_init()
1739 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample) in xe_oa_stream_init()
1740 stream->oa_buffer.circ_size = in xe_oa_stream_init()
1742 param->oa_buffer_size % stream->oa_buffer.format->size; in xe_oa_stream_init()
1744 stream->oa_buffer.circ_size = param->oa_buffer_size; in xe_oa_stream_init()
1746 stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set); in xe_oa_stream_init()
1747 if (!stream->oa_config) { in xe_oa_stream_init()
1748 drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set); in xe_oa_stream_init()
1757 if (XE_WA(stream->gt, 1509372804)) { in xe_oa_stream_init()
1763 stream->override_gucrc = true; in xe_oa_stream_init()
1767 xe_pm_runtime_get(stream->oa->xe); in xe_oa_stream_init()
1774 ret = xe_oa_alloc_oa_buffer(stream, param->oa_buffer_size); in xe_oa_stream_init()
1778 stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL, in xe_oa_stream_init()
1779 BIT(stream->hwe->logical_instance), 1, in xe_oa_stream_init()
1780 stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0); in xe_oa_stream_init()
1781 if (IS_ERR(stream->k_exec_q)) { in xe_oa_stream_init()
1782 ret = PTR_ERR(stream->k_exec_q); in xe_oa_stream_init()
1783 drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d", in xe_oa_stream_init()
1784 stream->gt->info.id, stream->hwe->name, ret); in xe_oa_stream_init()
1788 ret = xe_oa_enable_metric_set(stream); in xe_oa_stream_init()
1790 drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n"); in xe_oa_stream_init()
1794 drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n", in xe_oa_stream_init()
1795 stream->oa_config->uuid); in xe_oa_stream_init()
1797 WRITE_ONCE(stream->oa_unit->exclusive_stream, stream); in xe_oa_stream_init()
1799 hrtimer_setup(&stream->poll_check_timer, xe_oa_poll_check_timer_cb, CLOCK_MONOTONIC, in xe_oa_stream_init()
1801 init_waitqueue_head(&stream->poll_wq); in xe_oa_stream_init()
1803 spin_lock_init(&stream->oa_buffer.ptr_lock); in xe_oa_stream_init()
1804 mutex_init(&stream->stream_lock); in xe_oa_stream_init()
1809 xe_oa_disable_metric_set(stream); in xe_oa_stream_init()
1810 xe_exec_queue_put(stream->k_exec_q); in xe_oa_stream_init()
1812 xe_oa_free_oa_buffer(stream); in xe_oa_stream_init()
1815 xe_pm_runtime_put(stream->oa->xe); in xe_oa_stream_init()
1816 if (stream->override_gucrc) in xe_oa_stream_init()
1819 xe_oa_free_configs(stream); in xe_oa_stream_init()
1821 xe_file_put(stream->xef); in xe_oa_stream_init()
1828 struct xe_oa_stream *stream; in xe_oa_stream_open_ioctl_locked() local
1839 stream = kzalloc(sizeof(*stream), GFP_KERNEL); in xe_oa_stream_open_ioctl_locked()
1840 if (!stream) { in xe_oa_stream_open_ioctl_locked()
1845 stream->oa = oa; in xe_oa_stream_open_ioctl_locked()
1846 ret = xe_oa_stream_init(stream, param); in xe_oa_stream_open_ioctl_locked()
1851 ret = xe_oa_enable_locked(stream); in xe_oa_stream_open_ioctl_locked()
1856 stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0); in xe_oa_stream_open_ioctl_locked()
1863 drm_dev_get(&stream->oa->xe->drm); in xe_oa_stream_open_ioctl_locked()
1868 xe_oa_disable_locked(stream); in xe_oa_stream_open_ioctl_locked()
1870 xe_oa_stream_destroy(stream); in xe_oa_stream_open_ioctl_locked()
1872 kfree(stream); in xe_oa_stream_open_ioctl_locked()