Lines Matching refs:stream
421 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream) in gen12_oa_hw_tail_read() argument
423 struct intel_uncore *uncore = stream->uncore; in gen12_oa_hw_tail_read()
429 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream) in gen8_oa_hw_tail_read() argument
431 struct intel_uncore *uncore = stream->uncore; in gen8_oa_hw_tail_read()
436 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream) in gen7_oa_hw_tail_read() argument
438 struct intel_uncore *uncore = stream->uncore; in gen7_oa_hw_tail_read()
468 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) in oa_buffer_check_unlocked() argument
470 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in oa_buffer_check_unlocked()
471 int report_size = stream->oa_buffer.format->size; in oa_buffer_check_unlocked()
481 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
483 hw_tail = stream->perf->ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
492 if (hw_tail == stream->oa_buffer.aging_tail && in oa_buffer_check_unlocked()
493 (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) { in oa_buffer_check_unlocked()
498 stream->oa_buffer.tail = stream->oa_buffer.aging_tail; in oa_buffer_check_unlocked()
506 head = stream->oa_buffer.head - gtt_offset; in oa_buffer_check_unlocked()
507 aged_tail = stream->oa_buffer.tail - gtt_offset; in oa_buffer_check_unlocked()
524 u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail); in oa_buffer_check_unlocked()
533 __ratelimit(&stream->perf->tail_pointer_race)) in oa_buffer_check_unlocked()
534 drm_notice(&stream->uncore->i915->drm, in oa_buffer_check_unlocked()
538 stream->oa_buffer.tail = gtt_offset + tail; in oa_buffer_check_unlocked()
539 stream->oa_buffer.aging_tail = gtt_offset + hw_tail; in oa_buffer_check_unlocked()
540 stream->oa_buffer.aging_timestamp = now; in oa_buffer_check_unlocked()
543 pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset, in oa_buffer_check_unlocked()
544 stream->oa_buffer.head - gtt_offset) >= report_size; in oa_buffer_check_unlocked()
546 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
566 static int append_oa_status(struct i915_perf_stream *stream, in append_oa_status() argument
602 static int append_oa_sample(struct i915_perf_stream *stream, in append_oa_sample() argument
608 int report_size = stream->oa_buffer.format->size; in append_oa_sample()
613 header.size = stream->sample_size; in append_oa_sample()
652 static int gen8_append_oa_reports(struct i915_perf_stream *stream, in gen8_append_oa_reports() argument
657 struct intel_uncore *uncore = stream->uncore; in gen8_append_oa_reports()
658 int report_size = stream->oa_buffer.format->size; in gen8_append_oa_reports()
659 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
660 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
667 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen8_append_oa_reports()
670 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
672 head = stream->oa_buffer.head; in gen8_append_oa_reports()
673 tail = stream->oa_buffer.tail; in gen8_append_oa_reports()
675 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
733 (GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
737 ctx_id = report32[2] & stream->specific_ctx_id_mask; in gen8_append_oa_reports()
747 if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) && in gen8_append_oa_reports()
748 GRAPHICS_VER(stream->perf->i915) <= 11) in gen8_append_oa_reports()
782 if (!stream->ctx || in gen8_append_oa_reports()
783 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
784 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
791 if (stream->ctx && in gen8_append_oa_reports()
792 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
796 ret = append_oa_sample(stream, buf, count, offset, in gen8_append_oa_reports()
801 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
815 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
818 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
827 stream->oa_buffer.head = head; in gen8_append_oa_reports()
829 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
855 static int gen8_oa_read(struct i915_perf_stream *stream, in gen8_oa_read() argument
860 struct intel_uncore *uncore = stream->uncore; in gen8_oa_read()
865 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen8_oa_read()
868 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_oa_read()
888 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
893 drm_dbg(&stream->perf->i915->drm, in gen8_oa_read()
895 stream->period_exponent); in gen8_oa_read()
897 stream->perf->ops.oa_disable(stream); in gen8_oa_read()
898 stream->perf->ops.oa_enable(stream); in gen8_oa_read()
908 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
921 return gen8_append_oa_reports(stream, buf, count, offset); in gen8_oa_read()
945 static int gen7_append_oa_reports(struct i915_perf_stream *stream, in gen7_append_oa_reports() argument
950 struct intel_uncore *uncore = stream->uncore; in gen7_append_oa_reports()
951 int report_size = stream->oa_buffer.format->size; in gen7_append_oa_reports()
952 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
953 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
960 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen7_append_oa_reports()
963 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
965 head = stream->oa_buffer.head; in gen7_append_oa_reports()
966 tail = stream->oa_buffer.tail; in gen7_append_oa_reports()
968 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1018 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen7_append_oa_reports()
1024 ret = append_oa_sample(stream, buf, count, offset, report); in gen7_append_oa_reports()
1036 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1046 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1048 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1070 static int gen7_oa_read(struct i915_perf_stream *stream, in gen7_oa_read() argument
1075 struct intel_uncore *uncore = stream->uncore; in gen7_oa_read()
1079 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen7_oa_read()
1089 oastatus1 &= ~stream->perf->gen7_latched_oastatus1; in gen7_oa_read()
1112 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1117 drm_dbg(&stream->perf->i915->drm, in gen7_oa_read()
1119 stream->period_exponent); in gen7_oa_read()
1121 stream->perf->ops.oa_disable(stream); in gen7_oa_read()
1122 stream->perf->ops.oa_enable(stream); in gen7_oa_read()
1128 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1132 stream->perf->gen7_latched_oastatus1 |= in gen7_oa_read()
1136 return gen7_append_oa_reports(stream, buf, count, offset); in gen7_oa_read()
1153 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) in i915_oa_wait_unlocked() argument
1156 if (!stream->periodic) in i915_oa_wait_unlocked()
1159 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1160 oa_buffer_check_unlocked(stream)); in i915_oa_wait_unlocked()
1173 static void i915_oa_poll_wait(struct i915_perf_stream *stream, in i915_oa_poll_wait() argument
1177 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1192 static int i915_oa_read(struct i915_perf_stream *stream, in i915_oa_read() argument
1197 return stream->perf->ops.read(stream, buf, count, offset); in i915_oa_read()
1200 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream) in oa_pin_context() argument
1203 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1209 if (ce->engine != stream->engine) /* first match! */ in oa_pin_context()
1237 stream->pinned_ctx = ce; in oa_pin_context()
1238 return stream->pinned_ctx; in oa_pin_context()
1332 static int gen12_get_render_context_id(struct i915_perf_stream *stream) in gen12_get_render_context_id() argument
1337 if (intel_engine_uses_guc(stream->engine)) { in gen12_get_render_context_id()
1338 ret = gen12_guc_sw_ctx_id(stream->pinned_ctx, &ctx_id); in gen12_get_render_context_id()
1344 } else if (GRAPHICS_VER_FULL(stream->engine->i915) >= IP_VER(12, 50)) { in gen12_get_render_context_id()
1357 stream->specific_ctx_id = ctx_id & mask; in gen12_get_render_context_id()
1358 stream->specific_ctx_id_mask = mask; in gen12_get_render_context_id()
1444 static int oa_get_render_ctx_id(struct i915_perf_stream *stream) in oa_get_render_ctx_id() argument
1449 ce = oa_pin_context(stream); in oa_get_render_ctx_id()
1453 if (engine_supports_mi_query(stream->engine) && in oa_get_render_ctx_id()
1454 HAS_LOGICAL_RING_CONTEXTS(stream->perf->i915)) { in oa_get_render_ctx_id()
1462 drm_err(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1464 stream->engine->name); in oa_get_render_ctx_id()
1475 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1476 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1493 stream->specific_ctx_id = ce->lrc.lrca >> 12; in oa_get_render_ctx_id()
1499 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1502 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1504 stream->specific_ctx_id = stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1510 ret = gen12_get_render_context_id(stream); in oa_get_render_ctx_id()
1517 ce->tag = stream->specific_ctx_id; in oa_get_render_ctx_id()
1519 drm_dbg(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1521 stream->specific_ctx_id, in oa_get_render_ctx_id()
1522 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1534 static void oa_put_render_ctx_id(struct i915_perf_stream *stream) in oa_put_render_ctx_id() argument
1538 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1544 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1545 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1549 free_oa_buffer(struct i915_perf_stream *stream) in free_oa_buffer() argument
1551 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1554 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1558 free_oa_configs(struct i915_perf_stream *stream) in free_oa_configs() argument
1562 i915_oa_config_put(stream->oa_config); in free_oa_configs()
1563 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in free_oa_configs()
1568 free_noa_wait(struct i915_perf_stream *stream) in free_noa_wait() argument
1570 i915_vma_unpin_and_release(&stream->noa_wait, 0); in free_noa_wait()
1573 static void i915_oa_stream_destroy(struct i915_perf_stream *stream) in i915_oa_stream_destroy() argument
1575 struct i915_perf *perf = stream->perf; in i915_oa_stream_destroy()
1576 struct intel_gt *gt = stream->engine->gt; in i915_oa_stream_destroy()
1578 if (WARN_ON(stream != gt->perf.exclusive_stream)) in i915_oa_stream_destroy()
1588 perf->ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1590 free_oa_buffer(stream); in i915_oa_stream_destroy()
1601 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1602 intel_engine_pm_put(stream->engine); in i915_oa_stream_destroy()
1604 if (stream->ctx) in i915_oa_stream_destroy()
1605 oa_put_render_ctx_id(stream); in i915_oa_stream_destroy()
1607 free_oa_configs(stream); in i915_oa_stream_destroy()
1608 free_noa_wait(stream); in i915_oa_stream_destroy()
1617 static void gen7_init_oa_buffer(struct i915_perf_stream *stream) in gen7_init_oa_buffer() argument
1619 struct intel_uncore *uncore = stream->uncore; in gen7_init_oa_buffer()
1620 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1623 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1630 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1638 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1639 stream->oa_buffer.tail = gtt_offset; in gen7_init_oa_buffer()
1641 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1647 stream->perf->gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1660 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1663 static void gen8_init_oa_buffer(struct i915_perf_stream *stream) in gen8_init_oa_buffer() argument
1665 struct intel_uncore *uncore = stream->uncore; in gen8_init_oa_buffer()
1666 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1669 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1673 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1690 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1691 stream->oa_buffer.tail = gtt_offset; in gen8_init_oa_buffer()
1698 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1700 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1714 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1717 static void gen12_init_oa_buffer(struct i915_perf_stream *stream) in gen12_init_oa_buffer() argument
1719 struct intel_uncore *uncore = stream->uncore; in gen12_init_oa_buffer()
1720 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen12_init_oa_buffer()
1723 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1728 stream->oa_buffer.head = gtt_offset; in gen12_init_oa_buffer()
1744 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen12_init_oa_buffer()
1745 stream->oa_buffer.tail = gtt_offset; in gen12_init_oa_buffer()
1752 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen12_init_oa_buffer()
1754 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1768 memset(stream->oa_buffer.vaddr, 0, in gen12_init_oa_buffer()
1769 stream->oa_buffer.vma->size); in gen12_init_oa_buffer()
1772 static int alloc_oa_buffer(struct i915_perf_stream *stream) in alloc_oa_buffer() argument
1774 struct drm_i915_private *i915 = stream->perf->i915; in alloc_oa_buffer()
1775 struct intel_gt *gt = stream->engine->gt; in alloc_oa_buffer()
1780 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma)) in alloc_oa_buffer()
1786 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE); in alloc_oa_buffer()
1811 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1813 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1815 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1816 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1828 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1829 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1834 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs, in save_restore_register() argument
1843 if (GRAPHICS_VER(stream->perf->i915) >= 8) in save_restore_register()
1849 *cs++ = i915_ggtt_offset(stream->noa_wait) + offset + 4 * d; in save_restore_register()
1856 static int alloc_noa_wait(struct i915_perf_stream *stream) in alloc_noa_wait() argument
1858 struct drm_i915_private *i915 = stream->perf->i915; in alloc_noa_wait()
1859 struct intel_gt *gt = stream->engine->gt; in alloc_noa_wait()
1863 intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915), in alloc_noa_wait()
1864 atomic64_read(&stream->perf->noa_programming_delay)); in alloc_noa_wait()
1865 const u32 base = stream->engine->mmio_base; in alloc_noa_wait()
1922 stream->noa_wait = vma; in alloc_noa_wait()
1930 stream, cs, true /* save */, CS_GPR(i), in alloc_noa_wait()
1933 stream, cs, true /* save */, mi_predicate_result, in alloc_noa_wait()
2049 stream, cs, false /* restore */, CS_GPR(i), in alloc_noa_wait()
2052 stream, cs, false /* restore */, mi_predicate_result, in alloc_noa_wait()
2113 alloc_oa_config_buffer(struct i915_perf_stream *stream, in alloc_oa_config_buffer() argument
2133 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length); in alloc_oa_config_buffer()
2162 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ? in alloc_oa_config_buffer()
2165 *cs++ = i915_ggtt_offset(stream->noa_wait); in alloc_oa_config_buffer()
2172 &stream->engine->gt->ggtt->vm, in alloc_oa_config_buffer()
2180 llist_add(&oa_bo->node, &stream->oa_config_bos); in alloc_oa_config_buffer()
2201 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config) in get_oa_vma() argument
2209 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in get_oa_vma()
2217 oa_bo = alloc_oa_config_buffer(stream, oa_config); in get_oa_vma()
2226 emit_oa_config(struct i915_perf_stream *stream, in emit_oa_config() argument
2236 vma = get_oa_vma(stream, oa_config); in emit_oa_config()
2296 static struct intel_context *oa_context(struct i915_perf_stream *stream) in oa_context() argument
2298 return stream->pinned_ctx ?: stream->engine->kernel_context; in oa_context()
2302 hsw_enable_metric_set(struct i915_perf_stream *stream, in hsw_enable_metric_set() argument
2305 struct intel_uncore *uncore = stream->uncore; in hsw_enable_metric_set()
2322 return emit_oa_config(stream, in hsw_enable_metric_set()
2323 stream->oa_config, oa_context(stream), in hsw_enable_metric_set()
2327 static void hsw_disable_metric_set(struct i915_perf_stream *stream) in hsw_disable_metric_set() argument
2329 struct intel_uncore *uncore = stream->uncore; in hsw_disable_metric_set()
2369 const struct i915_perf_stream *stream) in gen8_update_reg_state_unlocked() argument
2371 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
2372 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
2387 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
2388 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
2393 oa_config_flex_reg(stream->oa_config, flex_regs[i]); in gen8_update_reg_state_unlocked()
2529 static int gen12_configure_oar_context(struct i915_perf_stream *stream, in gen12_configure_oar_context() argument
2533 struct intel_context *ce = stream->pinned_ctx; in gen12_configure_oar_context()
2534 u32 format = stream->oa_buffer.format->format; in gen12_configure_oar_context()
2535 u32 offset = stream->perf->ctx_oactxctrl_offset; in gen12_configure_oar_context()
2605 oa_configure_all_contexts(struct i915_perf_stream *stream, in oa_configure_all_contexts() argument
2610 struct drm_i915_private *i915 = stream->perf->i915; in oa_configure_all_contexts()
2612 struct intel_gt *gt = stream->engine->gt; in oa_configure_all_contexts()
2675 gen12_configure_all_contexts(struct i915_perf_stream *stream, in gen12_configure_all_contexts() argument
2686 return oa_configure_all_contexts(stream, in gen12_configure_all_contexts()
2692 lrc_configure_all_contexts(struct i915_perf_stream *stream, in lrc_configure_all_contexts() argument
2696 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in lrc_configure_all_contexts()
2698 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in lrc_configure_all_contexts()
2721 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in lrc_configure_all_contexts()
2722 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in lrc_configure_all_contexts()
2728 return oa_configure_all_contexts(stream, in lrc_configure_all_contexts()
2734 gen8_enable_metric_set(struct i915_perf_stream *stream, in gen8_enable_metric_set() argument
2737 struct intel_uncore *uncore = stream->uncore; in gen8_enable_metric_set()
2738 struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
2764 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) { in gen8_enable_metric_set()
2775 ret = lrc_configure_all_contexts(stream, oa_config, active); in gen8_enable_metric_set()
2779 return emit_oa_config(stream, in gen8_enable_metric_set()
2780 stream->oa_config, oa_context(stream), in gen8_enable_metric_set()
2784 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream) in oag_report_ctx_switches() argument
2787 (stream->sample_flags & SAMPLE_OA_REPORT) ? in oag_report_ctx_switches()
2792 gen12_enable_metric_set(struct i915_perf_stream *stream, in gen12_enable_metric_set() argument
2795 struct drm_i915_private *i915 = stream->perf->i915; in gen12_enable_metric_set()
2796 struct intel_uncore *uncore = stream->uncore; in gen12_enable_metric_set()
2797 struct i915_oa_config *oa_config = stream->oa_config; in gen12_enable_metric_set()
2798 bool periodic = stream->periodic; in gen12_enable_metric_set()
2799 u32 period_exponent = stream->period_exponent; in gen12_enable_metric_set()
2823 oag_report_ctx_switches(stream)); in gen12_enable_metric_set()
2846 ret = gen12_configure_all_contexts(stream, oa_config, active); in gen12_enable_metric_set()
2855 if (stream->ctx) { in gen12_enable_metric_set()
2856 ret = gen12_configure_oar_context(stream, active); in gen12_enable_metric_set()
2861 return emit_oa_config(stream, in gen12_enable_metric_set()
2862 stream->oa_config, oa_context(stream), in gen12_enable_metric_set()
2866 static void gen8_disable_metric_set(struct i915_perf_stream *stream) in gen8_disable_metric_set() argument
2868 struct intel_uncore *uncore = stream->uncore; in gen8_disable_metric_set()
2871 lrc_configure_all_contexts(stream, NULL, NULL); in gen8_disable_metric_set()
2876 static void gen11_disable_metric_set(struct i915_perf_stream *stream) in gen11_disable_metric_set() argument
2878 struct intel_uncore *uncore = stream->uncore; in gen11_disable_metric_set()
2881 lrc_configure_all_contexts(stream, NULL, NULL); in gen11_disable_metric_set()
2887 static void gen12_disable_metric_set(struct i915_perf_stream *stream) in gen12_disable_metric_set() argument
2889 struct intel_uncore *uncore = stream->uncore; in gen12_disable_metric_set()
2890 struct drm_i915_private *i915 = stream->perf->i915; in gen12_disable_metric_set()
2905 gen12_configure_all_contexts(stream, NULL, NULL); in gen12_disable_metric_set()
2908 if (stream->ctx) in gen12_disable_metric_set()
2909 gen12_configure_oar_context(stream, NULL); in gen12_disable_metric_set()
2921 static void gen7_oa_enable(struct i915_perf_stream *stream) in gen7_oa_enable() argument
2923 struct intel_uncore *uncore = stream->uncore; in gen7_oa_enable()
2924 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
2925 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
2926 bool periodic = stream->periodic; in gen7_oa_enable()
2927 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
2928 u32 report_format = stream->oa_buffer.format->format; in gen7_oa_enable()
2939 gen7_init_oa_buffer(stream); in gen7_oa_enable()
2951 static void gen8_oa_enable(struct i915_perf_stream *stream) in gen8_oa_enable() argument
2953 struct intel_uncore *uncore = stream->uncore; in gen8_oa_enable()
2954 u32 report_format = stream->oa_buffer.format->format; in gen8_oa_enable()
2965 gen8_init_oa_buffer(stream); in gen8_oa_enable()
2977 static void gen12_oa_enable(struct i915_perf_stream *stream) in gen12_oa_enable() argument
2979 struct intel_uncore *uncore = stream->uncore; in gen12_oa_enable()
2980 u32 report_format = stream->oa_buffer.format->format; in gen12_oa_enable()
2986 if (!(stream->sample_flags & SAMPLE_OA_REPORT)) in gen12_oa_enable()
2989 gen12_init_oa_buffer(stream); in gen12_oa_enable()
3005 static void i915_oa_stream_enable(struct i915_perf_stream *stream) in i915_oa_stream_enable() argument
3007 stream->pollin = false; in i915_oa_stream_enable()
3009 stream->perf->ops.oa_enable(stream); in i915_oa_stream_enable()
3011 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_enable()
3012 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
3013 ns_to_ktime(stream->poll_oa_period), in i915_oa_stream_enable()
3017 static void gen7_oa_disable(struct i915_perf_stream *stream) in gen7_oa_disable() argument
3019 struct intel_uncore *uncore = stream->uncore; in gen7_oa_disable()
3025 drm_err(&stream->perf->i915->drm, in gen7_oa_disable()
3029 static void gen8_oa_disable(struct i915_perf_stream *stream) in gen8_oa_disable() argument
3031 struct intel_uncore *uncore = stream->uncore; in gen8_oa_disable()
3037 drm_err(&stream->perf->i915->drm, in gen8_oa_disable()
3041 static void gen12_oa_disable(struct i915_perf_stream *stream) in gen12_oa_disable() argument
3043 struct intel_uncore *uncore = stream->uncore; in gen12_oa_disable()
3050 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
3058 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
3070 static void i915_oa_stream_disable(struct i915_perf_stream *stream) in i915_oa_stream_disable() argument
3072 stream->perf->ops.oa_disable(stream); in i915_oa_stream_disable()
3074 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_disable()
3075 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
3087 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream) in i915_perf_stream_enable_sync() argument
3096 err = stream->perf->ops.enable_metric_set(stream, active); in i915_perf_stream_enable_sync()
3181 static int i915_oa_stream_init(struct i915_perf_stream *stream, in i915_oa_stream_init() argument
3185 struct drm_i915_private *i915 = stream->perf->i915; in i915_oa_stream_init()
3186 struct i915_perf *perf = stream->perf; in i915_oa_stream_init()
3191 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3203 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3209 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) { in i915_oa_stream_init()
3210 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3216 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3227 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3233 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3238 stream->engine = props->engine; in i915_oa_stream_init()
3239 stream->uncore = stream->engine->gt->uncore; in i915_oa_stream_init()
3241 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
3243 stream->oa_buffer.format = &perf->oa_formats[props->oa_format]; in i915_oa_stream_init()
3244 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format->size == 0)) in i915_oa_stream_init()
3247 stream->sample_flags = props->sample_flags; in i915_oa_stream_init()
3248 stream->sample_size += stream->oa_buffer.format->size; in i915_oa_stream_init()
3250 stream->hold_preemption = props->hold_preemption; in i915_oa_stream_init()
3252 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
3253 if (stream->periodic) in i915_oa_stream_init()
3254 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
3256 if (stream->ctx) { in i915_oa_stream_init()
3257 ret = oa_get_render_ctx_id(stream); in i915_oa_stream_init()
3259 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3265 ret = alloc_noa_wait(stream); in i915_oa_stream_init()
3267 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3272 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set); in i915_oa_stream_init()
3273 if (!stream->oa_config) { in i915_oa_stream_init()
3274 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3292 intel_engine_pm_get(stream->engine); in i915_oa_stream_init()
3293 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
3306 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3312 ret = alloc_oa_buffer(stream); in i915_oa_stream_init()
3316 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
3318 stream->engine->gt->perf.sseu = props->sseu; in i915_oa_stream_init()
3319 WRITE_ONCE(gt->perf.exclusive_stream, stream); in i915_oa_stream_init()
3321 ret = i915_perf_stream_enable_sync(stream); in i915_oa_stream_init()
3323 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3328 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
3330 stream->oa_config->uuid); in i915_oa_stream_init()
3332 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
3334 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
3335 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
3336 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
3337 mutex_init(&stream->lock); in i915_oa_stream_init()
3343 perf->ops.disable_metric_set(stream); in i915_oa_stream_init()
3345 free_oa_buffer(stream); in i915_oa_stream_init()
3348 free_oa_configs(stream); in i915_oa_stream_init()
3350 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
3351 intel_engine_pm_put(stream->engine); in i915_oa_stream_init()
3354 free_noa_wait(stream); in i915_oa_stream_init()
3357 if (stream->ctx) in i915_oa_stream_init()
3358 oa_put_render_ctx_id(stream); in i915_oa_stream_init()
3366 struct i915_perf_stream *stream; in i915_oa_init_reg_state() local
3372 stream = READ_ONCE(engine->gt->perf.exclusive_stream); in i915_oa_init_reg_state()
3373 if (stream && GRAPHICS_VER(stream->perf->i915) < 12) in i915_oa_init_reg_state()
3374 gen8_update_reg_state_unlocked(ce, stream); in i915_oa_init_reg_state()
3400 struct i915_perf_stream *stream = file->private_data; in i915_perf_read() local
3408 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT)) in i915_perf_read()
3420 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
3424 mutex_lock(&stream->lock); in i915_perf_read()
3425 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3426 mutex_unlock(&stream->lock); in i915_perf_read()
3429 mutex_lock(&stream->lock); in i915_perf_read()
3430 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3431 mutex_unlock(&stream->lock); in i915_perf_read()
3446 stream->pollin = false; in i915_perf_read()
3454 struct i915_perf_stream *stream = in oa_poll_check_timer_cb() local
3455 container_of(hrtimer, typeof(*stream), poll_check_timer); in oa_poll_check_timer_cb()
3457 if (oa_buffer_check_unlocked(stream)) { in oa_poll_check_timer_cb()
3458 stream->pollin = true; in oa_poll_check_timer_cb()
3459 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
3463 ns_to_ktime(stream->poll_oa_period)); in oa_poll_check_timer_cb()
3480 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream, in i915_perf_poll_locked() argument
3486 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
3494 if (stream->pollin) in i915_perf_poll_locked()
3515 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll() local
3518 mutex_lock(&stream->lock); in i915_perf_poll()
3519 ret = i915_perf_poll_locked(stream, file, wait); in i915_perf_poll()
3520 mutex_unlock(&stream->lock); in i915_perf_poll()
3535 static void i915_perf_enable_locked(struct i915_perf_stream *stream) in i915_perf_enable_locked() argument
3537 if (stream->enabled) in i915_perf_enable_locked()
3541 stream->enabled = true; in i915_perf_enable_locked()
3543 if (stream->ops->enable) in i915_perf_enable_locked()
3544 stream->ops->enable(stream); in i915_perf_enable_locked()
3546 if (stream->hold_preemption) in i915_perf_enable_locked()
3547 intel_context_set_nopreempt(stream->pinned_ctx); in i915_perf_enable_locked()
3564 static void i915_perf_disable_locked(struct i915_perf_stream *stream) in i915_perf_disable_locked() argument
3566 if (!stream->enabled) in i915_perf_disable_locked()
3570 stream->enabled = false; in i915_perf_disable_locked()
3572 if (stream->hold_preemption) in i915_perf_disable_locked()
3573 intel_context_clear_nopreempt(stream->pinned_ctx); in i915_perf_disable_locked()
3575 if (stream->ops->disable) in i915_perf_disable_locked()
3576 stream->ops->disable(stream); in i915_perf_disable_locked()
3579 static long i915_perf_config_locked(struct i915_perf_stream *stream, in i915_perf_config_locked() argument
3583 long ret = stream->oa_config->id; in i915_perf_config_locked()
3585 config = i915_perf_get_oa_config(stream->perf, metrics_set); in i915_perf_config_locked()
3589 if (config != stream->oa_config) { in i915_perf_config_locked()
3601 err = emit_oa_config(stream, config, oa_context(stream), NULL); in i915_perf_config_locked()
3603 config = xchg(&stream->oa_config, config); in i915_perf_config_locked()
3622 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, in i915_perf_ioctl_locked() argument
3628 i915_perf_enable_locked(stream); in i915_perf_ioctl_locked()
3631 i915_perf_disable_locked(stream); in i915_perf_ioctl_locked()
3634 return i915_perf_config_locked(stream, arg); in i915_perf_ioctl_locked()
3655 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl() local
3658 mutex_lock(&stream->lock); in i915_perf_ioctl()
3659 ret = i915_perf_ioctl_locked(stream, cmd, arg); in i915_perf_ioctl()
3660 mutex_unlock(&stream->lock); in i915_perf_ioctl()
3675 static void i915_perf_destroy_locked(struct i915_perf_stream *stream) in i915_perf_destroy_locked() argument
3677 if (stream->enabled) in i915_perf_destroy_locked()
3678 i915_perf_disable_locked(stream); in i915_perf_destroy_locked()
3680 if (stream->ops->destroy) in i915_perf_destroy_locked()
3681 stream->ops->destroy(stream); in i915_perf_destroy_locked()
3683 if (stream->ctx) in i915_perf_destroy_locked()
3684 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
3686 kfree(stream); in i915_perf_destroy_locked()
3702 struct i915_perf_stream *stream = file->private_data; in i915_perf_release() local
3703 struct i915_perf *perf = stream->perf; in i915_perf_release()
3704 struct intel_gt *gt = stream->engine->gt; in i915_perf_release()
3712 i915_perf_destroy_locked(stream); in i915_perf_release()
3767 struct i915_perf_stream *stream = NULL; in i915_perf_open_ioctl_locked() local
3843 stream = kzalloc(sizeof(*stream), GFP_KERNEL); in i915_perf_open_ioctl_locked()
3844 if (!stream) { in i915_perf_open_ioctl_locked()
3849 stream->perf = perf; in i915_perf_open_ioctl_locked()
3850 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
3851 stream->poll_oa_period = props->poll_oa_period; in i915_perf_open_ioctl_locked()
3853 ret = i915_oa_stream_init(stream, param, props); in i915_perf_open_ioctl_locked()
3861 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
3871 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); in i915_perf_open_ioctl_locked()
3878 i915_perf_enable_locked(stream); in i915_perf_open_ioctl_locked()
3888 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
3889 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
3891 kfree(stream); in i915_perf_open_ioctl_locked()