Lines Matching refs:cpuc
105 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
122 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in __intel_pmu_lbr_enable() local
136 if (cpuc->lbr_sel) in __intel_pmu_lbr_enable()
137 lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask; in __intel_pmu_lbr_enable()
138 if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel) in __intel_pmu_lbr_enable()
191 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_reset() local
198 cpuc->last_task_ctx = NULL; in intel_pmu_lbr_reset()
199 cpuc->last_log_id = 0; in intel_pmu_lbr_reset()
200 if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select) in intel_pmu_lbr_reset()
361 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_restore() local
384 if (cpuc->lbr_select) in intel_pmu_lbr_restore()
426 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in __intel_pmu_lbr_restore() local
439 if ((ctx == cpuc->last_task_ctx) && in __intel_pmu_lbr_restore()
440 (task_context_opt(ctx)->log_id == cpuc->last_log_id) && in __intel_pmu_lbr_restore()
453 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_save() local
470 if (cpuc->lbr_select) in intel_pmu_lbr_save()
503 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in __intel_pmu_lbr_save() local
514 cpuc->last_task_ctx = ctx; in __intel_pmu_lbr_save()
515 cpuc->last_log_id = ++task_context_opt(ctx)->log_id; in __intel_pmu_lbr_save()
543 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_sched_task() local
546 if (!cpuc->lbr_users) in intel_pmu_lbr_sched_task()
580 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_add() local
586 cpuc->lbr_select = 1; in intel_pmu_lbr_add()
588 cpuc->br_sel = event->hw.branch_reg.reg; in intel_pmu_lbr_add()
590 if (branch_user_callstack(cpuc->br_sel) && event->pmu_ctx->task_ctx_data) in intel_pmu_lbr_add()
613 cpuc->lbr_pebs_users++; in intel_pmu_lbr_add()
615 if (!cpuc->lbr_users++ && !event->total_time_running) in intel_pmu_lbr_add()
622 struct cpu_hw_events *cpuc; in release_lbr_buffers() local
629 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); in release_lbr_buffers()
631 if (kmem_cache && cpuc->lbr_xsave) { in release_lbr_buffers()
632 kmem_cache_free(kmem_cache, cpuc->lbr_xsave); in release_lbr_buffers()
633 cpuc->lbr_xsave = NULL; in release_lbr_buffers()
641 struct cpu_hw_events *cpuc; in reserve_lbr_buffers() local
648 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); in reserve_lbr_buffers()
650 if (!kmem_cache || cpuc->lbr_xsave) in reserve_lbr_buffers()
653 cpuc->lbr_xsave = kmem_cache_alloc_node(kmem_cache, in reserve_lbr_buffers()
661 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_del() local
666 if (branch_user_callstack(cpuc->br_sel) && in intel_pmu_lbr_del()
671 cpuc->lbr_select = 0; in intel_pmu_lbr_del()
674 cpuc->lbr_pebs_users--; in intel_pmu_lbr_del()
675 cpuc->lbr_users--; in intel_pmu_lbr_del()
676 WARN_ON_ONCE(cpuc->lbr_users < 0); in intel_pmu_lbr_del()
677 WARN_ON_ONCE(cpuc->lbr_pebs_users < 0); in intel_pmu_lbr_del()
683 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in vlbr_exclude_host() local
686 (unsigned long *)&cpuc->intel_ctrl_guest_mask); in vlbr_exclude_host()
691 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_enable_all() local
693 if (cpuc->lbr_users && !vlbr_exclude_host()) in intel_pmu_lbr_enable_all()
699 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_disable_all() local
701 if (cpuc->lbr_users && !vlbr_exclude_host()) { in intel_pmu_lbr_disable_all()
709 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) in intel_pmu_lbr_read_32() argument
712 struct perf_branch_entry *br = cpuc->lbr_entries; in intel_pmu_lbr_read_32()
734 cpuc->lbr_stack.nr = i; in intel_pmu_lbr_read_32()
735 cpuc->lbr_stack.hw_idx = tos; in intel_pmu_lbr_read_32()
743 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) in intel_pmu_lbr_read_64() argument
747 struct perf_branch_entry *br = cpuc->lbr_entries; in intel_pmu_lbr_read_64()
753 if (cpuc->lbr_sel) { in intel_pmu_lbr_read_64()
754 need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO); in intel_pmu_lbr_read_64()
755 if (cpuc->lbr_sel->config & LBR_CALL_STACK) in intel_pmu_lbr_read_64()
829 cpuc->lbr_stack.nr = out; in intel_pmu_lbr_read_64()
830 cpuc->lbr_stack.hw_idx = tos; in intel_pmu_lbr_read_64()
869 static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, in intel_pmu_store_lbr() argument
879 e = &cpuc->lbr_entries[i]; in intel_pmu_store_lbr()
903 cpuc->lbr_stack.nr = i; in intel_pmu_store_lbr()
906 static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc) in intel_pmu_arch_lbr_read() argument
908 intel_pmu_store_lbr(cpuc, NULL); in intel_pmu_arch_lbr_read()
911 static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc) in intel_pmu_arch_lbr_read_xsave() argument
913 struct x86_perf_task_context_arch_lbr_xsave *xsave = cpuc->lbr_xsave; in intel_pmu_arch_lbr_read_xsave()
916 intel_pmu_store_lbr(cpuc, NULL); in intel_pmu_arch_lbr_read_xsave()
921 intel_pmu_store_lbr(cpuc, xsave->lbr.entries); in intel_pmu_arch_lbr_read_xsave()
926 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_lbr_read() local
934 if (!cpuc->lbr_users || vlbr_exclude_host() || in intel_pmu_lbr_read()
935 cpuc->lbr_users == cpuc->lbr_pebs_users) in intel_pmu_lbr_read()
938 x86_pmu.lbr_read(cpuc); in intel_pmu_lbr_read()
940 intel_pmu_lbr_filter(cpuc); in intel_pmu_lbr_read()
1123 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) in intel_pmu_lbr_filter() argument
1126 int br_sel = cpuc->br_sel; in intel_pmu_lbr_filter()
1135 for (i = 0; i < cpuc->lbr_stack.nr; i++) { in intel_pmu_lbr_filter()
1137 from = cpuc->lbr_entries[i].from; in intel_pmu_lbr_filter()
1138 to = cpuc->lbr_entries[i].to; in intel_pmu_lbr_filter()
1139 type = cpuc->lbr_entries[i].type; in intel_pmu_lbr_filter()
1151 type = branch_type(from, to, cpuc->lbr_entries[i].abort); in intel_pmu_lbr_filter()
1153 if (cpuc->lbr_entries[i].in_tx) in intel_pmu_lbr_filter()
1161 cpuc->lbr_entries[i].from = 0; in intel_pmu_lbr_filter()
1166 cpuc->lbr_entries[i].type = common_branch_type(type); in intel_pmu_lbr_filter()
1173 for (i = 0; i < cpuc->lbr_stack.nr; ) { in intel_pmu_lbr_filter()
1174 if (!cpuc->lbr_entries[i].from) { in intel_pmu_lbr_filter()
1176 while (++j < cpuc->lbr_stack.nr) in intel_pmu_lbr_filter()
1177 cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j]; in intel_pmu_lbr_filter()
1178 cpuc->lbr_stack.nr--; in intel_pmu_lbr_filter()
1179 if (!cpuc->lbr_entries[i].from) in intel_pmu_lbr_filter()
1188 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); in intel_pmu_store_pebs_lbrs() local
1192 (cpuc->n_pebs == cpuc->n_large_pebs)) in intel_pmu_store_pebs_lbrs()
1193 cpuc->lbr_stack.hw_idx = -1ULL; in intel_pmu_store_pebs_lbrs()
1195 cpuc->lbr_stack.hw_idx = intel_pmu_lbr_tos(); in intel_pmu_store_pebs_lbrs()
1197 intel_pmu_store_lbr(cpuc, lbr); in intel_pmu_store_pebs_lbrs()
1198 intel_pmu_lbr_filter(cpuc); in intel_pmu_store_pebs_lbrs()