Lines Matching refs:pt
181 struct intel_pt *pt; member
231 static void intel_pt_dump(struct intel_pt *pt __maybe_unused, in intel_pt_dump()
271 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf, in intel_pt_dump_event() argument
275 intel_pt_dump(pt, buf, len); in intel_pt_dump_event()
291 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_dump_sample() local
295 intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size); in intel_pt_dump_sample()
298 static bool intel_pt_log_events(struct intel_pt *pt, u64 tm) in intel_pt_log_events() argument
300 struct perf_time_interval *range = pt->synth_opts.ptime_range; in intel_pt_log_events()
301 int n = pt->synth_opts.range_num; in intel_pt_log_events()
303 if (pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS) in intel_pt_log_events()
306 if (pt->synth_opts.log_minus_flags & AUXTRACE_LOG_FLG_ALL_PERF_EVTS) in intel_pt_log_events()
353 struct intel_pt *pt = ptq->pt; in intel_pt_findnew_vmcs_info() local
355 if (!vmcs && !pt->dflt_tsc_offset) in intel_pt_findnew_vmcs_info()
358 return intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, pt->dflt_tsc_offset); in intel_pt_findnew_vmcs_info()
361 static void intel_pt_free_vmcs_info(struct intel_pt *pt) in intel_pt_free_vmcs_info() argument
366 n = rb_first(&pt->vmcs_info); in intel_pt_free_vmcs_info()
370 rb_erase(&v->rb_node, &pt->vmcs_info); in intel_pt_free_vmcs_info()
375 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a, in intel_pt_do_fix_overlap() argument
382 pt->have_tsc, &consecutive, in intel_pt_do_fix_overlap()
383 pt->synth_opts.vm_time_correlation); in intel_pt_do_fix_overlap()
391 if (pt->synth_opts.vm_time_correlation) in intel_pt_do_fix_overlap()
408 int fd = perf_data__fd(ptq->pt->session->data); in intel_pt_get_buffer()
415 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode; in intel_pt_get_buffer()
417 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer)) in intel_pt_get_buffer()
459 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_lookahead()
509 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_get_trace()
681 return ip >= ptq->pt->kernel_start ? in intel_pt_nr_cpumode()
696 struct machines *machines = &ptq->pt->session->machines; in intel_pt_get_guest()
752 struct machine *machine = ptq->pt->machine; in intel_pt_walk_next_insn()
773 if (ptq->pt->have_guest_sideband) { in intel_pt_walk_next_insn()
799 thread = ptq->pt->unknown_thread; in intel_pt_walk_next_insn()
925 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip, in intel_pt_match_pgd_ip() argument
933 list_for_each_entry(filt, &pt->filts.head, list) { in intel_pt_match_pgd_ip()
973 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
976 } else if (ip >= ptq->pt->kernel_start) { in __intel_pt_pgd_ip()
977 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
991 return intel_pt_match_pgd_ip(ptq->pt, ip, offset, in __intel_pt_pgd_ip()
1000 static bool intel_pt_get_config(struct intel_pt *pt, in intel_pt_get_config() argument
1003 if (attr->type == pt->pmu_type) { in intel_pt_get_config()
1012 static bool intel_pt_exclude_kernel(struct intel_pt *pt) in intel_pt_exclude_kernel() argument
1016 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_exclude_kernel()
1017 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) && in intel_pt_exclude_kernel()
1024 static bool intel_pt_return_compression(struct intel_pt *pt) in intel_pt_return_compression() argument
1029 if (!pt->noretcomp_bit) in intel_pt_return_compression()
1032 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_return_compression()
1033 if (intel_pt_get_config(pt, &evsel->core.attr, &config) && in intel_pt_return_compression()
1034 (config & pt->noretcomp_bit)) in intel_pt_return_compression()
1040 static bool intel_pt_branch_enable(struct intel_pt *pt) in intel_pt_branch_enable() argument
1045 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_branch_enable()
1046 if (intel_pt_get_config(pt, &evsel->core.attr, &config) && in intel_pt_branch_enable()
1054 static bool intel_pt_disabled_tnt(struct intel_pt *pt) in intel_pt_disabled_tnt() argument
1059 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_disabled_tnt()
1060 if (intel_pt_get_config(pt, &evsel->core.attr, &config) && in intel_pt_disabled_tnt()
1067 static unsigned int intel_pt_mtc_period(struct intel_pt *pt) in intel_pt_mtc_period() argument
1073 if (!pt->mtc_freq_bits) in intel_pt_mtc_period()
1076 for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++) in intel_pt_mtc_period()
1079 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_mtc_period()
1080 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) in intel_pt_mtc_period()
1081 return (config & pt->mtc_freq_bits) >> shift; in intel_pt_mtc_period()
1086 static bool intel_pt_timeless_decoding(struct intel_pt *pt) in intel_pt_timeless_decoding() argument
1092 if (!pt->tsc_bit || !pt->cap_user_time_zero || pt->synth_opts.timeless_decoding) in intel_pt_timeless_decoding()
1095 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_timeless_decoding()
1098 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) { in intel_pt_timeless_decoding()
1099 if (config & pt->tsc_bit) in intel_pt_timeless_decoding()
1108 static bool intel_pt_tracing_kernel(struct intel_pt *pt) in intel_pt_tracing_kernel() argument
1112 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_tracing_kernel()
1113 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) && in intel_pt_tracing_kernel()
1120 static bool intel_pt_have_tsc(struct intel_pt *pt) in intel_pt_have_tsc() argument
1126 if (!pt->tsc_bit) in intel_pt_have_tsc()
1129 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_have_tsc()
1130 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) { in intel_pt_have_tsc()
1131 if (config & pt->tsc_bit) in intel_pt_have_tsc()
1140 static bool intel_pt_have_mtc(struct intel_pt *pt) in intel_pt_have_mtc() argument
1145 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_have_mtc()
1146 if (intel_pt_get_config(pt, &evsel->core.attr, &config) && in intel_pt_have_mtc()
1147 (config & pt->mtc_bit)) in intel_pt_have_mtc()
1153 static bool intel_pt_sampling_mode(struct intel_pt *pt) in intel_pt_sampling_mode() argument
1157 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_sampling_mode()
1165 static u64 intel_pt_ctl(struct intel_pt *pt) in intel_pt_ctl() argument
1170 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_ctl()
1171 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) in intel_pt_ctl()
1177 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns) in intel_pt_ns_to_ticks() argument
1181 quot = ns / pt->tc.time_mult; in intel_pt_ns_to_ticks()
1182 rem = ns % pt->tc.time_mult; in intel_pt_ns_to_ticks()
1183 return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) / in intel_pt_ns_to_ticks()
1184 pt->tc.time_mult; in intel_pt_ns_to_ticks()
1187 static struct ip_callchain *intel_pt_alloc_chain(struct intel_pt *pt) in intel_pt_alloc_chain() argument
1192 sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64); in intel_pt_alloc_chain()
1196 static int intel_pt_callchain_init(struct intel_pt *pt) in intel_pt_callchain_init() argument
1200 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_callchain_init()
1205 pt->chain = intel_pt_alloc_chain(pt); in intel_pt_callchain_init()
1206 if (!pt->chain) in intel_pt_callchain_init()
1212 static void intel_pt_add_callchain(struct intel_pt *pt, in intel_pt_add_callchain() argument
1215 struct thread *thread = machine__findnew_thread(pt->machine, in intel_pt_add_callchain()
1219 thread_stack__sample_late(thread, sample->cpu, pt->chain, in intel_pt_add_callchain()
1220 pt->synth_opts.callchain_sz + 1, sample->ip, in intel_pt_add_callchain()
1221 pt->kernel_start); in intel_pt_add_callchain()
1223 sample->callchain = pt->chain; in intel_pt_add_callchain()
1234 static int intel_pt_br_stack_init(struct intel_pt *pt) in intel_pt_br_stack_init() argument
1238 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_br_stack_init()
1243 pt->br_stack = intel_pt_alloc_br_stack(pt->br_stack_sz); in intel_pt_br_stack_init()
1244 if (!pt->br_stack) in intel_pt_br_stack_init()
1250 static void intel_pt_add_br_stack(struct intel_pt *pt, in intel_pt_add_br_stack() argument
1253 struct thread *thread = machine__findnew_thread(pt->machine, in intel_pt_add_br_stack()
1257 thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack, in intel_pt_add_br_stack()
1258 pt->br_stack_sz, sample->ip, in intel_pt_add_br_stack()
1259 pt->kernel_start); in intel_pt_add_br_stack()
1261 sample->branch_stack = pt->br_stack; in intel_pt_add_br_stack()
1267 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, in intel_pt_alloc_queue() argument
1271 struct perf_env *env = pt->machine->env; in intel_pt_alloc_queue()
1278 if (pt->synth_opts.callchain) { in intel_pt_alloc_queue()
1279 ptq->chain = intel_pt_alloc_chain(pt); in intel_pt_alloc_queue()
1284 if (pt->synth_opts.last_branch || pt->synth_opts.other_events) { in intel_pt_alloc_queue()
1285 unsigned int entry_cnt = max(LBRS_MAX, pt->br_stack_sz); in intel_pt_alloc_queue()
1296 ptq->pt = pt; in intel_pt_alloc_queue()
1298 ptq->exclude_kernel = intel_pt_exclude_kernel(pt); in intel_pt_alloc_queue()
1309 params.return_compression = intel_pt_return_compression(pt); in intel_pt_alloc_queue()
1310 params.branch_enable = intel_pt_branch_enable(pt); in intel_pt_alloc_queue()
1311 params.ctl = intel_pt_ctl(pt); in intel_pt_alloc_queue()
1312 params.max_non_turbo_ratio = pt->max_non_turbo_ratio; in intel_pt_alloc_queue()
1313 params.mtc_period = intel_pt_mtc_period(pt); in intel_pt_alloc_queue()
1314 params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n; in intel_pt_alloc_queue()
1315 params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d; in intel_pt_alloc_queue()
1316 params.quick = pt->synth_opts.quick; in intel_pt_alloc_queue()
1317 params.vm_time_correlation = pt->synth_opts.vm_time_correlation; in intel_pt_alloc_queue()
1318 params.vm_tm_corr_dry_run = pt->synth_opts.vm_tm_corr_dry_run; in intel_pt_alloc_queue()
1319 params.first_timestamp = pt->first_timestamp; in intel_pt_alloc_queue()
1320 params.max_loops = pt->max_loops; in intel_pt_alloc_queue()
1323 if (params.branch_enable && intel_pt_disabled_tnt(pt) && !params.quick) in intel_pt_alloc_queue()
1326 if (pt->filts.cnt > 0) in intel_pt_alloc_queue()
1329 if (pt->synth_opts.instructions || pt->synth_opts.cycles) { in intel_pt_alloc_queue()
1330 if (pt->synth_opts.period) { in intel_pt_alloc_queue()
1331 switch (pt->synth_opts.period_type) { in intel_pt_alloc_queue()
1335 params.period = pt->synth_opts.period; in intel_pt_alloc_queue()
1339 params.period = pt->synth_opts.period; in intel_pt_alloc_queue()
1343 params.period = intel_pt_ns_to_ticks(pt, in intel_pt_alloc_queue()
1344 pt->synth_opts.period); in intel_pt_alloc_queue()
1390 static void intel_pt_first_timestamp(struct intel_pt *pt, u64 timestamp) in intel_pt_first_timestamp() argument
1394 pt->first_timestamp = timestamp; in intel_pt_first_timestamp()
1396 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_first_timestamp()
1397 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_first_timestamp()
1407 struct machines *machines = &ptq->pt->session->machines; in intel_pt_get_guest_from_sideband()
1454 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt, in intel_pt_set_pid_tid_cpu() argument
1459 if (queue->tid == -1 || pt->have_sched_switch) { in intel_pt_set_pid_tid_cpu()
1460 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu); in intel_pt_set_pid_tid_cpu()
1467 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid); in intel_pt_set_pid_tid_cpu()
1475 if (pt->have_guest_sideband && intel_pt_get_guest_from_sideband(ptq)) { in intel_pt_set_pid_tid_cpu()
1485 struct intel_pt *pt = ptq->pt; in intel_pt_sample_flags() local
1518 if (pt->cap_event_trace) { in intel_pt_sample_flags()
1530 static void intel_pt_setup_time_range(struct intel_pt *pt, in intel_pt_setup_time_range() argument
1533 if (!pt->range_cnt) in intel_pt_setup_time_range()
1536 ptq->sel_timestamp = pt->time_ranges[0].start; in intel_pt_setup_time_range()
1542 ptq->sel_timestamp = pt->time_ranges[0].end; in intel_pt_setup_time_range()
1547 static int intel_pt_setup_queue(struct intel_pt *pt, in intel_pt_setup_queue() argument
1557 ptq = intel_pt_alloc_queue(pt, queue_nr); in intel_pt_setup_queue()
1568 if (pt->sampling_mode && !pt->snapshot_mode && in intel_pt_setup_queue()
1569 pt->timeless_decoding) in intel_pt_setup_queue()
1572 ptq->sync_switch = pt->sync_switch; in intel_pt_setup_queue()
1574 intel_pt_setup_time_range(pt, ptq); in intel_pt_setup_queue()
1583 if (pt->timeless_decoding) in intel_pt_setup_queue()
1620 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1629 static int intel_pt_setup_queues(struct intel_pt *pt) in intel_pt_setup_queues() argument
1634 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_setup_queues()
1635 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i); in intel_pt_setup_queues()
1642 static inline bool intel_pt_skip_event(struct intel_pt *pt) in intel_pt_skip_event() argument
1644 return pt->synth_opts.initial_skip && in intel_pt_skip_event()
1645 pt->num_events++ < pt->synth_opts.initial_skip; in intel_pt_skip_event()
1653 static inline bool intel_pt_skip_cbr_event(struct intel_pt *pt) in intel_pt_skip_cbr_event() argument
1655 return pt->synth_opts.initial_skip && in intel_pt_skip_cbr_event()
1656 pt->num_events + 4 < pt->synth_opts.initial_skip; in intel_pt_skip_cbr_event()
1669 if (ptq->pt->have_guest_sideband) { in intel_pt_prep_a_sample()
1684 static void intel_pt_prep_b_sample(struct intel_pt *pt, in intel_pt_prep_b_sample() argument
1691 if (!pt->timeless_decoding) in intel_pt_prep_b_sample()
1692 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_prep_b_sample()
1710 static inline int intel_pt_opt_inject(struct intel_pt *pt, in intel_pt_opt_inject() argument
1714 if (!pt->synth_opts.inject) in intel_pt_opt_inject()
1720 static int intel_pt_deliver_synth_event(struct intel_pt *pt, in intel_pt_deliver_synth_event() argument
1726 ret = intel_pt_opt_inject(pt, event, sample, type); in intel_pt_deliver_synth_event()
1730 ret = perf_session__deliver_synth_event(pt->session, event, sample); in intel_pt_deliver_synth_event()
1739 struct intel_pt *pt = ptq->pt; in intel_pt_synth_branch_sample() local
1748 if (pt->branches_filter && !(pt->branches_filter & ptq->flags)) in intel_pt_synth_branch_sample()
1751 if (intel_pt_skip_event(pt)) in intel_pt_synth_branch_sample()
1754 intel_pt_prep_b_sample(pt, ptq, event, &sample); in intel_pt_synth_branch_sample()
1756 sample.id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1757 sample.stream_id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1763 if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) { in intel_pt_synth_branch_sample()
1783 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_branch_sample()
1784 pt->branches_sample_type); in intel_pt_synth_branch_sample()
1787 static void intel_pt_prep_sample(struct intel_pt *pt, in intel_pt_prep_sample() argument
1792 intel_pt_prep_b_sample(pt, ptq, event, sample); in intel_pt_prep_sample()
1794 if (pt->synth_opts.callchain) { in intel_pt_prep_sample()
1796 pt->synth_opts.callchain_sz + 1, in intel_pt_prep_sample()
1797 sample->ip, pt->kernel_start); in intel_pt_prep_sample()
1801 if (pt->synth_opts.last_branch) { in intel_pt_prep_sample()
1803 pt->br_stack_sz); in intel_pt_prep_sample()
1810 struct intel_pt *pt = ptq->pt; in intel_pt_synth_instruction_sample() local
1814 if (intel_pt_skip_event(pt)) in intel_pt_synth_instruction_sample()
1817 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_instruction_sample()
1819 sample.id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1820 sample.stream_id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1821 if (pt->synth_opts.quick) in intel_pt_synth_instruction_sample()
1836 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_instruction_sample()
1837 pt->instructions_sample_type); in intel_pt_synth_instruction_sample()
1842 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cycle_sample() local
1850 if (!period || intel_pt_skip_event(pt)) in intel_pt_synth_cycle_sample()
1853 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_cycle_sample()
1855 sample.id = ptq->pt->cycles_id; in intel_pt_synth_cycle_sample()
1856 sample.stream_id = ptq->pt->cycles_id; in intel_pt_synth_cycle_sample()
1864 return intel_pt_deliver_synth_event(pt, event, &sample, pt->cycles_sample_type); in intel_pt_synth_cycle_sample()
1869 struct intel_pt *pt = ptq->pt; in intel_pt_synth_transaction_sample() local
1873 if (intel_pt_skip_event(pt)) in intel_pt_synth_transaction_sample()
1876 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_transaction_sample()
1878 sample.id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1879 sample.stream_id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1881 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_transaction_sample()
1882 pt->transactions_sample_type); in intel_pt_synth_transaction_sample()
1885 static void intel_pt_prep_p_sample(struct intel_pt *pt, in intel_pt_prep_p_sample() argument
1890 intel_pt_prep_sample(pt, ptq, event, sample); in intel_pt_prep_p_sample()
1902 struct intel_pt *pt = ptq->pt; in intel_pt_synth_ptwrite_sample() local
1907 if (intel_pt_skip_event(pt)) in intel_pt_synth_ptwrite_sample()
1910 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_ptwrite_sample()
1912 sample.id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1913 sample.stream_id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1922 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_ptwrite_sample()
1923 pt->ptwrites_sample_type); in intel_pt_synth_ptwrite_sample()
1928 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cbr_sample() local
1934 if (intel_pt_skip_cbr_event(pt)) in intel_pt_synth_cbr_sample()
1939 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_cbr_sample()
1941 sample.id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1942 sample.stream_id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1944 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16); in intel_pt_synth_cbr_sample()
1946 raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz); in intel_pt_synth_cbr_sample()
1952 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_cbr_sample()
1953 pt->pwr_events_sample_type); in intel_pt_synth_cbr_sample()
1958 struct intel_pt *pt = ptq->pt; in intel_pt_synth_psb_sample() local
1963 if (intel_pt_skip_event(pt)) in intel_pt_synth_psb_sample()
1966 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_psb_sample()
1968 sample.id = ptq->pt->psb_id; in intel_pt_synth_psb_sample()
1969 sample.stream_id = ptq->pt->psb_id; in intel_pt_synth_psb_sample()
1978 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_psb_sample()
1979 pt->pwr_events_sample_type); in intel_pt_synth_psb_sample()
1984 struct intel_pt *pt = ptq->pt; in intel_pt_synth_mwait_sample() local
1989 if (intel_pt_skip_event(pt)) in intel_pt_synth_mwait_sample()
1992 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_mwait_sample()
1994 sample.id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1995 sample.stream_id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
2003 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_mwait_sample()
2004 pt->pwr_events_sample_type); in intel_pt_synth_mwait_sample()
2009 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwre_sample() local
2014 if (intel_pt_skip_event(pt)) in intel_pt_synth_pwre_sample()
2017 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwre_sample()
2019 sample.id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
2020 sample.stream_id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
2028 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_pwre_sample()
2029 pt->pwr_events_sample_type); in intel_pt_synth_pwre_sample()
2034 struct intel_pt *pt = ptq->pt; in intel_pt_synth_exstop_sample() local
2039 if (intel_pt_skip_event(pt)) in intel_pt_synth_exstop_sample()
2042 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_exstop_sample()
2044 sample.id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
2045 sample.stream_id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
2053 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_exstop_sample()
2054 pt->pwr_events_sample_type); in intel_pt_synth_exstop_sample()
2059 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwrx_sample() local
2064 if (intel_pt_skip_event(pt)) in intel_pt_synth_pwrx_sample()
2067 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwrx_sample()
2069 sample.id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
2070 sample.stream_id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
2078 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_pwrx_sample()
2079 pt->pwr_events_sample_type); in intel_pt_synth_pwrx_sample()
2214 struct intel_pt *pt = ptq->pt; in intel_pt_do_synth_pebs_sample() local
2219 if (intel_pt_skip_event(pt)) in intel_pt_do_synth_pebs_sample()
2249 else if (!pt->timeless_decoding) in intel_pt_do_synth_pebs_sample()
2252 sample.time = tsc_to_perf_time(timestamp, &pt->tc); in intel_pt_do_synth_pebs_sample()
2256 pt->synth_opts.callchain) { in intel_pt_do_synth_pebs_sample()
2258 pt->synth_opts.callchain_sz, sample.ip, in intel_pt_do_synth_pebs_sample()
2259 pt->kernel_start); in intel_pt_do_synth_pebs_sample()
2284 } else if (pt->synth_opts.last_branch) { in intel_pt_do_synth_pebs_sample()
2287 pt->br_stack_sz); in intel_pt_do_synth_pebs_sample()
2338 return intel_pt_deliver_synth_event(pt, event, &sample, sample_type); in intel_pt_do_synth_pebs_sample()
2343 struct intel_pt *pt = ptq->pt; in intel_pt_synth_single_pebs_sample() local
2344 struct evsel *evsel = pt->pebs_evsel; in intel_pt_synth_single_pebs_sample()
2354 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pebs_sample() local
2359 if (!pt->single_pebs) in intel_pt_synth_pebs_sample()
2367 if (!pt->single_pebs) in intel_pt_synth_pebs_sample()
2382 struct intel_pt *pt = ptq->pt; in intel_pt_synth_events_sample() local
2391 if (intel_pt_skip_event(pt)) in intel_pt_synth_events_sample()
2394 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_events_sample()
2396 sample.id = ptq->pt->evt_id; in intel_pt_synth_events_sample()
2397 sample.stream_id = ptq->pt->evt_id; in intel_pt_synth_events_sample()
2415 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_events_sample()
2416 pt->evt_sample_type); in intel_pt_synth_events_sample()
2421 struct intel_pt *pt = ptq->pt; in intel_pt_synth_iflag_chg_sample() local
2426 if (intel_pt_skip_event(pt)) in intel_pt_synth_iflag_chg_sample()
2429 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_iflag_chg_sample()
2431 sample.id = ptq->pt->iflag_chg_id; in intel_pt_synth_iflag_chg_sample()
2432 sample.stream_id = ptq->pt->iflag_chg_id; in intel_pt_synth_iflag_chg_sample()
2448 return intel_pt_deliver_synth_event(pt, event, &sample, in intel_pt_synth_iflag_chg_sample()
2449 pt->iflag_chg_sample_type); in intel_pt_synth_iflag_chg_sample()
2452 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu, in intel_pt_synth_error() argument
2456 bool dump_log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR; in intel_pt_synth_error()
2457 bool log_on_stdout = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT; in intel_pt_synth_error()
2462 if (pt->synth_opts.error_minus_flags) { in intel_pt_synth_error()
2464 pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_OVERFLOW) in intel_pt_synth_error()
2467 pt->synth_opts.error_minus_flags & AUXTRACE_ERR_FLG_DATA_LOST) in intel_pt_synth_error()
2487 err = perf_session__deliver_synth_event(pt->session, &event, NULL); in intel_pt_synth_error()
2498 struct intel_pt *pt = ptq->pt; in intel_ptq_synth_error() local
2505 tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc); in intel_ptq_synth_error()
2507 if (pt->have_guest_sideband && state->from_nr) { in intel_ptq_synth_error()
2514 return intel_pt_synth_error(pt, state->err, ptq->cpu, pid, tid, in intel_ptq_synth_error()
2518 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq) in intel_pt_next_tid() argument
2529 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid); in intel_pt_next_tid()
2531 queue = &pt->queues.queue_array[ptq->queue_nr]; in intel_pt_next_tid()
2532 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_next_tid()
2541 struct intel_pt *pt = ptq->pt; in intel_pt_is_switch_ip() local
2543 return ip == pt->switch_ip && in intel_pt_is_switch_ip()
2555 struct intel_pt *pt = ptq->pt; in intel_pt_sample() local
2563 if (pt->synth_opts.approx_ipc) { in intel_pt_sample()
2581 if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) { in intel_pt_sample()
2587 if (pt->synth_opts.intr_events) { in intel_pt_sample()
2600 if (pt->sample_pwr_events) { in intel_pt_sample()
2636 if (pt->sample_instructions) { in intel_pt_sample()
2641 if (pt->sample_cycles) { in intel_pt_sample()
2648 if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) { in intel_pt_sample()
2654 if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) { in intel_pt_sample()
2663 if (pt->use_thread_stack) { in intel_pt_sample()
2666 state->trace_nr, pt->callstack, in intel_pt_sample()
2667 pt->br_stack_sz_plus, in intel_pt_sample()
2668 pt->mispred_all); in intel_pt_sample()
2673 if (pt->sample_branches) { in intel_pt_sample()
2707 err = intel_pt_next_tid(pt, ptq); in intel_pt_sample()
2721 state->to_ip == pt->ptss_ip && in intel_pt_sample()
2729 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip) in intel_pt_switch_ip() argument
2731 struct machine *machine = pt->machine; in intel_pt_switch_ip()
2763 if (pt->have_sched_switch == 1) in intel_pt_switch_ip()
2781 static void intel_pt_enable_sync_switch(struct intel_pt *pt) in intel_pt_enable_sync_switch() argument
2785 if (pt->sync_switch_not_supported) in intel_pt_enable_sync_switch()
2788 pt->sync_switch = true; in intel_pt_enable_sync_switch()
2790 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_enable_sync_switch()
2791 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_enable_sync_switch()
2799 static void intel_pt_disable_sync_switch(struct intel_pt *pt) in intel_pt_disable_sync_switch() argument
2803 pt->sync_switch = false; in intel_pt_disable_sync_switch()
2805 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_disable_sync_switch()
2806 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_disable_sync_switch()
2811 intel_pt_next_tid(pt, ptq); in intel_pt_disable_sync_switch()
2822 struct intel_pt *pt = ptq->pt; in intel_pt_next_time() local
2827 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end; in intel_pt_next_time()
2829 } else if (ptq->sel_idx + 1 < pt->range_cnt) { in intel_pt_next_time()
2833 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start; in intel_pt_next_time()
2861 intel_pt_next_tid(ptq->pt, ptq); in intel_pt_time_filter()
2891 struct intel_pt *pt = ptq->pt; in intel_pt_run_decoder() local
2895 if (!pt->kernel_start) { in intel_pt_run_decoder()
2896 pt->kernel_start = machine__kernel_start(pt->machine); in intel_pt_run_decoder()
2897 if (pt->per_cpu_mmaps && in intel_pt_run_decoder()
2898 (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) && in intel_pt_run_decoder()
2899 !pt->timeless_decoding && intel_pt_tracing_kernel(pt) && in intel_pt_run_decoder()
2900 !pt->sampling_mode && !pt->synth_opts.vm_time_correlation) { in intel_pt_run_decoder()
2901 pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip); in intel_pt_run_decoder()
2902 if (pt->switch_ip) { in intel_pt_run_decoder()
2904 pt->switch_ip, pt->ptss_ip); in intel_pt_run_decoder()
2905 intel_pt_enable_sync_switch(pt); in intel_pt_run_decoder()
2922 state->from_ip >= pt->kernel_start) { in intel_pt_run_decoder()
2924 intel_pt_next_tid(pt, ptq); in intel_pt_run_decoder()
2927 if (pt->synth_opts.errors) { in intel_pt_run_decoder()
2940 if (pt->est_tsc && in intel_pt_run_decoder()
2941 (state->from_ip >= pt->kernel_start || !state->from_ip) && in intel_pt_run_decoder()
2942 state->to_ip && state->to_ip < pt->kernel_start) { in intel_pt_run_decoder()
2964 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) { in intel_pt_run_decoder()
2972 static inline int intel_pt_update_queues(struct intel_pt *pt) in intel_pt_update_queues() argument
2974 if (pt->queues.new_data) { in intel_pt_update_queues()
2975 pt->queues.new_data = false; in intel_pt_update_queues()
2976 return intel_pt_setup_queues(pt); in intel_pt_update_queues()
2981 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp) in intel_pt_process_queues() argument
2991 if (!pt->heap.heap_cnt) in intel_pt_process_queues()
2994 if (pt->heap.heap_array[0].ordinal >= timestamp) in intel_pt_process_queues()
2997 queue_nr = pt->heap.heap_array[0].queue_nr; in intel_pt_process_queues()
2998 queue = &pt->queues.queue_array[queue_nr]; in intel_pt_process_queues()
3002 queue_nr, pt->heap.heap_array[0].ordinal, in intel_pt_process_queues()
3005 auxtrace_heap__pop(&pt->heap); in intel_pt_process_queues()
3007 if (pt->heap.heap_cnt) { in intel_pt_process_queues()
3008 ts = pt->heap.heap_array[0].ordinal + 1; in intel_pt_process_queues()
3015 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_process_queues()
3020 auxtrace_heap__add(&pt->heap, queue_nr, ts); in intel_pt_process_queues()
3025 ret = auxtrace_heap__add(&pt->heap, queue_nr, ts); in intel_pt_process_queues()
3036 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid, in intel_pt_process_timeless_queues() argument
3039 struct auxtrace_queues *queues = &pt->queues; in intel_pt_process_timeless_queues()
3044 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_process_timeless_queues()
3049 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_process_timeless_queues()
3060 struct machine *m = ptq->pt->machine; in intel_pt_sample_set_pid_tid_cpu()
3084 static int intel_pt_process_timeless_sample(struct intel_pt *pt, in intel_pt_process_timeless_sample() argument
3091 queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session); in intel_pt_process_timeless_sample()
3106 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample) in intel_pt_lost() argument
3108 return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu, in intel_pt_lost()
3113 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu) in intel_pt_cpu_to_ptq() argument
3117 if (cpu < 0 || !pt->queues.nr_queues) in intel_pt_cpu_to_ptq()
3120 if ((unsigned)cpu >= pt->queues.nr_queues) in intel_pt_cpu_to_ptq()
3121 i = pt->queues.nr_queues - 1; in intel_pt_cpu_to_ptq()
3125 if (pt->queues.queue_array[i].cpu == cpu) in intel_pt_cpu_to_ptq()
3126 return pt->queues.queue_array[i].priv; in intel_pt_cpu_to_ptq()
3129 if (pt->queues.queue_array[--i].cpu == cpu) in intel_pt_cpu_to_ptq()
3130 return pt->queues.queue_array[i].priv; in intel_pt_cpu_to_ptq()
3133 for (; j < pt->queues.nr_queues; j++) { in intel_pt_cpu_to_ptq()
3134 if (pt->queues.queue_array[j].cpu == cpu) in intel_pt_cpu_to_ptq()
3135 return pt->queues.queue_array[j].priv; in intel_pt_cpu_to_ptq()
3141 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid, in intel_pt_sync_switch() argument
3147 if (!pt->sync_switch) in intel_pt_sync_switch()
3150 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_sync_switch()
3165 &pt->tc); in intel_pt_sync_switch()
3166 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr, in intel_pt_sync_switch()
3187 static int intel_pt_process_switch(struct intel_pt *pt, in intel_pt_process_switch() argument
3192 struct evsel *evsel = evlist__id2evsel(pt->session->evlist, sample->id); in intel_pt_process_switch()
3194 if (evsel != pt->switch_evsel) in intel_pt_process_switch()
3202 &pt->tc)); in intel_pt_process_switch()
3204 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time); in intel_pt_process_switch()
3208 return machine__set_current_tid(pt->machine, cpu, -1, tid); in intel_pt_process_switch()
3212 static int intel_pt_context_switch_in(struct intel_pt *pt, in intel_pt_context_switch_in() argument
3219 if (pt->sync_switch) { in intel_pt_context_switch_in()
3222 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_context_switch_in()
3244 if (machine__get_current_tid(pt->machine, cpu) == tid) in intel_pt_context_switch_in()
3247 return machine__set_current_tid(pt->machine, cpu, pid, tid); in intel_pt_context_switch_in()
3250 static int intel_pt_guest_context_switch(struct intel_pt *pt, in intel_pt_guest_context_switch() argument
3255 struct machines *machines = &pt->session->machines; in intel_pt_guest_context_switch()
3258 pt->have_guest_sideband = true; in intel_pt_guest_context_switch()
3264 pt->sync_switch_not_supported = true; in intel_pt_guest_context_switch()
3265 if (pt->sync_switch) in intel_pt_guest_context_switch()
3266 intel_pt_disable_sync_switch(pt); in intel_pt_guest_context_switch()
3277 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event, in intel_pt_context_switch() argument
3285 return intel_pt_guest_context_switch(pt, event, sample); in intel_pt_context_switch()
3289 if (pt->have_sched_switch == 3) { in intel_pt_context_switch()
3291 return intel_pt_context_switch_in(pt, sample); in intel_pt_context_switch()
3308 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time); in intel_pt_context_switch()
3312 return machine__set_current_tid(pt->machine, cpu, pid, tid); in intel_pt_context_switch()
3315 static int intel_pt_process_itrace_start(struct intel_pt *pt, in intel_pt_process_itrace_start() argument
3319 if (!pt->per_cpu_mmaps) in intel_pt_process_itrace_start()
3325 perf_time_to_tsc(sample->time, &pt->tc)); in intel_pt_process_itrace_start()
3327 return machine__set_current_tid(pt->machine, sample->cpu, in intel_pt_process_itrace_start()
3332 static int intel_pt_process_aux_output_hw_id(struct intel_pt *pt, in intel_pt_process_aux_output_hw_id() argument
3341 queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session); in intel_pt_process_aux_output_hw_id()
3342 evsel = evlist__id2evsel_strict(pt->session->evlist, sample->id); in intel_pt_process_aux_output_hw_id()
3368 static int intel_pt_text_poke(struct intel_pt *pt, union perf_event *event) in intel_pt_text_poke() argument
3374 struct thread *thread = pt->unknown_thread; in intel_pt_text_poke()
3376 struct machine *machine = pt->machine; in intel_pt_text_poke()
3422 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_process_event() local
3436 timestamp = perf_time_to_tsc(sample->time, &pt->tc); in intel_pt_process_event()
3440 if (timestamp || pt->timeless_decoding) { in intel_pt_process_event()
3441 err = intel_pt_update_queues(pt); in intel_pt_process_event()
3446 if (pt->timeless_decoding) { in intel_pt_process_event()
3447 if (pt->sampling_mode) { in intel_pt_process_event()
3449 err = intel_pt_process_timeless_sample(pt, in intel_pt_process_event()
3452 err = intel_pt_process_timeless_queues(pt, in intel_pt_process_event()
3457 if (!pt->first_timestamp) in intel_pt_process_event()
3458 intel_pt_first_timestamp(pt, timestamp); in intel_pt_process_event()
3459 err = intel_pt_process_queues(pt, timestamp); in intel_pt_process_event()
3465 if (pt->synth_opts.add_callchain && !sample->callchain) in intel_pt_process_event()
3466 intel_pt_add_callchain(pt, sample); in intel_pt_process_event()
3467 if (pt->synth_opts.add_last_branch && !sample->branch_stack) in intel_pt_process_event()
3468 intel_pt_add_br_stack(pt, sample); in intel_pt_process_event()
3473 pt->synth_opts.errors) { in intel_pt_process_event()
3474 err = intel_pt_lost(pt, sample); in intel_pt_process_event()
3480 if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE) in intel_pt_process_event()
3481 err = intel_pt_process_switch(pt, sample); in intel_pt_process_event()
3485 err = intel_pt_process_itrace_start(pt, event, sample); in intel_pt_process_event()
3487 err = intel_pt_process_aux_output_hw_id(pt, event, sample); in intel_pt_process_event()
3490 err = intel_pt_context_switch(pt, event, sample); in intel_pt_process_event()
3493 err = intel_pt_text_poke(pt, event); in intel_pt_process_event()
3495 if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) { in intel_pt_process_event()
3506 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_flush() local
3516 ret = intel_pt_update_queues(pt); in intel_pt_flush()
3520 if (pt->timeless_decoding) in intel_pt_flush()
3521 return intel_pt_process_timeless_queues(pt, -1, in intel_pt_flush()
3524 return intel_pt_process_queues(pt, MAX_TIMESTAMP); in intel_pt_flush()
3529 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_free_events() local
3531 struct auxtrace_queues *queues = &pt->queues; in intel_pt_free_events()
3544 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_free() local
3547 auxtrace_heap__free(&pt->heap); in intel_pt_free()
3550 intel_pt_free_vmcs_info(pt); in intel_pt_free()
3551 thread__put(pt->unknown_thread); in intel_pt_free()
3552 addr_filters__exit(&pt->filts); in intel_pt_free()
3553 zfree(&pt->chain); in intel_pt_free()
3554 zfree(&pt->filter); in intel_pt_free()
3555 zfree(&pt->time_ranges); in intel_pt_free()
3556 free(pt); in intel_pt_free()
3562 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_evsel_is_auxtrace() local
3565 return evsel->core.attr.type == pt->pmu_type; in intel_pt_evsel_is_auxtrace()
3572 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_process_auxtrace_event() local
3575 if (!pt->data_queued) { in intel_pt_process_auxtrace_event()
3589 err = auxtrace_queues__add_event(&pt->queues, session, event, in intel_pt_process_auxtrace_event()
3597 intel_pt_dump_event(pt, buffer->data, in intel_pt_process_auxtrace_event()
3611 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_queue_data() local
3616 return auxtrace_queues__add_event(&pt->queues, session, event, in intel_pt_queue_data()
3621 timestamp = perf_time_to_tsc(sample->time, &pt->tc); in intel_pt_queue_data()
3625 return auxtrace_queues__add_sample(&pt->queues, session, sample, in intel_pt_queue_data()
3682 static struct evsel *intel_pt_evsel(struct intel_pt *pt, in intel_pt_evsel() argument
3688 if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids) in intel_pt_evsel()
3695 static int intel_pt_synth_events(struct intel_pt *pt, in intel_pt_synth_events() argument
3699 struct evsel *evsel = intel_pt_evsel(pt, evlist); in intel_pt_synth_events()
3715 if (pt->timeless_decoding) in intel_pt_synth_events()
3719 if (!pt->per_cpu_mmaps) in intel_pt_synth_events()
3733 if (pt->synth_opts.branches) { in intel_pt_synth_events()
3740 pt->sample_branches = true; in intel_pt_synth_events()
3741 pt->branches_sample_type = attr.sample_type; in intel_pt_synth_events()
3742 pt->branches_id = id; in intel_pt_synth_events()
3747 if (pt->synth_opts.callchain) in intel_pt_synth_events()
3749 if (pt->synth_opts.last_branch) { in intel_pt_synth_events()
3759 if (pt->synth_opts.instructions) { in intel_pt_synth_events()
3761 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS) in intel_pt_synth_events()
3763 intel_pt_ns_to_ticks(pt, pt->synth_opts.period); in intel_pt_synth_events()
3765 attr.sample_period = pt->synth_opts.period; in intel_pt_synth_events()
3769 pt->sample_instructions = true; in intel_pt_synth_events()
3770 pt->instructions_sample_type = attr.sample_type; in intel_pt_synth_events()
3771 pt->instructions_id = id; in intel_pt_synth_events()
3775 if (pt->synth_opts.cycles) { in intel_pt_synth_events()
3777 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS) in intel_pt_synth_events()
3779 intel_pt_ns_to_ticks(pt, pt->synth_opts.period); in intel_pt_synth_events()
3781 attr.sample_period = pt->synth_opts.period; in intel_pt_synth_events()
3785 pt->sample_cycles = true; in intel_pt_synth_events()
3786 pt->cycles_sample_type = attr.sample_type; in intel_pt_synth_events()
3787 pt->cycles_id = id; in intel_pt_synth_events()
3794 if (pt->synth_opts.transactions) { in intel_pt_synth_events()
3799 pt->sample_transactions = true; in intel_pt_synth_events()
3800 pt->transactions_sample_type = attr.sample_type; in intel_pt_synth_events()
3801 pt->transactions_id = id; in intel_pt_synth_events()
3809 if (pt->synth_opts.ptwrites) { in intel_pt_synth_events()
3814 pt->sample_ptwrites = true; in intel_pt_synth_events()
3815 pt->ptwrites_sample_type = attr.sample_type; in intel_pt_synth_events()
3816 pt->ptwrites_id = id; in intel_pt_synth_events()
3821 if (pt->synth_opts.pwr_events) { in intel_pt_synth_events()
3822 pt->sample_pwr_events = true; in intel_pt_synth_events()
3823 pt->pwr_events_sample_type = attr.sample_type; in intel_pt_synth_events()
3829 pt->cbr_id = id; in intel_pt_synth_events()
3837 pt->psb_id = id; in intel_pt_synth_events()
3842 if (pt->synth_opts.pwr_events && (evsel->core.attr.config & INTEL_PT_CFG_PWR_EVT_EN)) { in intel_pt_synth_events()
3847 pt->mwait_id = id; in intel_pt_synth_events()
3855 pt->pwre_id = id; in intel_pt_synth_events()
3863 pt->exstop_id = id; in intel_pt_synth_events()
3871 pt->pwrx_id = id; in intel_pt_synth_events()
3876 if (pt->synth_opts.intr_events && (evsel->core.attr.config & INTEL_PT_CFG_EVT_EN)) { in intel_pt_synth_events()
3881 pt->evt_sample_type = attr.sample_type; in intel_pt_synth_events()
3882 pt->evt_id = id; in intel_pt_synth_events()
3887 if (pt->synth_opts.intr_events && pt->cap_event_trace) { in intel_pt_synth_events()
3892 pt->iflag_chg_sample_type = attr.sample_type; in intel_pt_synth_events()
3893 pt->iflag_chg_id = id; in intel_pt_synth_events()
3901 static void intel_pt_setup_pebs_events(struct intel_pt *pt) in intel_pt_setup_pebs_events() argument
3905 if (!pt->synth_opts.other_events) in intel_pt_setup_pebs_events()
3908 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_setup_pebs_events()
3910 if (pt->single_pebs) { in intel_pt_setup_pebs_events()
3911 pt->single_pebs = false; in intel_pt_setup_pebs_events()
3914 pt->single_pebs = true; in intel_pt_setup_pebs_events()
3915 pt->sample_pebs = true; in intel_pt_setup_pebs_events()
3916 pt->pebs_evsel = evsel; in intel_pt_setup_pebs_events()
3949 struct intel_pt *pt = data; in intel_pt_perf_config() local
3952 pt->mispred_all = perf_config_bool(var, value); in intel_pt_perf_config()
3955 perf_config_int(&pt->max_loops, var, value); in intel_pt_perf_config()
3961 static u64 intel_pt_tsc_start(u64 ns, struct intel_pt *pt) in intel_pt_tsc_start() argument
3965 tsc = perf_time_to_tsc(ns, &pt->tc); in intel_pt_tsc_start()
3968 tm = tsc_to_perf_time(tsc, &pt->tc); in intel_pt_tsc_start()
3975 tm = tsc_to_perf_time(++tsc, &pt->tc); in intel_pt_tsc_start()
3981 static u64 intel_pt_tsc_end(u64 ns, struct intel_pt *pt) in intel_pt_tsc_end() argument
3985 tsc = perf_time_to_tsc(ns, &pt->tc); in intel_pt_tsc_end()
3988 tm = tsc_to_perf_time(tsc, &pt->tc); in intel_pt_tsc_end()
3995 tm = tsc_to_perf_time(--tsc, &pt->tc); in intel_pt_tsc_end()
4000 static int intel_pt_setup_time_ranges(struct intel_pt *pt, in intel_pt_setup_time_ranges() argument
4007 if (!n || !p || pt->timeless_decoding) in intel_pt_setup_time_ranges()
4010 pt->time_ranges = calloc(n, sizeof(struct range)); in intel_pt_setup_time_ranges()
4011 if (!pt->time_ranges) in intel_pt_setup_time_ranges()
4014 pt->range_cnt = n; in intel_pt_setup_time_ranges()
4019 struct range *r = &pt->time_ranges[i]; in intel_pt_setup_time_ranges()
4027 r->start = ts ? intel_pt_tsc_start(ts, pt) : 0; in intel_pt_setup_time_ranges()
4028 r->end = te ? intel_pt_tsc_end(te, pt) : 0; in intel_pt_setup_time_ranges()
4039 static int intel_pt_parse_vm_tm_corr_arg(struct intel_pt *pt, char **args) in intel_pt_parse_vm_tm_corr_arg() argument
4056 pt->dflt_tsc_offset = tsc_offset; in intel_pt_parse_vm_tm_corr_arg()
4067 vmcs_info = intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, tsc_offset); in intel_pt_parse_vm_tm_corr_arg()
4079 static int intel_pt_parse_vm_tm_corr_args(struct intel_pt *pt) in intel_pt_parse_vm_tm_corr_args() argument
4081 char *args = pt->synth_opts.vm_tm_corr_args; in intel_pt_parse_vm_tm_corr_args()
4088 ret = intel_pt_parse_vm_tm_corr_arg(pt, &args); in intel_pt_parse_vm_tm_corr_args()
4153 struct intel_pt *pt; in intel_pt_process_auxtrace_info() local
4162 pt = zalloc(sizeof(struct intel_pt)); in intel_pt_process_auxtrace_info()
4163 if (!pt) in intel_pt_process_auxtrace_info()
4166 pt->vmcs_info = RB_ROOT; in intel_pt_process_auxtrace_info()
4168 addr_filters__init(&pt->filts); in intel_pt_process_auxtrace_info()
4170 err = perf_config(intel_pt_perf_config, pt); in intel_pt_process_auxtrace_info()
4174 err = auxtrace_queues__init(&pt->queues); in intel_pt_process_auxtrace_info()
4179 pt->synth_opts = *session->itrace_synth_opts; in intel_pt_process_auxtrace_info()
4183 itrace_synth_opts__set_default(&pt->synth_opts, opts->default_no_sample); in intel_pt_process_auxtrace_info()
4185 pt->synth_opts.branches = false; in intel_pt_process_auxtrace_info()
4186 pt->synth_opts.callchain = true; in intel_pt_process_auxtrace_info()
4187 pt->synth_opts.add_callchain = true; in intel_pt_process_auxtrace_info()
4189 pt->synth_opts.thread_stack = opts->thread_stack; in intel_pt_process_auxtrace_info()
4192 if (!(pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT)) in intel_pt_process_auxtrace_info()
4195 pt->session = session; in intel_pt_process_auxtrace_info()
4196 pt->machine = &session->machines.host; /* No kvm support */ in intel_pt_process_auxtrace_info()
4197 pt->auxtrace_type = auxtrace_info->type; in intel_pt_process_auxtrace_info()
4198 pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE]; in intel_pt_process_auxtrace_info()
4199 pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT]; in intel_pt_process_auxtrace_info()
4200 pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT]; in intel_pt_process_auxtrace_info()
4201 pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO]; in intel_pt_process_auxtrace_info()
4202 pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO]; in intel_pt_process_auxtrace_info()
4203 pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT]; in intel_pt_process_auxtrace_info()
4204 pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT]; in intel_pt_process_auxtrace_info()
4205 pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH]; in intel_pt_process_auxtrace_info()
4206 pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE]; in intel_pt_process_auxtrace_info()
4207 pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS]; in intel_pt_process_auxtrace_info()
4212 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT]; in intel_pt_process_auxtrace_info()
4213 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS]; in intel_pt_process_auxtrace_info()
4214 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N]; in intel_pt_process_auxtrace_info()
4215 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D]; in intel_pt_process_auxtrace_info()
4216 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT]; in intel_pt_process_auxtrace_info()
4222 pt->max_non_turbo_ratio = in intel_pt_process_auxtrace_info()
4249 pt->filter = memdup(filter, len); in intel_pt_process_auxtrace_info()
4250 if (!pt->filter) { in intel_pt_process_auxtrace_info()
4255 mem_bswap_64(pt->filter, len); in intel_pt_process_auxtrace_info()
4256 if (pt->filter[len - 1]) { in intel_pt_process_auxtrace_info()
4261 err = addr_filters__parse_bare_filter(&pt->filts, in intel_pt_process_auxtrace_info()
4266 intel_pt_print_info_str("Filter string", pt->filter); in intel_pt_process_auxtrace_info()
4270 pt->cap_event_trace = *info++; in intel_pt_process_auxtrace_info()
4273 pt->cap_event_trace); in intel_pt_process_auxtrace_info()
4276 pt->timeless_decoding = intel_pt_timeless_decoding(pt); in intel_pt_process_auxtrace_info()
4277 if (pt->timeless_decoding && !pt->tc.time_mult) in intel_pt_process_auxtrace_info()
4278 pt->tc.time_mult = 1; in intel_pt_process_auxtrace_info()
4279 pt->have_tsc = intel_pt_have_tsc(pt); in intel_pt_process_auxtrace_info()
4280 pt->sampling_mode = intel_pt_sampling_mode(pt); in intel_pt_process_auxtrace_info()
4281 pt->est_tsc = !pt->timeless_decoding; in intel_pt_process_auxtrace_info()
4283 if (pt->synth_opts.vm_time_correlation) { in intel_pt_process_auxtrace_info()
4284 if (pt->timeless_decoding) { in intel_pt_process_auxtrace_info()
4295 if (!intel_pt_have_mtc(pt)) { in intel_pt_process_auxtrace_info()
4300 err = intel_pt_parse_vm_tm_corr_args(pt); in intel_pt_process_auxtrace_info()
4305 pt->unknown_thread = thread__new(999999999, 999999999); in intel_pt_process_auxtrace_info()
4306 if (!pt->unknown_thread) { in intel_pt_process_auxtrace_info()
4317 INIT_LIST_HEAD(&pt->unknown_thread->node); in intel_pt_process_auxtrace_info()
4319 err = thread__set_comm(pt->unknown_thread, "unknown", 0); in intel_pt_process_auxtrace_info()
4322 if (thread__init_maps(pt->unknown_thread, pt->machine)) { in intel_pt_process_auxtrace_info()
4327 pt->auxtrace.process_event = intel_pt_process_event; in intel_pt_process_auxtrace_info()
4328 pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event; in intel_pt_process_auxtrace_info()
4329 pt->auxtrace.queue_data = intel_pt_queue_data; in intel_pt_process_auxtrace_info()
4330 pt->auxtrace.dump_auxtrace_sample = intel_pt_dump_sample; in intel_pt_process_auxtrace_info()
4331 pt->auxtrace.flush_events = intel_pt_flush; in intel_pt_process_auxtrace_info()
4332 pt->auxtrace.free_events = intel_pt_free_events; in intel_pt_process_auxtrace_info()
4333 pt->auxtrace.free = intel_pt_free; in intel_pt_process_auxtrace_info()
4334 pt->auxtrace.evsel_is_auxtrace = intel_pt_evsel_is_auxtrace; in intel_pt_process_auxtrace_info()
4335 session->auxtrace = &pt->auxtrace; in intel_pt_process_auxtrace_info()
4340 if (pt->have_sched_switch == 1) { in intel_pt_process_auxtrace_info()
4341 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist); in intel_pt_process_auxtrace_info()
4342 if (!pt->switch_evsel) { in intel_pt_process_auxtrace_info()
4347 } else if (pt->have_sched_switch == 2 && in intel_pt_process_auxtrace_info()
4354 if (pt->synth_opts.log) { in intel_pt_process_auxtrace_info()
4355 bool log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR; in intel_pt_process_auxtrace_info()
4356 unsigned int log_on_error_size = pt->synth_opts.log_on_error_size; in intel_pt_process_auxtrace_info()
4362 if (pt->tc.time_mult) { in intel_pt_process_auxtrace_info()
4363 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000); in intel_pt_process_auxtrace_info()
4365 if (!pt->max_non_turbo_ratio) in intel_pt_process_auxtrace_info()
4366 pt->max_non_turbo_ratio = in intel_pt_process_auxtrace_info()
4370 pt->max_non_turbo_ratio); in intel_pt_process_auxtrace_info()
4371 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000; in intel_pt_process_auxtrace_info()
4374 err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts); in intel_pt_process_auxtrace_info()
4378 if (pt->synth_opts.calls) in intel_pt_process_auxtrace_info()
4379 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | in intel_pt_process_auxtrace_info()
4381 if (pt->synth_opts.returns) in intel_pt_process_auxtrace_info()
4382 pt->branches_filter |= PERF_IP_FLAG_RETURN | in intel_pt_process_auxtrace_info()
4385 if ((pt->synth_opts.callchain || pt->synth_opts.add_callchain) && in intel_pt_process_auxtrace_info()
4390 pt->synth_opts.callchain = false; in intel_pt_process_auxtrace_info()
4391 pt->synth_opts.add_callchain = false; in intel_pt_process_auxtrace_info()
4395 if (pt->synth_opts.add_callchain) { in intel_pt_process_auxtrace_info()
4396 err = intel_pt_callchain_init(pt); in intel_pt_process_auxtrace_info()
4401 if (pt->synth_opts.last_branch || pt->synth_opts.add_last_branch) { in intel_pt_process_auxtrace_info()
4402 pt->br_stack_sz = pt->synth_opts.last_branch_sz; in intel_pt_process_auxtrace_info()
4403 pt->br_stack_sz_plus = pt->br_stack_sz; in intel_pt_process_auxtrace_info()
4406 if (pt->synth_opts.add_last_branch) { in intel_pt_process_auxtrace_info()
4407 err = intel_pt_br_stack_init(pt); in intel_pt_process_auxtrace_info()
4417 if (intel_pt_tracing_kernel(pt)) in intel_pt_process_auxtrace_info()
4418 pt->br_stack_sz_plus += 1024; in intel_pt_process_auxtrace_info()
4420 pt->br_stack_sz_plus += 1; in intel_pt_process_auxtrace_info()
4423 pt->use_thread_stack = pt->synth_opts.callchain || in intel_pt_process_auxtrace_info()
4424 pt->synth_opts.add_callchain || in intel_pt_process_auxtrace_info()
4425 pt->synth_opts.thread_stack || in intel_pt_process_auxtrace_info()
4426 pt->synth_opts.last_branch || in intel_pt_process_auxtrace_info()
4427 pt->synth_opts.add_last_branch; in intel_pt_process_auxtrace_info()
4429 pt->callstack = pt->synth_opts.callchain || in intel_pt_process_auxtrace_info()
4430 pt->synth_opts.add_callchain || in intel_pt_process_auxtrace_info()
4431 pt->synth_opts.thread_stack; in intel_pt_process_auxtrace_info()
4433 err = intel_pt_synth_events(pt, session); in intel_pt_process_auxtrace_info()
4437 intel_pt_setup_pebs_events(pt); in intel_pt_process_auxtrace_info()
4445 if (pt->sampling_mode || list_empty(&session->auxtrace_index)) in intel_pt_process_auxtrace_info()
4448 err = auxtrace_queues__process_index(&pt->queues, session); in intel_pt_process_auxtrace_info()
4452 if (pt->queues.populated) in intel_pt_process_auxtrace_info()
4453 pt->data_queued = true; in intel_pt_process_auxtrace_info()
4455 if (pt->timeless_decoding) in intel_pt_process_auxtrace_info()
4461 zfree(&pt->chain); in intel_pt_process_auxtrace_info()
4462 thread__zput(pt->unknown_thread); in intel_pt_process_auxtrace_info()
4465 auxtrace_queues__free(&pt->queues); in intel_pt_process_auxtrace_info()
4468 addr_filters__exit(&pt->filts); in intel_pt_process_auxtrace_info()
4469 zfree(&pt->filter); in intel_pt_process_auxtrace_info()
4470 zfree(&pt->time_ranges); in intel_pt_process_auxtrace_info()
4471 free(pt); in intel_pt_process_auxtrace_info()