1 /**
2 * @file nmi_int.c
3 *
4 * @remark Copyright 2002 OProfile authors
5 * @remark Read the file COPYING
6 *
7 * @author John Levon <levon@movementarian.org>
8 *
9 * Modified for Xen: by Aravind Menon & Jose Renato Santos
10 * These modifications are:
11 * Copyright (C) 2005 Hewlett-Packard Co.
12 */
13
14 #include <xen/event.h>
15 #include <xen/types.h>
16 #include <xen/errno.h>
17 #include <xen/init.h>
18 #include <xen/string.h>
19 #include <xen/delay.h>
20 #include <xen/xenoprof.h>
21 #include <public/xen.h>
22 #include <asm/msr.h>
23 #include <asm/apic.h>
24 #include <asm/regs.h>
25 #include <asm/current.h>
26 #include <asm/nmi.h>
27
28 #include "op_counter.h"
29 #include "op_x86_model.h"
30
31 struct op_counter_config counter_config[OP_MAX_COUNTER];
32 struct op_ibs_config ibs_config;
33
34 struct op_x86_model_spec const *__read_mostly model;
35 static struct op_msrs cpu_msrs[NR_CPUS];
36 static unsigned long saved_lvtpc[NR_CPUS];
37
38 static char *cpu_type;
39
passive_domain_msr_op_checks(unsigned int msr,int * typep,int * indexp)40 static int passive_domain_msr_op_checks(unsigned int msr, int *typep, int *indexp)
41 {
42 struct vpmu_struct *vpmu = vcpu_vpmu(current);
43 if ( model == NULL )
44 return 0;
45 if ( model->is_arch_pmu_msr == NULL )
46 return 0;
47 if ( !model->is_arch_pmu_msr(msr, typep, indexp) )
48 return 0;
49
50 if ( !vpmu_is_set(vpmu, VPMU_PASSIVE_DOMAIN_ALLOCATED) )
51 if ( ! model->allocated_msr(current) )
52 return 0;
53 return 1;
54 }
55
passive_domain_do_rdmsr(unsigned int msr,uint64_t * msr_content)56 int passive_domain_do_rdmsr(unsigned int msr, uint64_t *msr_content)
57 {
58 int type, index;
59
60 if ( !passive_domain_msr_op_checks(msr, &type, &index))
61 return 0;
62
63 model->load_msr(current, type, index, msr_content);
64 return 1;
65 }
66
passive_domain_do_wrmsr(unsigned int msr,uint64_t msr_content)67 int passive_domain_do_wrmsr(unsigned int msr, uint64_t msr_content)
68 {
69 int type, index;
70
71 if ( !passive_domain_msr_op_checks(msr, &type, &index))
72 return 0;
73
74 model->save_msr(current, type, index, msr_content);
75 return 1;
76 }
77
passive_domain_destroy(struct vcpu * v)78 void passive_domain_destroy(struct vcpu *v)
79 {
80 struct vpmu_struct *vpmu = vcpu_vpmu(v);
81 if ( vpmu_is_set(vpmu, VPMU_PASSIVE_DOMAIN_ALLOCATED) )
82 model->free_msr(v);
83 }
84
nmi_callback(const struct cpu_user_regs * regs,int cpu)85 static int nmi_callback(const struct cpu_user_regs *regs, int cpu)
86 {
87 int xen_mode, ovf;
88
89 ovf = model->check_ctrs(cpu, &cpu_msrs[cpu], regs);
90 xen_mode = ring_0(regs);
91 if ( ovf && is_active(current->domain) && !xen_mode )
92 send_guest_vcpu_virq(current, VIRQ_XENOPROF);
93
94 if ( ovf == 2 )
95 current->nmi_pending = 1;
96 return 1;
97 }
98
99
nmi_cpu_save_registers(struct op_msrs * msrs)100 static void nmi_cpu_save_registers(struct op_msrs *msrs)
101 {
102 unsigned int const nr_ctrs = model->num_counters;
103 unsigned int const nr_ctrls = model->num_controls;
104 struct op_msr *counters = msrs->counters;
105 struct op_msr *controls = msrs->controls;
106 unsigned int i;
107
108 for (i = 0; i < nr_ctrs; ++i) {
109 rdmsrl(counters[i].addr, counters[i].value);
110 }
111
112 for (i = 0; i < nr_ctrls; ++i) {
113 rdmsrl(controls[i].addr, controls[i].value);
114 }
115 }
116
117
nmi_save_registers(void * dummy)118 static void nmi_save_registers(void * dummy)
119 {
120 int cpu = smp_processor_id();
121 struct op_msrs * msrs = &cpu_msrs[cpu];
122 model->fill_in_addresses(msrs);
123 nmi_cpu_save_registers(msrs);
124 }
125
126
free_msrs(void)127 static void free_msrs(void)
128 {
129 int i;
130 for (i = 0; i < nr_cpu_ids; ++i) {
131 xfree(cpu_msrs[i].counters);
132 cpu_msrs[i].counters = NULL;
133 xfree(cpu_msrs[i].controls);
134 cpu_msrs[i].controls = NULL;
135 }
136 }
137
138
allocate_msrs(void)139 static int allocate_msrs(void)
140 {
141 int success = 1;
142 size_t controls_size = sizeof(struct op_msr) * model->num_controls;
143 size_t counters_size = sizeof(struct op_msr) * model->num_counters;
144
145 int i;
146 for_each_online_cpu (i) {
147 cpu_msrs[i].counters = xmalloc_bytes(counters_size);
148 if (!cpu_msrs[i].counters) {
149 success = 0;
150 break;
151 }
152 cpu_msrs[i].controls = xmalloc_bytes(controls_size);
153 if (!cpu_msrs[i].controls) {
154 success = 0;
155 break;
156 }
157 }
158
159 if (!success)
160 free_msrs();
161
162 return success;
163 }
164
165
nmi_cpu_setup(void * dummy)166 static void nmi_cpu_setup(void * dummy)
167 {
168 int cpu = smp_processor_id();
169 struct op_msrs * msrs = &cpu_msrs[cpu];
170 model->setup_ctrs(msrs);
171 }
172
173
nmi_setup_events(void)174 int nmi_setup_events(void)
175 {
176 on_each_cpu(nmi_cpu_setup, NULL, 1);
177 return 0;
178 }
179
nmi_reserve_counters(void)180 int nmi_reserve_counters(void)
181 {
182 if (!allocate_msrs())
183 return -ENOMEM;
184
185 /* We walk a thin line between law and rape here.
186 * We need to be careful to install our NMI handler
187 * without actually triggering any NMIs as this will
188 * break the core code horrifically.
189 */
190 if (reserve_lapic_nmi() < 0) {
191 free_msrs();
192 return -EBUSY;
193 }
194 /* We need to serialize save and setup for HT because the subset
195 * of msrs are distinct for save and setup operations
196 */
197 on_each_cpu(nmi_save_registers, NULL, 1);
198 return 0;
199 }
200
nmi_enable_virq(void)201 int nmi_enable_virq(void)
202 {
203 set_nmi_callback(nmi_callback);
204 return 0;
205 }
206
207
nmi_disable_virq(void)208 void nmi_disable_virq(void)
209 {
210 unset_nmi_callback();
211 }
212
213
nmi_restore_registers(struct op_msrs * msrs)214 static void nmi_restore_registers(struct op_msrs * msrs)
215 {
216 unsigned int const nr_ctrs = model->num_counters;
217 unsigned int const nr_ctrls = model->num_controls;
218 struct op_msr * counters = msrs->counters;
219 struct op_msr * controls = msrs->controls;
220 unsigned int i;
221
222 for (i = 0; i < nr_ctrls; ++i) {
223 wrmsrl(controls[i].addr, controls[i].value);
224 }
225
226 for (i = 0; i < nr_ctrs; ++i) {
227 wrmsrl(counters[i].addr, counters[i].value);
228 }
229 }
230
231
nmi_cpu_shutdown(void * dummy)232 static void nmi_cpu_shutdown(void * dummy)
233 {
234 int cpu = smp_processor_id();
235 struct op_msrs * msrs = &cpu_msrs[cpu];
236 nmi_restore_registers(msrs);
237 }
238
239
nmi_release_counters(void)240 void nmi_release_counters(void)
241 {
242 on_each_cpu(nmi_cpu_shutdown, NULL, 1);
243 release_lapic_nmi();
244 free_msrs();
245 }
246
247
nmi_cpu_start(void * dummy)248 static void nmi_cpu_start(void * dummy)
249 {
250 int cpu = smp_processor_id();
251 struct op_msrs const * msrs = &cpu_msrs[cpu];
252 saved_lvtpc[cpu] = apic_read(APIC_LVTPC);
253 apic_write(APIC_LVTPC, APIC_DM_NMI);
254 model->start(msrs);
255 }
256
257
nmi_start(void)258 int nmi_start(void)
259 {
260 on_each_cpu(nmi_cpu_start, NULL, 1);
261 return 0;
262 }
263
264
nmi_cpu_stop(void * dummy)265 static void nmi_cpu_stop(void * dummy)
266 {
267 unsigned int v;
268 int cpu = smp_processor_id();
269 struct op_msrs const * msrs = &cpu_msrs[cpu];
270 model->stop(msrs);
271
272 /* restoring APIC_LVTPC can trigger an apic error because the delivery
273 * mode and vector nr combination can be illegal. That's by design: on
274 * power on apic lvt contain a zero vector nr which are legal only for
275 * NMI delivery mode. So inhibit apic err before restoring lvtpc
276 */
277 if ( (apic_read(APIC_LVTPC) & APIC_MODE_MASK) != APIC_DM_NMI
278 || (apic_read(APIC_LVTPC) & APIC_LVT_MASKED) )
279 {
280 printk("nmi_stop: APIC not good %ul\n", apic_read(APIC_LVTPC));
281 mdelay(5000);
282 }
283 v = apic_read(APIC_LVTERR);
284 apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
285 apic_write(APIC_LVTPC, saved_lvtpc[cpu]);
286 apic_write(APIC_LVTERR, v);
287 }
288
289
nmi_stop(void)290 void nmi_stop(void)
291 {
292 on_each_cpu(nmi_cpu_stop, NULL, 1);
293 }
294
295
p4_init(char ** cpu_type)296 static int __init p4_init(char ** cpu_type)
297 {
298 __u8 cpu_model = current_cpu_data.x86_model;
299
300 if ((cpu_model > 6) || (cpu_model == 5)) {
301 printk("xenoprof: Initialization failed. "
302 "Intel processor model %d for pentium 4 family is not "
303 "supported\n", cpu_model);
304 return 0;
305 }
306
307 switch (current_cpu_data.x86_num_siblings) {
308 case 1:
309 *cpu_type = "i386/p4";
310 model = &op_p4_spec;
311 return 1;
312
313 case 2:
314 *cpu_type = "i386/p4-ht";
315 model = &op_p4_ht2_spec;
316 return 1;
317 }
318
319 printk("Xenoprof ERROR: P4 HyperThreading detected with > 2 threads\n");
320
321 return 0;
322 }
323
324
325 static int force_arch_perfmon;
326
force_cpu_type(const char * str)327 static int force_cpu_type(const char *str)
328 {
329 if (!strcmp(str, "arch_perfmon")) {
330 force_arch_perfmon = 1;
331 printk(KERN_INFO "oprofile: forcing architectural perfmon\n");
332 }
333 else
334 return -EINVAL;
335
336 return 0;
337 }
338 custom_param("cpu_type", force_cpu_type);
339
ppro_init(char ** cpu_type)340 static int __init ppro_init(char ** cpu_type)
341 {
342 __u8 cpu_model = current_cpu_data.x86_model;
343
344 if (force_arch_perfmon && cpu_has_arch_perfmon)
345 return 0;
346
347 switch (cpu_model) {
348 case 14:
349 *cpu_type = "i386/core";
350 break;
351 case 15:
352 *cpu_type = "i386/core_2";
353 ppro_has_global_ctrl = 1;
354 break;
355 default:
356 /* Unknown */
357 return 0;
358 }
359
360 model = &op_ppro_spec;
361 return 1;
362 }
363
arch_perfmon_init(char ** cpu_type)364 static int __init arch_perfmon_init(char **cpu_type)
365 {
366 if (!cpu_has_arch_perfmon)
367 return 0;
368 *cpu_type = "i386/arch_perfmon";
369 model = &op_arch_perfmon_spec;
370 arch_perfmon_setup_counters();
371 ppro_has_global_ctrl = 1;
372 return 1;
373 }
374
nmi_init(void)375 static int __init nmi_init(void)
376 {
377 __u8 vendor = current_cpu_data.x86_vendor;
378 __u8 family = current_cpu_data.x86;
379 __u8 _model = current_cpu_data.x86_model;
380
381 if (!cpu_has_apic) {
382 printk("xenoprof: Initialization failed. No APIC\n");
383 return -ENODEV;
384 }
385
386 switch (vendor) {
387 case X86_VENDOR_AMD:
388 /* Needs to be at least an Athlon (or hammer in 32bit mode) */
389
390 switch (family) {
391 default:
392 printk("xenoprof: Initialization failed. "
393 "AMD processor family %d is not "
394 "supported\n", family);
395 return -ENODEV;
396 case 0xf:
397 model = &op_athlon_spec;
398 cpu_type = "x86-64/hammer";
399 break;
400 case 0x10:
401 model = &op_athlon_spec;
402 cpu_type = "x86-64/family10";
403 ibs_init();
404 break;
405 case 0x11:
406 model = &op_athlon_spec;
407 cpu_type = "x86-64/family11h";
408 break;
409 case 0x12:
410 model = &op_athlon_spec;
411 cpu_type = "x86-64/family12h";
412 break;
413 case 0x14:
414 model = &op_athlon_spec;
415 cpu_type = "x86-64/family14h";
416 break;
417 case 0x15:
418 model = &op_amd_fam15h_spec;
419 cpu_type = "x86-64/family15h";
420 break;
421 case 0x16:
422 model = &op_athlon_spec;
423 cpu_type = "x86-64/family16h";
424 break;
425 }
426 break;
427
428 case X86_VENDOR_INTEL:
429 switch (family) {
430 /* Pentium IV */
431 case 0xf:
432 p4_init(&cpu_type);
433 break;
434
435 /* A P6-class processor */
436 case 6:
437 ppro_init(&cpu_type);
438 break;
439
440 default:
441 break;
442 }
443 if (!cpu_type && !arch_perfmon_init(&cpu_type)) {
444 printk("xenoprof: Initialization failed. "
445 "Intel processor family %d model %d "
446 "is not supported\n", family, _model);
447 return -ENODEV;
448 }
449 break;
450
451 default:
452 printk("xenoprof: Initialization failed. "
453 "Unsupported processor. Unknown vendor %d\n",
454 vendor);
455 return -ENODEV;
456 }
457
458 return 0;
459 }
460
461 __initcall(nmi_init);
462
xenoprof_arch_init(int * num_events,char * _cpu_type)463 int xenoprof_arch_init(int *num_events, char *_cpu_type)
464 {
465 if (cpu_type == NULL)
466 return -ENODEV;
467 *num_events = model->num_counters;
468 strlcpy(_cpu_type, cpu_type, XENOPROF_CPU_TYPE_SIZE);
469 return 0;
470 }
471