1 /**
2 * @file op_model_athlon.h
3 * athlon / K7 model-specific MSR operations
4 *
5 * @remark Copyright 2002 OProfile authors
6 * @remark Read the file COPYING
7 *
8 * @author John Levon
9 * @author Philippe Elie
10 * @author Graydon Hoare
11 */
12
13 #include <xen/types.h>
14 #include <asm/msr.h>
15 #include <asm/io.h>
16 #include <asm/apic.h>
17 #include <asm/processor.h>
18 #include <xen/xenoprof.h>
19 #include <asm/regs.h>
20 #include <asm/current.h>
21 #include <asm/hvm/support.h>
22 #include <xen/pci_regs.h>
23 #include <xen/pci_ids.h>
24
25 #include "op_x86_model.h"
26 #include "op_counter.h"
27
28 #define K7_NUM_COUNTERS 4
29 #define K7_NUM_CONTROLS 4
30
31 #define FAM15H_NUM_COUNTERS 6
32 #define FAM15H_NUM_CONTROLS 6
33
34 #define MAX_COUNTERS FAM15H_NUM_COUNTERS
35
36 #define CTR_READ(msr_content,msrs,c) do {rdmsrl(msrs->counters[(c)].addr, (msr_content));} while (0)
37 #define CTR_WRITE(l,msrs,c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1);} while (0)
38 #define CTR_OVERFLOWED(n) (!((n) & (1ULL<<31)))
39
40 #define CTRL_READ(msr_content,msrs,c) do {rdmsrl(msrs->controls[(c)].addr, (msr_content));} while (0)
41 #define CTRL_WRITE(msr_content,msrs,c) do {wrmsrl(msrs->controls[(c)].addr, (msr_content));} while (0)
42 #define CTRL_SET_ACTIVE(n) (n |= (1ULL<<22))
43 #define CTRL_SET_INACTIVE(n) (n &= ~(1ULL<<22))
44 #define CTRL_CLEAR(val) (val &= (1ULL<<21))
45 #define CTRL_SET_ENABLE(val) (val |= 1ULL<<20)
46 #define CTRL_SET_USR(val,u) (val |= ((u & 1) << 16))
47 #define CTRL_SET_KERN(val,k) (val |= ((k & 1) << 17))
48 #define CTRL_SET_UM(val, m) (val |= ((m & 0xff) << 8))
49 #define CTRL_SET_EVENT(val, e) (val |= (((e >> 8) & 0xf) | (e & 0xff)))
50 #define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 0x1ULL) << 41))
51 #define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 0x1ULL) << 40))
52
53 static unsigned long reset_value[MAX_COUNTERS];
54
55 extern char svm_stgi_label[];
56
57 u32 ibs_caps = 0;
58 static u64 ibs_op_ctl;
59
60 /* IBS cpuid feature detection */
61 #define IBS_CPUID_FEATURES 0x8000001b
62
63 /* IBS MSRs */
64 #define MSR_AMD64_IBSFETCHCTL 0xc0011030
65 #define MSR_AMD64_IBSFETCHLINAD 0xc0011031
66 #define MSR_AMD64_IBSFETCHPHYSAD 0xc0011032
67 #define MSR_AMD64_IBSOPCTL 0xc0011033
68 #define MSR_AMD64_IBSOPRIP 0xc0011034
69 #define MSR_AMD64_IBSOPDATA 0xc0011035
70 #define MSR_AMD64_IBSOPDATA2 0xc0011036
71 #define MSR_AMD64_IBSOPDATA3 0xc0011037
72 #define MSR_AMD64_IBSDCLINAD 0xc0011038
73 #define MSR_AMD64_IBSDCPHYSAD 0xc0011039
74 #define MSR_AMD64_IBSCTL 0xc001103a
75
76 /*
77 * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
78 * bit 0 is used to indicate the existence of IBS.
79 */
80 #define IBS_CAPS_AVAIL (1LL<<0)
81 #define IBS_CAPS_RDWROPCNT (1LL<<3)
82 #define IBS_CAPS_OPCNT (1LL<<4)
83
84 /* IBS randomization macros */
85 #define IBS_RANDOM_BITS 12
86 #define IBS_RANDOM_MASK ((1ULL << IBS_RANDOM_BITS) - 1)
87 #define IBS_RANDOM_MAXCNT_OFFSET (1ULL << (IBS_RANDOM_BITS - 5))
88
89 /* IbsFetchCtl bits/masks */
90 #define IBS_FETCH_RAND_EN (1ULL<<57)
91 #define IBS_FETCH_VAL (1ULL<<49)
92 #define IBS_FETCH_ENABLE (1ULL<<48)
93 #define IBS_FETCH_CNT 0xFFFF0000ULL
94 #define IBS_FETCH_MAX_CNT 0x0000FFFFULL
95
96 /* IbsOpCtl bits */
97 #define IBS_OP_CNT_CTL (1ULL<<19)
98 #define IBS_OP_VAL (1ULL<<18)
99 #define IBS_OP_ENABLE (1ULL<<17)
100 #define IBS_OP_MAX_CNT 0x0000FFFFULL
101
102 /* IBS sample identifier */
103 #define IBS_FETCH_CODE 13
104 #define IBS_OP_CODE 14
105
106 #define clamp(val, min, max) ({ \
107 typeof(val) __val = (val); \
108 typeof(min) __min = (min); \
109 typeof(max) __max = (max); \
110 (void) (&__val == &__min); \
111 (void) (&__val == &__max); \
112 __val = __val < __min ? __min: __val; \
113 __val > __max ? __max: __val; })
114
115 /*
116 * 16-bit Linear Feedback Shift Register (LFSR)
117 */
lfsr_random(void)118 static unsigned int lfsr_random(void)
119 {
120 static unsigned int lfsr_value = 0xF00D;
121 unsigned int bit;
122
123 /* Compute next bit to shift in */
124 bit = ((lfsr_value >> 0) ^
125 (lfsr_value >> 2) ^
126 (lfsr_value >> 3) ^
127 (lfsr_value >> 5)) & 0x0001;
128
129 /* Advance to next register value */
130 lfsr_value = (lfsr_value >> 1) | (bit << 15);
131
132 return lfsr_value;
133 }
134
135 /*
136 * IBS software randomization
137 *
138 * The IBS periodic op counter is randomized in software. The lower 12
139 * bits of the 20 bit counter are randomized. IbsOpCurCnt is
140 * initialized with a 12 bit random value.
141 */
op_amd_randomize_ibs_op(u64 val)142 static inline u64 op_amd_randomize_ibs_op(u64 val)
143 {
144 unsigned int random = lfsr_random();
145
146 if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
147 /*
148 * Work around if the hw can not write to IbsOpCurCnt
149 *
150 * Randomize the lower 8 bits of the 16 bit
151 * IbsOpMaxCnt [15:0] value in the range of -128 to
152 * +127 by adding/subtracting an offset to the
153 * maximum count (IbsOpMaxCnt).
154 *
155 * To avoid over or underflows and protect upper bits
156 * starting at bit 16, the initial value for
157 * IbsOpMaxCnt must fit in the range from 0x0081 to
158 * 0xff80.
159 */
160 val += (s8)(random >> 4);
161 else
162 val |= (u64)(random & IBS_RANDOM_MASK) << 32;
163
164 return val;
165 }
166
athlon_fill_in_addresses(struct op_msrs * const msrs)167 static void athlon_fill_in_addresses(struct op_msrs * const msrs)
168 {
169 msrs->counters[0].addr = MSR_K7_PERFCTR0;
170 msrs->counters[1].addr = MSR_K7_PERFCTR1;
171 msrs->counters[2].addr = MSR_K7_PERFCTR2;
172 msrs->counters[3].addr = MSR_K7_PERFCTR3;
173
174 msrs->controls[0].addr = MSR_K7_EVNTSEL0;
175 msrs->controls[1].addr = MSR_K7_EVNTSEL1;
176 msrs->controls[2].addr = MSR_K7_EVNTSEL2;
177 msrs->controls[3].addr = MSR_K7_EVNTSEL3;
178 }
179
fam15h_fill_in_addresses(struct op_msrs * const msrs)180 static void fam15h_fill_in_addresses(struct op_msrs * const msrs)
181 {
182 msrs->counters[0].addr = MSR_AMD_FAM15H_PERFCTR0;
183 msrs->counters[1].addr = MSR_AMD_FAM15H_PERFCTR1;
184 msrs->counters[2].addr = MSR_AMD_FAM15H_PERFCTR2;
185 msrs->counters[3].addr = MSR_AMD_FAM15H_PERFCTR3;
186 msrs->counters[4].addr = MSR_AMD_FAM15H_PERFCTR4;
187 msrs->counters[5].addr = MSR_AMD_FAM15H_PERFCTR5;
188
189 msrs->controls[0].addr = MSR_AMD_FAM15H_EVNTSEL0;
190 msrs->controls[1].addr = MSR_AMD_FAM15H_EVNTSEL1;
191 msrs->controls[2].addr = MSR_AMD_FAM15H_EVNTSEL2;
192 msrs->controls[3].addr = MSR_AMD_FAM15H_EVNTSEL3;
193 msrs->controls[4].addr = MSR_AMD_FAM15H_EVNTSEL4;
194 msrs->controls[5].addr = MSR_AMD_FAM15H_EVNTSEL5;
195 }
196
athlon_setup_ctrs(struct op_msrs const * const msrs)197 static void athlon_setup_ctrs(struct op_msrs const * const msrs)
198 {
199 uint64_t msr_content;
200 int i;
201 unsigned int const nr_ctrs = model->num_counters;
202 unsigned int const nr_ctrls = model->num_controls;
203
204 /* clear all counters */
205 for (i = 0 ; i < nr_ctrls; ++i) {
206 CTRL_READ(msr_content, msrs, i);
207 CTRL_CLEAR(msr_content);
208 CTRL_WRITE(msr_content, msrs, i);
209 }
210
211 /* avoid a false detection of ctr overflows in NMI handler */
212 for (i = 0; i < nr_ctrs; ++i) {
213 CTR_WRITE(1, msrs, i);
214 }
215
216 /* enable active counters */
217 for (i = 0; i < nr_ctrs; ++i) {
218 if (counter_config[i].enabled) {
219 reset_value[i] = counter_config[i].count;
220
221 CTR_WRITE(counter_config[i].count, msrs, i);
222
223 CTRL_READ(msr_content, msrs, i);
224 CTRL_CLEAR(msr_content);
225 CTRL_SET_ENABLE(msr_content);
226 CTRL_SET_USR(msr_content, counter_config[i].user);
227 CTRL_SET_KERN(msr_content, counter_config[i].kernel);
228 CTRL_SET_UM(msr_content, counter_config[i].unit_mask);
229 CTRL_SET_EVENT(msr_content, counter_config[i].event);
230 CTRL_SET_HOST_ONLY(msr_content, 0);
231 CTRL_SET_GUEST_ONLY(msr_content, 0);
232 CTRL_WRITE(msr_content, msrs, i);
233 } else {
234 reset_value[i] = 0;
235 }
236 }
237 }
238
239 static inline void
ibs_log_event(u64 data,struct cpu_user_regs const * const regs,int mode)240 ibs_log_event(u64 data, struct cpu_user_regs const * const regs, int mode)
241 {
242 struct vcpu *v = current;
243 u32 temp = 0;
244
245 temp = data & 0xFFFFFFFF;
246 xenoprof_log_event(v, regs, temp, mode, 0);
247
248 temp = (data >> 32) & 0xFFFFFFFF;
249 xenoprof_log_event(v, regs, temp, mode, 0);
250
251 }
252
handle_ibs(int mode,struct cpu_user_regs const * const regs)253 static inline int handle_ibs(int mode, struct cpu_user_regs const * const regs)
254 {
255 u64 val, ctl;
256 struct vcpu *v = current;
257
258 if (!ibs_caps)
259 return 1;
260
261 if (ibs_config.fetch_enabled) {
262 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
263 if (ctl & IBS_FETCH_VAL) {
264 rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
265 xenoprof_log_event(v, regs, IBS_FETCH_CODE, mode, 0);
266 xenoprof_log_event(v, regs, val, mode, 0);
267
268 ibs_log_event(val, regs, mode);
269 ibs_log_event(ctl, regs, mode);
270
271 rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
272 ibs_log_event(val, regs, mode);
273
274 /* reenable the IRQ */
275 ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
276 ctl |= IBS_FETCH_ENABLE;
277 wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
278 }
279 }
280
281 if (ibs_config.op_enabled) {
282 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
283 if (ctl & IBS_OP_VAL) {
284
285 rdmsrl(MSR_AMD64_IBSOPRIP, val);
286 xenoprof_log_event(v, regs, IBS_OP_CODE, mode, 0);
287 xenoprof_log_event(v, regs, val, mode, 0);
288
289 ibs_log_event(val, regs, mode);
290
291 rdmsrl(MSR_AMD64_IBSOPDATA, val);
292 ibs_log_event(val, regs, mode);
293 rdmsrl(MSR_AMD64_IBSOPDATA2, val);
294 ibs_log_event(val, regs, mode);
295 rdmsrl(MSR_AMD64_IBSOPDATA3, val);
296 ibs_log_event(val, regs, mode);
297 rdmsrl(MSR_AMD64_IBSDCLINAD, val);
298 ibs_log_event(val, regs, mode);
299 rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
300 ibs_log_event(val, regs, mode);
301
302 /* reenable the IRQ */
303 ctl = op_amd_randomize_ibs_op(ibs_op_ctl);
304 wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
305 }
306 }
307
308 return 1;
309 }
310
athlon_check_ctrs(unsigned int const cpu,struct op_msrs const * const msrs,struct cpu_user_regs const * const regs)311 static int athlon_check_ctrs(unsigned int const cpu,
312 struct op_msrs const * const msrs,
313 struct cpu_user_regs const * const regs)
314
315 {
316 uint64_t msr_content;
317 int i;
318 int ovf = 0;
319 unsigned long eip = regs->rip;
320 int mode = 0;
321 struct vcpu *v = current;
322 struct cpu_user_regs *guest_regs = guest_cpu_user_regs();
323 unsigned int const nr_ctrs = model->num_counters;
324
325 if (!guest_mode(regs) &&
326 (eip == (unsigned long)svm_stgi_label)) {
327 /* SVM guest was running when NMI occurred */
328 ASSERT(is_hvm_vcpu(v));
329 eip = guest_regs->rip;
330 mode = xenoprofile_get_mode(v, guest_regs);
331 } else
332 mode = xenoprofile_get_mode(v, regs);
333
334 for (i = 0 ; i < nr_ctrs; ++i) {
335 CTR_READ(msr_content, msrs, i);
336 if (CTR_OVERFLOWED(msr_content)) {
337 xenoprof_log_event(current, regs, eip, mode, i);
338 CTR_WRITE(reset_value[i], msrs, i);
339 ovf = 1;
340 }
341 }
342
343 ovf = handle_ibs(mode, regs);
344 /* See op_model_ppro.c */
345 return ovf;
346 }
347
start_ibs(void)348 static inline void start_ibs(void)
349 {
350 u64 val = 0;
351
352 if (!ibs_caps)
353 return;
354
355 if (ibs_config.fetch_enabled) {
356 val = (ibs_config.max_cnt_fetch >> 4) & IBS_FETCH_MAX_CNT;
357 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
358 val |= IBS_FETCH_ENABLE;
359 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
360 }
361
362 if (ibs_config.op_enabled) {
363 ibs_op_ctl = ibs_config.max_cnt_op >> 4;
364 if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
365 /*
366 * IbsOpCurCnt not supported. See
367 * op_amd_randomize_ibs_op() for details.
368 */
369 ibs_op_ctl = clamp((unsigned long long)ibs_op_ctl,
370 0x0081ULL, 0xFF80ULL);
371 } else {
372 /*
373 * The start value is randomized with a
374 * positive offset, we need to compensate it
375 * with the half of the randomized range. Also
376 * avoid underflows.
377 */
378 ibs_op_ctl = min(ibs_op_ctl + IBS_RANDOM_MAXCNT_OFFSET,
379 IBS_OP_MAX_CNT);
380 }
381 if (ibs_caps & IBS_CAPS_OPCNT && ibs_config.dispatched_ops)
382 ibs_op_ctl |= IBS_OP_CNT_CTL;
383 ibs_op_ctl |= IBS_OP_ENABLE;
384 val = op_amd_randomize_ibs_op(ibs_op_ctl);
385 wrmsrl(MSR_AMD64_IBSOPCTL, val);
386 }
387 }
388
athlon_start(struct op_msrs const * const msrs)389 static void athlon_start(struct op_msrs const * const msrs)
390 {
391 uint64_t msr_content;
392 int i;
393 unsigned int const nr_ctrs = model->num_counters;
394 for (i = 0 ; i < nr_ctrs ; ++i) {
395 if (reset_value[i]) {
396 CTRL_READ(msr_content, msrs, i);
397 CTRL_SET_ACTIVE(msr_content);
398 CTRL_WRITE(msr_content, msrs, i);
399 }
400 }
401 start_ibs();
402 }
403
stop_ibs(void)404 static void stop_ibs(void)
405 {
406 if (!ibs_caps)
407 return;
408
409 if (ibs_config.fetch_enabled)
410 /* clear max count and enable */
411 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
412
413 if (ibs_config.op_enabled)
414 /* clear max count and enable */
415 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
416 }
417
athlon_stop(struct op_msrs const * const msrs)418 static void athlon_stop(struct op_msrs const * const msrs)
419 {
420 uint64_t msr_content;
421 int i;
422 unsigned int const nr_ctrs = model->num_counters;
423
424 /* Subtle: stop on all counters to avoid race with
425 * setting our pm callback */
426 for (i = 0 ; i < nr_ctrs ; ++i) {
427 CTRL_READ(msr_content, msrs, i);
428 CTRL_SET_INACTIVE(msr_content);
429 CTRL_WRITE(msr_content, msrs, i);
430 }
431
432 stop_ibs();
433 }
434
435 #define IBSCTL_LVTOFFSETVAL (1 << 8)
436 #define APIC_EILVT_MSG_NMI 0x4
437 #define APIC_EILVT_LVTOFF_IBS 1
438 #define APIC_EILVTn(n) (0x500 + 0x10 * n)
init_ibs_nmi_per_cpu(void * arg)439 static inline void __init init_ibs_nmi_per_cpu(void *arg)
440 {
441 unsigned long reg;
442
443 reg = (APIC_EILVT_LVTOFF_IBS << 4) + APIC_EILVTn(0);
444 apic_write(reg, APIC_EILVT_MSG_NMI << 8);
445 }
446
447 #define PCI_DEVICE_ID_AMD_10H_NB_MISC 0x1203
448 #define IBSCTL 0x1cc
init_ibs_nmi(void)449 static int __init init_ibs_nmi(void)
450 {
451 int bus, dev, func;
452 u32 id, value;
453 u16 vendor_id, dev_id;
454 int nodes;
455
456 /* per CPU setup */
457 on_each_cpu(init_ibs_nmi_per_cpu, NULL, 1);
458
459 nodes = 0;
460 for (bus = 0; bus < 256; bus++) {
461 for (dev = 0; dev < 32; dev++) {
462 for (func = 0; func < 8; func++) {
463 id = pci_conf_read32(0, bus, dev, func, PCI_VENDOR_ID);
464
465 vendor_id = id & 0xffff;
466 dev_id = (id >> 16) & 0xffff;
467
468 if ((vendor_id == PCI_VENDOR_ID_AMD) &&
469 (dev_id == PCI_DEVICE_ID_AMD_10H_NB_MISC)) {
470
471 pci_conf_write32(0, bus, dev, func, IBSCTL,
472 IBSCTL_LVTOFFSETVAL | APIC_EILVT_LVTOFF_IBS);
473
474 value = pci_conf_read32(0, bus, dev, func, IBSCTL);
475
476 if (value != (IBSCTL_LVTOFFSETVAL |
477 APIC_EILVT_LVTOFF_IBS)) {
478 printk("Xenoprofile: Failed to setup IBS LVT offset, "
479 "IBSCTL = %#x\n", value);
480 return 1;
481 }
482 nodes++;
483 }
484 }
485 }
486 }
487
488 if (!nodes) {
489 printk("Xenoprofile: No CPU node configured for IBS\n");
490 return 1;
491 }
492
493 return 0;
494 }
495
get_ibs_caps(void)496 static void __init get_ibs_caps(void)
497 {
498 if (!boot_cpu_has(X86_FEATURE_IBS))
499 return;
500
501 /* check IBS cpuid feature flags */
502 if (current_cpu_data.extended_cpuid_level >= IBS_CPUID_FEATURES)
503 ibs_caps = cpuid_eax(IBS_CPUID_FEATURES);
504 if (!(ibs_caps & IBS_CAPS_AVAIL))
505 /* cpuid flags not valid */
506 ibs_caps = 0;
507 }
508
ibs_init(void)509 void __init ibs_init(void)
510 {
511 get_ibs_caps();
512
513 if ( !ibs_caps )
514 return;
515
516 if (init_ibs_nmi()) {
517 ibs_caps = 0;
518 return;
519 }
520
521 printk("Xenoprofile: AMD IBS detected (%#x)\n",
522 (unsigned)ibs_caps);
523 }
524
525 struct op_x86_model_spec const op_athlon_spec = {
526 .num_counters = K7_NUM_COUNTERS,
527 .num_controls = K7_NUM_CONTROLS,
528 .fill_in_addresses = &athlon_fill_in_addresses,
529 .setup_ctrs = &athlon_setup_ctrs,
530 .check_ctrs = &athlon_check_ctrs,
531 .start = &athlon_start,
532 .stop = &athlon_stop
533 };
534
535 struct op_x86_model_spec const op_amd_fam15h_spec = {
536 .num_counters = FAM15H_NUM_COUNTERS,
537 .num_controls = FAM15H_NUM_CONTROLS,
538 .fill_in_addresses = &fam15h_fill_in_addresses,
539 .setup_ctrs = &athlon_setup_ctrs,
540 .check_ctrs = &athlon_check_ctrs,
541 .start = &athlon_start,
542 .stop = &athlon_stop
543 };
544