/**
 * @file op_model_p4.c
 * P4 model-specific MSR operations
 *
 * @remark Copyright 2002 OProfile authors
 * @remark Read the file COPYING
 *
 * @author Graydon Hoare
 */

#include <xen/types.h>
#include <asm/msr.h>
#include <asm/io.h>
#include <asm/apic.h>
#include <asm/processor.h>
#include <xen/xenoprof.h>
#include <asm/regs.h>
#include <asm/current.h>

#include "op_x86_model.h"
#include "op_counter.h"

#define NUM_EVENTS 39

#define NUM_COUNTERS_NON_HT 8
#define NUM_ESCRS_NON_HT 45
#define NUM_CCCRS_NON_HT 18
#define NUM_CONTROLS_NON_HT (NUM_ESCRS_NON_HT + NUM_CCCRS_NON_HT)

#define NUM_COUNTERS_HT2 4
#define NUM_ESCRS_HT2 23
#define NUM_CCCRS_HT2 9
#define NUM_CONTROLS_HT2 (NUM_ESCRS_HT2 + NUM_CCCRS_HT2)

static unsigned int num_counters = NUM_COUNTERS_NON_HT;


/* this has to be checked dynamically since the
   hyper-threadedness of a chip is discovered at
   kernel boot-time. */
static inline void setup_num_counters(void)
{
	if (boot_cpu_data.x86_num_siblings == 2) 	/* XXX */
		num_counters = NUM_COUNTERS_HT2;
}

static int inline addr_increment(void)
{
	return boot_cpu_data.x86_num_siblings == 2 ? 2 : 1;
}


/* tables to simulate simplified hardware view of p4 registers */
struct p4_counter_binding {
	int virt_counter;
	int counter_address;
	int cccr_address;
};

struct p4_event_binding {
	int escr_select;  /* value to put in CCCR */
	int event_select; /* value to put in ESCR */
	struct {
		int virt_counter; /* for this counter... */
		int escr_address; /* use this ESCR       */
	} bindings[2];
};

/* nb: these CTR_* defines are a duplicate of defines in
   event/i386.p4*events. */


#define CTR_BPU_0      (1 << 0)
#define CTR_MS_0       (1 << 1)
#define CTR_FLAME_0    (1 << 2)
#define CTR_IQ_4       (1 << 3)
#define CTR_BPU_2      (1 << 4)
#define CTR_MS_2       (1 << 5)
#define CTR_FLAME_2    (1 << 6)
#define CTR_IQ_5       (1 << 7)

static struct p4_counter_binding p4_counters [NUM_COUNTERS_NON_HT] = {
	{ CTR_BPU_0,   MSR_P4_BPU_PERFCTR0,   MSR_P4_BPU_CCCR0 },
	{ CTR_MS_0,    MSR_P4_MS_PERFCTR0,    MSR_P4_MS_CCCR0 },
	{ CTR_FLAME_0, MSR_P4_FLAME_PERFCTR0, MSR_P4_FLAME_CCCR0 },
	{ CTR_IQ_4,    MSR_P4_IQ_PERFCTR4,    MSR_P4_IQ_CCCR4 },
	{ CTR_BPU_2,   MSR_P4_BPU_PERFCTR2,   MSR_P4_BPU_CCCR2 },
	{ CTR_MS_2,    MSR_P4_MS_PERFCTR2,    MSR_P4_MS_CCCR2 },
	{ CTR_FLAME_2, MSR_P4_FLAME_PERFCTR2, MSR_P4_FLAME_CCCR2 },
	{ CTR_IQ_5,    MSR_P4_IQ_PERFCTR5,    MSR_P4_IQ_CCCR5 }
};

#define NUM_UNUSED_CCCRS	NUM_CCCRS_NON_HT - NUM_COUNTERS_NON_HT

/* All cccr we don't use. */
static int p4_unused_cccr[NUM_UNUSED_CCCRS] = {
	MSR_P4_BPU_CCCR1,	MSR_P4_BPU_CCCR3,
	MSR_P4_MS_CCCR1,	MSR_P4_MS_CCCR3,
	MSR_P4_FLAME_CCCR1,	MSR_P4_FLAME_CCCR3,
	MSR_P4_IQ_CCCR0,	MSR_P4_IQ_CCCR1,
	MSR_P4_IQ_CCCR2,	MSR_P4_IQ_CCCR3
};

/* p4 event codes in libop/op_event.h are indices into this table. */

static const struct p4_event_binding p4_events[NUM_EVENTS] = {
	
	{ /* BRANCH_RETIRED */
		0x05, 0x06, 
		{ {CTR_IQ_4, MSR_P4_CRU_ESCR2},
		  {CTR_IQ_5, MSR_P4_CRU_ESCR3} }
	},
	
	{ /* MISPRED_BRANCH_RETIRED */
		0x04, 0x03, 
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR0},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR1} }
	},
	
	{ /* TC_DELIVER_MODE */
		0x01, 0x01,
		{ { CTR_MS_0, MSR_P4_TC_ESCR0},  
		  { CTR_MS_2, MSR_P4_TC_ESCR1} }
	},
	
	{ /* BPU_FETCH_REQUEST */
		0x00, 0x03, 
		{ { CTR_BPU_0, MSR_P4_BPU_ESCR0},
		  { CTR_BPU_2, MSR_P4_BPU_ESCR1} }
	},

	{ /* ITLB_REFERENCE */
		0x03, 0x18,
		{ { CTR_BPU_0, MSR_P4_ITLB_ESCR0},
		  { CTR_BPU_2, MSR_P4_ITLB_ESCR1} }
	},

	{ /* MEMORY_CANCEL */
		0x05, 0x02,
		{ { CTR_FLAME_0, MSR_P4_DAC_ESCR0},
		  { CTR_FLAME_2, MSR_P4_DAC_ESCR1} }
	},

	{ /* MEMORY_COMPLETE */
		0x02, 0x08,
		{ { CTR_FLAME_0, MSR_P4_SAAT_ESCR0},
		  { CTR_FLAME_2, MSR_P4_SAAT_ESCR1} }
	},

	{ /* LOAD_PORT_REPLAY */
		0x02, 0x04, 
		{ { CTR_FLAME_0, MSR_P4_SAAT_ESCR0},
		  { CTR_FLAME_2, MSR_P4_SAAT_ESCR1} }
	},

	{ /* STORE_PORT_REPLAY */
		0x02, 0x05,
		{ { CTR_FLAME_0, MSR_P4_SAAT_ESCR0},
		  { CTR_FLAME_2, MSR_P4_SAAT_ESCR1} }
	},

	{ /* MOB_LOAD_REPLAY */
		0x02, 0x03,
		{ { CTR_BPU_0, MSR_P4_MOB_ESCR0},
		  { CTR_BPU_2, MSR_P4_MOB_ESCR1} }
	},

	{ /* PAGE_WALK_TYPE */
		0x04, 0x01,
		{ { CTR_BPU_0, MSR_P4_PMH_ESCR0},
		  { CTR_BPU_2, MSR_P4_PMH_ESCR1} }
	},

	{ /* BSQ_CACHE_REFERENCE */
		0x07, 0x0c, 
		{ { CTR_BPU_0, MSR_P4_BSU_ESCR0},
		  { CTR_BPU_2, MSR_P4_BSU_ESCR1} }
	},

	{ /* IOQ_ALLOCATION */
		0x06, 0x03, 
		{ { CTR_BPU_0, MSR_P4_FSB_ESCR0},
		  { 0, 0 } }
	},

	{ /* IOQ_ACTIVE_ENTRIES */
		0x06, 0x1a, 
		{ { CTR_BPU_2, MSR_P4_FSB_ESCR1},
		  { 0, 0 } }
	},

	{ /* FSB_DATA_ACTIVITY */
		0x06, 0x17, 
		{ { CTR_BPU_0, MSR_P4_FSB_ESCR0},
		  { CTR_BPU_2, MSR_P4_FSB_ESCR1} }
	},

	{ /* BSQ_ALLOCATION */
		0x07, 0x05, 
		{ { CTR_BPU_0, MSR_P4_BSU_ESCR0},
		  { 0, 0 } }
	},

	{ /* BSQ_ACTIVE_ENTRIES */
		0x07, 0x06,
		{ { CTR_BPU_2, MSR_P4_BSU_ESCR1 /* guess */},  
		  { 0, 0 } }
	},

	{ /* X87_ASSIST */
		0x05, 0x03, 
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR2},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
	},

	{ /* SSE_INPUT_ASSIST */
		0x01, 0x34,
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},
  
	{ /* PACKED_SP_UOP */
		0x01, 0x08, 
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},
  
	{ /* PACKED_DP_UOP */
		0x01, 0x0c, 
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},

	{ /* SCALAR_SP_UOP */
		0x01, 0x0a, 
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},

	{ /* SCALAR_DP_UOP */
		0x01, 0x0e,
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},

	{ /* 64BIT_MMX_UOP */
		0x01, 0x02, 
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},
  
	{ /* 128BIT_MMX_UOP */
		0x01, 0x1a, 
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},

	{ /* X87_FP_UOP */
		0x01, 0x04, 
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},
  
	{ /* X87_SIMD_MOVES_UOP */
		0x01, 0x2e, 
		{ { CTR_FLAME_0, MSR_P4_FIRM_ESCR0},
		  { CTR_FLAME_2, MSR_P4_FIRM_ESCR1} }
	},
  
	{ /* MACHINE_CLEAR */
		0x05, 0x02, 
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR2},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
	},

	{ /* GLOBAL_POWER_EVENTS */
		0x06, 0x13 /* older manual says 0x05, newer 0x13 */,
		{ { CTR_BPU_0, MSR_P4_FSB_ESCR0},
		  { CTR_BPU_2, MSR_P4_FSB_ESCR1} }
	},
  
	{ /* TC_MS_XFER */
		0x00, 0x05, 
		{ { CTR_MS_0, MSR_P4_MS_ESCR0},
		  { CTR_MS_2, MSR_P4_MS_ESCR1} }
	},

	{ /* UOP_QUEUE_WRITES */
		0x00, 0x09,
		{ { CTR_MS_0, MSR_P4_MS_ESCR0},
		  { CTR_MS_2, MSR_P4_MS_ESCR1} }
	},

	{ /* FRONT_END_EVENT */
		0x05, 0x08,
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR2},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
	},

	{ /* EXECUTION_EVENT */
		0x05, 0x0c,
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR2},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
	},

	{ /* REPLAY_EVENT */
		0x05, 0x09,
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR2},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR3} }
	},

	{ /* INSTR_RETIRED */
		0x04, 0x02, 
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR0},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR1} }
	},

	{ /* UOPS_RETIRED */
		0x04, 0x01,
		{ { CTR_IQ_4, MSR_P4_CRU_ESCR0},
		  { CTR_IQ_5, MSR_P4_CRU_ESCR1} }
	},

	{ /* UOP_TYPE */    
		0x02, 0x02, 
		{ { CTR_IQ_4, MSR_P4_RAT_ESCR0},
		  { CTR_IQ_5, MSR_P4_RAT_ESCR1} }
	},

	{ /* RETIRED_MISPRED_BRANCH_TYPE */
		0x02, 0x05, 
		{ { CTR_MS_0, MSR_P4_TBPU_ESCR0},
		  { CTR_MS_2, MSR_P4_TBPU_ESCR1} }
	},

	{ /* RETIRED_BRANCH_TYPE */
		0x02, 0x04,
		{ { CTR_MS_0, MSR_P4_TBPU_ESCR0},
		  { CTR_MS_2, MSR_P4_TBPU_ESCR1} }
	}
};


#define MISC_PMC_ENABLED_P(x) ((x) & 1ULL << 7)

#define ESCR_RESERVED_BITS 0x80000003ULL
#define ESCR_CLEAR(escr) ((escr) &= ESCR_RESERVED_BITS)
#define ESCR_SET_USR_0(escr, usr) ((escr) |= (((usr) & 1ULL) << 2))
#define ESCR_SET_OS_0(escr, os) ((escr) |= (((os) & 1ULL) << 3))
#define ESCR_SET_USR_1(escr, usr) ((escr) |= (((usr) & 1ULL)))
#define ESCR_SET_OS_1(escr, os) ((escr) |= (((os) & 1ULL) << 1))
#define ESCR_SET_EVENT_SELECT(escr, sel) ((escr) |= (((sel) & 0x3fULL) << 25))
#define ESCR_SET_EVENT_MASK(escr, mask) ((escr) |= (((mask) & 0xffffULL) << 9))
#define ESCR_READ(escr,ev,i) do {rdmsrl(ev->bindings[(i)].escr_address, (escr));} while (0)
#define ESCR_WRITE(escr,ev,i) do {wrmsrl(ev->bindings[(i)].escr_address, (escr));} while (0)

#define CCCR_RESERVED_BITS 0x38030FFFULL
#define CCCR_CLEAR(cccr) ((cccr) &= CCCR_RESERVED_BITS)
#define CCCR_SET_REQUIRED_BITS(cccr) ((cccr) |= 0x00030000ULL)
#define CCCR_SET_ESCR_SELECT(cccr, sel) ((cccr) |= (((sel) & 0x07ULL) << 13))
#define CCCR_SET_PMI_OVF_0(cccr) ((cccr) |= (1ULL<<26))
#define CCCR_SET_PMI_OVF_1(cccr) ((cccr) |= (1ULL<<27))
#define CCCR_SET_ENABLE(cccr) ((cccr) |= (1ULL<<12))
#define CCCR_SET_DISABLE(cccr) ((cccr) &= ~(1ULL<<12))
#define CCCR_READ(msr_content, i) do {rdmsrl(p4_counters[(i)].cccr_address, (msr_content));} while (0)
#define CCCR_WRITE(msr_content, i) do {wrmsrl(p4_counters[(i)].cccr_address, (msr_content));} while (0)
#define CCCR_OVF_P(cccr) ((cccr) & (1ULL<<31))
#define CCCR_CLEAR_OVF(cccr) ((cccr) &= (~(1ULL<<31)))

#define CTR_READ(msr_content,i) do {rdmsrl(p4_counters[(i)].counter_address, (msr_content));} while (0)
#define CTR_WRITE(msr_content,i) do {wrmsrl(p4_counters[(i)].counter_address, -(msr_content));} while (0)
#define CTR_OVERFLOW_P(ctr) (!((ctr) & 0x80000000ULL))


/* this assigns a "stagger" to the current CPU, which is used throughout
   the code in this module as an extra array offset, to select the "even"
   or "odd" part of all the divided resources. */
static unsigned int get_stagger(void)
{
	int cpu = smp_processor_id();
	return (cpu != cpumask_first(per_cpu(cpu_sibling_mask, cpu)));
}


/* finally, mediate access to a real hardware counter
   by passing a "virtual" counter numer to this macro,
   along with your stagger setting. */
#define VIRT_CTR(stagger, i) ((i) + ((num_counters) * (stagger)))

static unsigned long reset_value[NUM_COUNTERS_NON_HT];


static void p4_fill_in_addresses(struct op_msrs * const msrs)
{
	unsigned int i;
	unsigned int addr, stag;

	setup_num_counters();
	stag = get_stagger();

	/* the counter registers we pay attention to */
	for (i = 0; i < num_counters; ++i) {
		msrs->counters[i].addr = 
			p4_counters[VIRT_CTR(stag, i)].counter_address;
	}

	/* FIXME: bad feeling, we don't save the 10 counters we don't use. */

	/* 18 CCCR registers */
	for (i = 0, addr = MSR_P4_BPU_CCCR0 + stag;
	     addr <= MSR_P4_IQ_CCCR5; ++i, addr += addr_increment()) {
		msrs->controls[i].addr = addr;
	}
	
	/* 43 ESCR registers in three or four discontiguous group */
	for (addr = MSR_P4_BSU_ESCR0 + stag;
	     addr < MSR_P4_IQ_ESCR0; ++i, addr += addr_increment()) {
		msrs->controls[i].addr = addr;
	}

	/* no IQ_ESCR0/1 on some models, we save a seconde time BSU_ESCR0/1
	 * to avoid special case in nmi_{save|restore}_registers() */
	if (boot_cpu_data.x86_model >= 0x3) {
		for (addr = MSR_P4_BSU_ESCR0 + stag;
		     addr <= MSR_P4_BSU_ESCR1; ++i, addr += addr_increment()) {
			msrs->controls[i].addr = addr;
		}
	} else {
		for (addr = MSR_P4_IQ_ESCR0 + stag;
		     addr <= MSR_P4_IQ_ESCR1; ++i, addr += addr_increment()) {
			msrs->controls[i].addr = addr;
		}
	}

	for (addr = MSR_P4_RAT_ESCR0 + stag;
	     addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) {
		msrs->controls[i].addr = addr;
	}
	
	for (addr = MSR_P4_MS_ESCR0 + stag;
	     addr <= MSR_P4_TC_ESCR1; ++i, addr += addr_increment()) { 
		msrs->controls[i].addr = addr;
	}
	
	for (addr = MSR_P4_IX_ESCR0 + stag;
	     addr <= MSR_P4_CRU_ESCR3; ++i, addr += addr_increment()) { 
		msrs->controls[i].addr = addr;
	}

	/* there are 2 remaining non-contiguously located ESCRs */

	if (num_counters == NUM_COUNTERS_NON_HT) {		
		/* standard non-HT CPUs handle both remaining ESCRs*/
		msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
		msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;

	} else if (stag == 0) {
		/* HT CPUs give the first remainder to the even thread, as
		   the 32nd control register */
		msrs->controls[i++].addr = MSR_P4_CRU_ESCR4;

	} else {
		/* and two copies of the second to the odd thread,
		   for the 22st and 23nd control registers */
		msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
		msrs->controls[i++].addr = MSR_P4_CRU_ESCR5;
	}
}


static void pmc_setup_one_p4_counter(unsigned int ctr)
{
	int i;
	int const maxbind = 2;
	uint64_t cccr = 0;
	uint64_t escr = 0;
	unsigned int counter_bit;
	const struct p4_event_binding *ev = NULL;
	unsigned int stag;

	stag = get_stagger();
	
	/* convert from counter *number* to counter *bit* */
	counter_bit = 1 << VIRT_CTR(stag, ctr);
	
	/* find our event binding structure. */
	if (counter_config[ctr].event <= 0 || counter_config[ctr].event > NUM_EVENTS) {
		printk(KERN_ERR "oprofile: P4 event code %#lx out of range\n",
		       counter_config[ctr].event);
		return;
	}
	
	ev = &(p4_events[counter_config[ctr].event - 1]);
	
	for (i = 0; i < maxbind; i++) {
		if (ev->bindings[i].virt_counter & counter_bit) {

			/* modify ESCR */
			ESCR_READ(escr, ev, i);
			ESCR_CLEAR(escr);
			if (stag == 0) {
				ESCR_SET_USR_0(escr, counter_config[ctr].user);
				ESCR_SET_OS_0(escr, counter_config[ctr].kernel);
			} else {
				ESCR_SET_USR_1(escr, counter_config[ctr].user);
				ESCR_SET_OS_1(escr, counter_config[ctr].kernel);
			}
			ESCR_SET_EVENT_SELECT(escr, ev->event_select);
			ESCR_SET_EVENT_MASK(escr, counter_config[ctr].unit_mask);			
			ESCR_WRITE(escr, ev, i);
		       
			/* modify CCCR */
			CCCR_READ(cccr, VIRT_CTR(stag, ctr));
			CCCR_CLEAR(cccr);
			CCCR_SET_REQUIRED_BITS(cccr);
			CCCR_SET_ESCR_SELECT(cccr, ev->escr_select);
			if (stag == 0) {
				CCCR_SET_PMI_OVF_0(cccr);
			} else {
				CCCR_SET_PMI_OVF_1(cccr);
			}
			CCCR_WRITE(cccr, VIRT_CTR(stag, ctr));
			return;
		}
	}

	printk(KERN_ERR 
	       "oprofile: P4 event code %#lx no binding, stag %d ctr %d\n",
	       counter_config[ctr].event, stag, ctr);
}


static void p4_setup_ctrs(struct op_msrs const * const msrs)
{
	unsigned int i;
	uint64_t msr_content;
	unsigned int addr;
	unsigned int stag;

	stag = get_stagger();

	rdmsrl(MSR_IA32_MISC_ENABLE, msr_content);
	if (! MISC_PMC_ENABLED_P(msr_content)) {
		printk(KERN_ERR "oprofile: P4 PMC not available\n");
		return;
	}

	/* clear the cccrs we will use */
	for (i = 0 ; i < num_counters ; i++) {
		rdmsrl(p4_counters[VIRT_CTR(stag, i)].cccr_address, msr_content);
		CCCR_CLEAR(msr_content);
		CCCR_SET_REQUIRED_BITS(msr_content);
		wrmsrl(p4_counters[VIRT_CTR(stag, i)].cccr_address, msr_content);
	}

	/* clear cccrs outside our concern */
	for (i = stag ; i < NUM_UNUSED_CCCRS ; i += addr_increment()) {
		rdmsrl(p4_unused_cccr[i], msr_content);
		CCCR_CLEAR(msr_content);
		CCCR_SET_REQUIRED_BITS(msr_content);
		wrmsrl(p4_unused_cccr[i], msr_content);
	}

	/* clear all escrs (including those outside our concern) */
	for (addr = MSR_P4_BSU_ESCR0 + stag;
	     addr <  MSR_P4_IQ_ESCR0; addr += addr_increment()) {
		wrmsrl(addr, 0x0ULL);
	}

	/* On older models clear also MSR_P4_IQ_ESCR0/1 */
	if (boot_cpu_data.x86_model < 0x3) {
		wrmsrl(MSR_P4_IQ_ESCR0, 0x0ULL);
		wrmsrl(MSR_P4_IQ_ESCR1, 0x0ULL);
	}

	for (addr = MSR_P4_RAT_ESCR0 + stag;
	     addr <= MSR_P4_SSU_ESCR0; ++i, addr += addr_increment()) {
		wrmsrl(addr, 0x0ULL);
	}
	
	for (addr = MSR_P4_MS_ESCR0 + stag;
	     addr <= MSR_P4_TC_ESCR1; addr += addr_increment()){ 
		wrmsrl(addr, 0x0ULL);
	}
	
	for (addr = MSR_P4_IX_ESCR0 + stag;
	     addr <= MSR_P4_CRU_ESCR3; addr += addr_increment()){ 
		wrmsrl(addr, 0x0ULL);
	}

	if (num_counters == NUM_COUNTERS_NON_HT) {		
		wrmsrl(MSR_P4_CRU_ESCR4, 0x0ULL);
		wrmsrl(MSR_P4_CRU_ESCR5, 0x0ULL);
	} else if (stag == 0) {
		wrmsrl(MSR_P4_CRU_ESCR4, 0x0ULL);
	} else {
		wrmsrl(MSR_P4_CRU_ESCR5, 0x0ULL);
	}		
	
	/* setup all counters */
	for (i = 0 ; i < num_counters ; ++i) {
		if (counter_config[i].enabled) {
			reset_value[i] = counter_config[i].count;
			pmc_setup_one_p4_counter(i);
			CTR_WRITE(counter_config[i].count, VIRT_CTR(stag, i));
		} else {
			reset_value[i] = 0;
		}
	}
}

static int p4_check_ctrs(unsigned int const cpu,
                         struct op_msrs const * const msrs,
                         struct cpu_user_regs const * const regs)
{
	unsigned long ctr, stag, real;
	uint64_t msr_content;
	int i;
	int ovf = 0;
	unsigned long eip = regs->rip;
	int mode = xenoprofile_get_mode(current, regs);

	stag = get_stagger();

	for (i = 0; i < num_counters; ++i) {
		
		if (!reset_value[i]) 
			continue;

		/* 
		 * there is some eccentricity in the hardware which
		 * requires that we perform 2 extra corrections:
		 *
		 * - check both the CCCR:OVF flag for overflow and the
		 *   counter high bit for un-flagged overflows.
		 *
		 * - write the counter back twice to ensure it gets
		 *   updated properly.
		 * 
		 * the former seems to be related to extra NMIs happening
		 * during the current NMI; the latter is reported as errata
		 * N15 in intel doc 249199-029, pentium 4 specification
		 * update, though their suggested work-around does not
		 * appear to solve the problem.
		 */
		
		real = VIRT_CTR(stag, i);

		CCCR_READ(msr_content, real);
 		CTR_READ(ctr, real);
		if (CCCR_OVF_P(msr_content) || CTR_OVERFLOW_P(ctr)) {
			xenoprof_log_event(current, regs, eip, mode, i);
			CTR_WRITE(reset_value[i], real);
			CCCR_CLEAR_OVF(msr_content);
			CCCR_WRITE(msr_content, real);
 			CTR_WRITE(reset_value[i], real);
			ovf = 1;
		}
	}

	/* P4 quirk: you have to re-unmask the apic vector */
	apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED);

	return ovf;
}


static void p4_start(struct op_msrs const * const msrs)
{
	unsigned int stag;
	uint64_t msr_content;
	int i;

	stag = get_stagger();

	for (i = 0; i < num_counters; ++i) {
		if (!reset_value[i])
			continue;
		CCCR_READ(msr_content, VIRT_CTR(stag, i));
		CCCR_SET_ENABLE(msr_content);
		CCCR_WRITE(msr_content, VIRT_CTR(stag, i));
	}
}


static void p4_stop(struct op_msrs const * const msrs)
{
	unsigned int stag;
	uint64_t msr_content;
	int i;

	stag = get_stagger();

	for (i = 0; i < num_counters; ++i) {
		CCCR_READ(msr_content, VIRT_CTR(stag, i));
		CCCR_SET_DISABLE(msr_content);
		CCCR_WRITE(msr_content, VIRT_CTR(stag, i));
	}
}


struct op_x86_model_spec const op_p4_ht2_spec = {
	.num_counters = NUM_COUNTERS_HT2,
	.num_controls = NUM_CONTROLS_HT2,
	.fill_in_addresses = &p4_fill_in_addresses,
	.setup_ctrs = &p4_setup_ctrs,
	.check_ctrs = &p4_check_ctrs,
	.start = &p4_start,
	.stop = &p4_stop
};


struct op_x86_model_spec const op_p4_spec = {
	.num_counters = NUM_COUNTERS_NON_HT,
	.num_controls = NUM_CONTROLS_NON_HT,
	.fill_in_addresses = &p4_fill_in_addresses,
	.setup_ctrs = &p4_setup_ctrs,
	.check_ctrs = &p4_check_ctrs,
	.start = &p4_start,
	.stop = &p4_stop
};