/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation, version 2 of the
 * License.
 */

/*
 * mctelem.c - x86 Machine Check Telemetry Transport
 */

#include <xen/init.h>
#include <xen/types.h>
#include <xen/kernel.h>
#include <xen/smp.h>
#include <xen/errno.h>
#include <xen/sched.h>
#include <xen/sched-if.h>
#include <xen/cpumask.h>
#include <xen/event.h>

#include <asm/processor.h>
#include <asm/system.h>
#include <asm/msr.h>

#include "mce.h"

struct mctelem_ent {
	struct mctelem_ent *mcte_next;	/* next in chronological order */
	struct mctelem_ent *mcte_prev;	/* previous in chronological order */
	uint32_t mcte_flags;		/* See MCTE_F_* below */
	uint32_t mcte_refcnt;		/* Reference count */
	void *mcte_data;		/* corresponding data payload */
};

#define	MCTE_F_CLASS_URGENT		0x0001U /* in use - urgent errors */
#define	MCTE_F_CLASS_NONURGENT		0x0002U /* in use - nonurgent errors */
#define	MCTE_F_STATE_FREE		0x0010U	/* on a freelist */
#define	MCTE_F_STATE_UNCOMMITTED	0x0020U	/* reserved; on no list */
#define	MCTE_F_STATE_COMMITTED		0x0040U	/* on a committed list */
#define	MCTE_F_STATE_PROCESSING		0x0080U	/* on a processing list */

#define	MCTE_F_MASK_CLASS	(MCTE_F_CLASS_URGENT | MCTE_F_CLASS_NONURGENT)
#define	MCTE_F_MASK_STATE	(MCTE_F_STATE_FREE | \
				MCTE_F_STATE_UNCOMMITTED | \
				MCTE_F_STATE_COMMITTED | \
				MCTE_F_STATE_PROCESSING)

#define	MCTE_CLASS(tep) ((tep)->mcte_flags & MCTE_F_MASK_CLASS)
#define	MCTE_SET_CLASS(tep, new) do { \
    (tep)->mcte_flags &= ~MCTE_F_MASK_CLASS; \
    (tep)->mcte_flags |= MCTE_F_CLASS_##new; } while (0)

#define	MCTE_STATE(tep) ((tep)->mcte_flags & MCTE_F_MASK_STATE)
#define	MCTE_TRANSITION_STATE(tep, old, new) do { \
    BUG_ON(MCTE_STATE(tep) != (MCTE_F_STATE_##old)); \
    (tep)->mcte_flags &= ~MCTE_F_MASK_STATE; \
    (tep)->mcte_flags |= (MCTE_F_STATE_##new); } while (0)

#define	MC_URGENT_NENT		10
#define	MC_NONURGENT_NENT	20

#define MC_NENT (MC_URGENT_NENT + MC_NONURGENT_NENT)

#define	MC_NCLASSES		(MC_NONURGENT + 1)

#define	COOKIE2MCTE(c)		((struct mctelem_ent *)(c))
#define	MCTE2COOKIE(tep)	((mctelem_cookie_t)(tep))

static struct mc_telem_ctl {
	/* Linked lists that thread the array members together.
	 *
	 * The free lists is a bit array where bit 1 means free.
	 * This as element number is quite small and is easy to
	 * atomically allocate that way.
	 *
	 * The committed list grows at the head and we do not maintain a
	 * tail pointer; insertions are performed atomically.  The head
	 * thus has the most-recently committed telemetry, i.e. the
	 * list is in reverse chronological order.  The committed list
	 * is singly-linked via mcte_prev pointers, and mcte_next is NULL.
	 * When we move telemetry from the committed list to the processing
	 * list we atomically unlink the committed list and keep a pointer
	 * to the head of that list;  we then traverse the list following
	 * mcte_prev and fill in mcte_next to doubly-link the list, and then
	 * append the tail of the list onto the processing list.  If we panic
	 * during this manipulation of the committed list we still have
	 * the pointer to its head so we can recover all entries during
	 * the panic flow (albeit in reverse chronological order).
	 *
	 * The processing list is updated in a controlled context, and
	 * we can lock it for updates.  The head of the processing list
	 * always has the oldest telemetry, and we append (as above)
	 * at the tail of the processing list. */
	DECLARE_BITMAP(mctc_free, MC_NENT);
	struct mctelem_ent *mctc_committed[MC_NCLASSES];
	struct mctelem_ent *mctc_processing_head[MC_NCLASSES];
	struct mctelem_ent *mctc_processing_tail[MC_NCLASSES];
	/*
	 * Telemetry array
	 */
	struct mctelem_ent *mctc_elems;
} mctctl;

struct mc_telem_cpu_ctl {
	/*
	 * Per-CPU processing lists, used for deferred (softirq)
	 * processing of telemetry.
	 *
	 * The two pending lists @lmce_pending and @pending grow at
	 * the head in the reverse chronological order.
	 *
	 * @pending and @lmce_pending on the same CPU are mutually
	 * exclusive, i.e. deferred MCE on a CPU are either all in
	 * @lmce_pending or all in @pending. In the former case, all
	 * deferred MCE are LMCE. In the latter case, both LMCE and
	 * non-local MCE can be in @pending, and @pending contains at
	 * least one non-local MCE if it's not empty.
	 *
	 * Changes to @pending and @lmce_pending should be performed
	 * via mctelem_process_deferred() and mctelem_defer(), in order
	 * to guarantee the above mutual exclusivity.
	 */
	struct mctelem_ent *pending, *lmce_pending;
	struct mctelem_ent *processing;
};
static DEFINE_PER_CPU(struct mc_telem_cpu_ctl, mctctl);

/* Lock protecting all processing lists */
static DEFINE_SPINLOCK(processing_lock);

static void mctelem_xchg_head(struct mctelem_ent **headp,
				struct mctelem_ent **linkp,
				struct mctelem_ent *new)
{
	for (;;) {
		struct mctelem_ent *old;

		*linkp = old = *headp;
		if (cmpxchgptr(headp, old, new) == old)
			break;
	}
}

/**
 * Append a telemetry of deferred MCE to a per-cpu pending list,
 * either @pending or @lmce_pending, according to rules below:
 *  - if @pending is not empty, then the new telemetry will be
 *    appended to @pending;
 *  - if @pending is empty and the new telemetry is for a deferred
 *    LMCE, then the new telemetry will be appended to @lmce_pending;
 *  - if @pending is empty and the new telemetry is for a deferred
 *    non-local MCE, all existing telemetries in @lmce_pending will be
 *    moved to @pending and then the new telemetry will be appended to
 *    @pending.
 *
 * This function must be called with MCIP bit set, so that it does not
 * need to worry about MC# re-occurring in this function.
 *
 * As a result, this function can preserve the mutual exclusivity
 * between @pending and @lmce_pending (see their comments in struct
 * mc_telem_cpu_ctl).
 *
 * Parameters:
 *  @cookie: telemetry of the deferred MCE
 *  @lmce:   indicate whether the telemetry is for LMCE
 */
void mctelem_defer(mctelem_cookie_t cookie, bool lmce)
{
	struct mctelem_ent *tep = COOKIE2MCTE(cookie);
	struct mc_telem_cpu_ctl *mctctl = &this_cpu(mctctl);

	ASSERT(mctctl->pending == NULL || mctctl->lmce_pending == NULL);

	if (mctctl->pending)
		mctelem_xchg_head(&mctctl->pending, &tep->mcte_next, tep);
	else if (lmce)
		mctelem_xchg_head(&mctctl->lmce_pending, &tep->mcte_next, tep);
	else {
		/*
		 * LMCE is supported on Skylake-server and later CPUs, on
		 * which mce_broadcast is always true. Therefore, non-empty
		 * mctctl->lmce_pending in this branch implies a broadcasting
		 * MC# is being handled, every CPU is in the exception
		 * context, and no one is consuming mctctl->pending at this
		 * moment. As a result, the following two exchanges together
		 * can be treated as atomic.
		 */
		if (mctctl->lmce_pending)
			mctelem_xchg_head(&mctctl->lmce_pending,
					  &mctctl->pending, NULL);
		mctelem_xchg_head(&mctctl->pending, &tep->mcte_next, tep);
	}
}

/**
 * Move telemetries of deferred MCE from the per-cpu pending list on
 * this or another CPU to the per-cpu processing list on this CPU, and
 * then process all deferred MCE on the processing list.
 *
 * This function can be called with MCIP bit set (e.g. from MC#
 * handler) or cleared (from MCE softirq handler). In the latter case,
 * MC# may re-occur in this function.
 *
 * Parameters:
 *  @cpu:  indicate the CPU where the pending list is
 *  @fn:   the function to handle the deferred MCE
 *  @lmce: indicate which pending list on @cpu is handled
 */
void mctelem_process_deferred(unsigned int cpu,
			      int (*fn)(mctelem_cookie_t),
			      bool lmce)
{
	struct mctelem_ent *tep;
	struct mctelem_ent *head, *prev;
	struct mc_telem_cpu_ctl *mctctl = &per_cpu(mctctl, cpu);
	int ret;

	/*
	 * First, unhook the list of telemetry structures, and
	 * hook it up to the processing list head for this CPU.
	 *
	 * If @lmce is true and a non-local MC# occurs before the
	 * following atomic exchange, @lmce will not hold after
	 * resumption, because all telemetries in @lmce_pending on
	 * @cpu are moved to @pending on @cpu in mcheck_cmn_handler().
	 * In such a case, no telemetries will be handled in this
	 * function after resumption. Another round of MCE softirq,
	 * which was raised by above mcheck_cmn_handler(), will handle
	 * those moved telemetries in @pending on @cpu.
	 *
	 * Any MC# occurring after the following atomic exchange will be
	 * handled by another round of MCE softirq.
	 */
	mctelem_xchg_head(lmce ? &mctctl->lmce_pending : &mctctl->pending,
			  &this_cpu(mctctl.processing), NULL);

	head = this_cpu(mctctl.processing);

	/*
	 * Then, fix up the list to include prev pointers, to make
	 * things a little easier, as the list must be traversed in
	 * chronological order, which is backward from the order they
	 * are in.
	 */
	for (tep = head, prev = NULL; tep != NULL; tep = tep->mcte_next) {
		tep->mcte_prev = prev;
		prev = tep;
	}

	/*
	 * Now walk the list of telemetry structures, handling each
	 * one of them. Unhooking the structure here does not need to
	 * be atomic, as this list is only accessed from a softirq
	 * context; the MCE handler does not touch it.
	 */
	for (tep = prev; tep != NULL; tep = prev) {
		prev = tep->mcte_prev;
		tep->mcte_next = tep->mcte_prev = NULL;

		ret = fn(MCTE2COOKIE(tep));
		if (prev != NULL)
			prev->mcte_next = NULL;
		tep->mcte_prev = tep->mcte_next = NULL;
		if (ret != 0)
			mctelem_commit(MCTE2COOKIE(tep));
		else
			mctelem_dismiss(MCTE2COOKIE(tep));
	}
}

bool mctelem_has_deferred(unsigned int cpu)
{
	if (per_cpu(mctctl.pending, cpu) != NULL)
		return true;
	return false;
}

bool mctelem_has_deferred_lmce(unsigned int cpu)
{
	return per_cpu(mctctl.lmce_pending, cpu) != NULL;
}

/* Free an entry to its native free list; the entry must not be linked on
 * any list.
 */
static void mctelem_free(struct mctelem_ent *tep)
{
	BUG_ON(tep->mcte_refcnt != 0);
	BUG_ON(MCTE_STATE(tep) != MCTE_F_STATE_FREE);

	tep->mcte_prev = NULL;
	tep->mcte_next = NULL;

	/* set free in array */
	set_bit(tep - mctctl.mctc_elems, mctctl.mctc_free);
}

/* Increment the reference count of an entry that is not linked on to
 * any list and which only the caller has a pointer to.
 */
static void mctelem_hold(struct mctelem_ent *tep)
{
	tep->mcte_refcnt++;
}

/* Increment the reference count on an entry that is linked at the head of
 * a processing list.  The caller is responsible for locking the list.
 */
static void mctelem_processing_hold(struct mctelem_ent *tep)
{
	int which = MCTE_CLASS(tep) == MCTE_F_CLASS_URGENT ?
	    MC_URGENT : MC_NONURGENT;

	BUG_ON(tep != mctctl.mctc_processing_head[which]);
	tep->mcte_refcnt++;
}

/* Decrement the reference count on an entry that is linked at the head of
 * a processing list.  The caller is responsible for locking the list.
 */
static void mctelem_processing_release(struct mctelem_ent *tep)
{
	int which = MCTE_CLASS(tep) == MCTE_F_CLASS_URGENT ?
	    MC_URGENT : MC_NONURGENT;

	BUG_ON(tep != mctctl.mctc_processing_head[which]);
	if (--tep->mcte_refcnt == 0) {
		MCTE_TRANSITION_STATE(tep, PROCESSING, FREE);
		mctctl.mctc_processing_head[which] = tep->mcte_next;
		mctelem_free(tep);
	}
}

void __init mctelem_init(unsigned int datasz)
{
	char *datarr;
	unsigned int i;

	BUILD_BUG_ON(MC_URGENT != 0 || MC_NONURGENT != 1 || MC_NCLASSES != 2);

	datasz = (datasz & ~0xf) + 0x10;	/* 16 byte roundup */

	if ((mctctl.mctc_elems = xmalloc_array(struct mctelem_ent,
	    MC_NENT)) == NULL ||
	    (datarr = xmalloc_bytes(MC_NENT * datasz)) == NULL) {
		xfree(mctctl.mctc_elems);
		printk("Allocations for MCA telemetry failed\n");
		return;
	}

	for (i = 0; i < MC_NENT; i++) {
		struct mctelem_ent *tep;

		tep = mctctl.mctc_elems + i;
		tep->mcte_flags = MCTE_F_STATE_FREE;
		tep->mcte_refcnt = 0;
		tep->mcte_data = datarr + i * datasz;

		__set_bit(i, mctctl.mctc_free);
		tep->mcte_next = NULL;
		tep->mcte_prev = NULL;
	}
}

/* incremented non-atomically when reserve fails */
static int mctelem_drop_count;

/* Reserve a telemetry entry, or return NULL if none available.
 * If we return an entry then the caller must subsequently call exactly one of
 * mctelem_dismiss or mctelem_commit for that entry.
 */
mctelem_cookie_t mctelem_reserve(mctelem_class_t which)
{
	unsigned bit;
	unsigned start_bit = (which == MC_URGENT) ? 0 : MC_URGENT_NENT;

	for (;;) {
		bit = find_next_bit(mctctl.mctc_free, MC_NENT, start_bit);

		if (bit >= MC_NENT) {
			mctelem_drop_count++;
			return (NULL);
		}

		/* try to allocate, atomically clear free bit */
		if (test_and_clear_bit(bit, mctctl.mctc_free)) {
			/* return element we got */
			struct mctelem_ent *tep = mctctl.mctc_elems + bit;

			mctelem_hold(tep);
			MCTE_TRANSITION_STATE(tep, FREE, UNCOMMITTED);
			tep->mcte_next = NULL;
			tep->mcte_prev = NULL;
			if (which == MC_URGENT)
				MCTE_SET_CLASS(tep, URGENT);
			else
				MCTE_SET_CLASS(tep, NONURGENT);
			return MCTE2COOKIE(tep);
		}
	}
}

void *mctelem_dataptr(mctelem_cookie_t cookie)
{
	struct mctelem_ent *tep = COOKIE2MCTE(cookie);

	return tep->mcte_data;
}

/* Release a previously reserved entry back to the freelist without
 * submitting it for logging.  The entry must not be linked on to any
 * list - that's how mctelem_reserve handed it out.
 */
void mctelem_dismiss(mctelem_cookie_t cookie)
{
	struct mctelem_ent *tep = COOKIE2MCTE(cookie);

	tep->mcte_refcnt--;
	MCTE_TRANSITION_STATE(tep, UNCOMMITTED, FREE);
	mctelem_free(tep);
}

/* Commit an entry with completed telemetry for logging.  The caller must
 * not reference the entry after this call.  Note that we add entries
 * at the head of the committed list, so that list therefore has entries
 * in reverse chronological order.
 */
void mctelem_commit(mctelem_cookie_t cookie)
{
	struct mctelem_ent *tep = COOKIE2MCTE(cookie);
	mctelem_class_t target = MCTE_CLASS(tep) == MCTE_F_CLASS_URGENT ?
	    MC_URGENT : MC_NONURGENT;

	BUG_ON(tep->mcte_next != NULL || tep->mcte_prev != NULL);
	MCTE_TRANSITION_STATE(tep, UNCOMMITTED, COMMITTED);

	mctelem_xchg_head(&mctctl.mctc_committed[target], &tep->mcte_prev, tep);
}

/* Move telemetry from committed list to processing list, reversing the
 * list into chronological order.  The processing list has been
 * locked by the caller, and may be non-empty.  We append the
 * reversed committed list on to the tail of the processing list.
 * The committed list may grow even while we run, so use atomic
 * operations to swap NULL to the freelist head.
 *
 * Note that "chronological order" means the order in which producers
 * won additions to the processing list, which may not reflect the
 * strict chronological order of the associated events if events are
 * closely spaced in time and contend for the processing list at once.
 */

static struct mctelem_ent *dangling[MC_NCLASSES];

static void mctelem_append_processing(mctelem_class_t which)
{
	mctelem_class_t target = which == MC_URGENT ?
	    MC_URGENT : MC_NONURGENT;
	struct mctelem_ent **commlp = &mctctl.mctc_committed[target];
	struct mctelem_ent **proclhp = &mctctl.mctc_processing_head[target];
	struct mctelem_ent **procltp = &mctctl.mctc_processing_tail[target];
	struct mctelem_ent *tep, *ltep;

	/* Check for an empty list; no race since we hold the processing lock */
	if (*commlp == NULL)
		return;

	/* Atomically unlink the committed list, and keep a pointer to
	 * the list we unlink in a well-known location so it can be
	 * picked up in panic code should we panic between this unlink
	 * and the append to the processing list. */
	mctelem_xchg_head(commlp, &dangling[target], NULL);

	if (dangling[target] == NULL)
		return;

	/* Traverse the list following the previous pointers (reverse
	 * chronological order).  For each entry fill in the next pointer
	 * and transition the element state.  */
	for (tep = dangling[target], ltep = NULL; tep != NULL;
	    tep = tep->mcte_prev) {
		MCTE_TRANSITION_STATE(tep, COMMITTED, PROCESSING);
		tep->mcte_next = ltep;
		ltep = tep;
	}

	/* ltep points to the head of a chronologically ordered linked
	 * list of telemetry entries ending at the most recent entry
	 * dangling[target] if mcte_next is followed; tack this on to
	 * the processing list.
	 */
	if (*proclhp == NULL) {
		*proclhp = ltep;
		*procltp = dangling[target];
	} else {
		(*procltp)->mcte_next = ltep;
		ltep->mcte_prev = *procltp;
		*procltp = dangling[target];
	}
	wmb();
	dangling[target] = NULL;
	wmb();
}

mctelem_cookie_t mctelem_consume_oldest_begin(mctelem_class_t which)
{
	mctelem_class_t target = (which == MC_URGENT) ?
	    MC_URGENT : MC_NONURGENT;
	struct mctelem_ent *tep;

	spin_lock(&processing_lock);
	mctelem_append_processing(target);
	if ((tep = mctctl.mctc_processing_head[target]) == NULL) {
		spin_unlock(&processing_lock);
		return NULL;
	}

	mctelem_processing_hold(tep);
	wmb();
	spin_unlock(&processing_lock);
	return MCTE2COOKIE(tep);
}

void mctelem_consume_oldest_end(mctelem_cookie_t cookie)
{
	struct mctelem_ent *tep = COOKIE2MCTE(cookie);

	spin_lock(&processing_lock);
	mctelem_processing_release(tep);
	wmb();
	spin_unlock(&processing_lock);
}

void mctelem_ack(mctelem_class_t which, mctelem_cookie_t cookie)
{
	mctelem_class_t target = (which == MC_URGENT) ?
	    MC_URGENT : MC_NONURGENT;
	struct mctelem_ent *tep = COOKIE2MCTE(cookie);

	if (tep == NULL)
		return;

	spin_lock(&processing_lock);
	if (tep == mctctl.mctc_processing_head[target])
		mctelem_processing_release(tep);
	wmb();
	spin_unlock(&processing_lock);
}

/*
 * Local variables:
 * mode: C
 * c-file-style: "BSD"
 * c-basic-offset: 4
 * indent-tabs-mode: t
 * tab-width: 8
 * End:
 */