xref: /kernel/dev/pcie/pcie_irqs.cpp

// Copyright 2016 The Fuchsia Authors
// Copyright (c) 2016, Google, Inc. All rights reserved
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT
//
#include <assert.h>
#include <debug.h>
#include <dev/interrupt.h>
#include <dev/pcie_bus_driver.h>
#include <dev/pcie_bridge.h>
#include <dev/pcie_root.h>
#include <err.h>
#include <kernel/spinlock.h>
#include <list.h>
#include <pow2.h>
#include <string.h>
#include <trace.h>
#include <vm/vm.h>

#include <dev/pci_config.h>
#include <dev/pcie_device.h>

#include <fbl/alloc_checker.h>
#include <fbl/auto_lock.h>
#include <ktl/move.h>

using fbl::AutoLock;

#define LOCAL_TRACE 0

/******************************************************************************
 *
 * Helper routines common to all IRQ modes.
 *
 ******************************************************************************/
void PcieDevice::ResetCommonIrqBookkeeping() {
    if (irq_.handler_count > 1) {
        DEBUG_ASSERT(irq_.handlers != &irq_.singleton_handler);
        delete[] irq_.handlers;
    }

    irq_.singleton_handler.handler = nullptr;
    irq_.singleton_handler.ctx = nullptr;
    irq_.singleton_handler.dev = nullptr;
    irq_.mode          = PCIE_IRQ_MODE_DISABLED;
    irq_.handlers      = nullptr;
    irq_.handler_count = 0;
}

zx_status_t PcieDevice::AllocIrqHandlers(uint requested_irqs, bool is_masked) {
    DEBUG_ASSERT(requested_irqs);
    DEBUG_ASSERT(!irq_.handlers);
    DEBUG_ASSERT(!irq_.handler_count);

    if (requested_irqs == 1) {
        irq_.handlers      = &irq_.singleton_handler;
        irq_.handler_count = 1;
    } else {
        fbl::AllocChecker ac;
        irq_.handlers = new (&ac) pcie_irq_handler_state_t[requested_irqs];

        if (!ac.check())
            return ZX_ERR_NO_MEMORY;

        irq_.handler_count = requested_irqs;
    }

    for (uint i = 0; i < irq_.handler_count; ++i) {
        DEBUG_ASSERT(irq_.handlers[i].handler == nullptr);
        DEBUG_ASSERT(irq_.handlers[i].dev == nullptr);
        DEBUG_ASSERT(irq_.handlers[i].ctx == nullptr);
        irq_.handlers[i].dev        = this;
        irq_.handlers[i].pci_irq_id = i;
        irq_.handlers[i].masked     = is_masked;
    }

    return ZX_OK;
}

/******************************************************************************
 *
 * Legacy IRQ mode routines.
 *
 ******************************************************************************/
fbl::RefPtr<SharedLegacyIrqHandler> SharedLegacyIrqHandler::Create(uint irq_id) {
    fbl::AllocChecker ac;

    SharedLegacyIrqHandler* handler = new (&ac) SharedLegacyIrqHandler(irq_id);
    if (!ac.check()) {
        TRACEF("Failed to create shared legacy IRQ handler for system IRQ ID %u\n", irq_id);
        return nullptr;
    }

    return fbl::AdoptRef(handler);
}

SharedLegacyIrqHandler::SharedLegacyIrqHandler(uint irq_id)
    : irq_id_(irq_id) {
    list_initialize(&device_handler_list_);
    mask_interrupt(irq_id_);  // This should not be needed, but just in case.
    zx_status_t status = register_int_handler(irq_id_, HandlerThunk, this);
    DEBUG_ASSERT(status == ZX_OK);
}

SharedLegacyIrqHandler::~SharedLegacyIrqHandler() {
    DEBUG_ASSERT(list_is_empty(&device_handler_list_));
    mask_interrupt(irq_id_);
    zx_status_t status = register_int_handler(irq_id_, nullptr, nullptr);
    DEBUG_ASSERT(status == ZX_OK);
}

void SharedLegacyIrqHandler::Handler() {
    /* Go over the list of device's which share this legacy IRQ and give them a
     * chance to handle any interrupts which may be pending in their device.
     * Keep track of whether or not any device has requested a re-schedule event
     * at the end of this IRQ. */
    AutoSpinLockNoIrqSave list_lock(&device_handler_list_lock_);

    if (list_is_empty(&device_handler_list_)) {
        TRACEF("Received legacy PCI INT (system IRQ %u), but there are no devices registered to "
               "handle this interrupt.  This is Very Bad.  Disabling the interrupt at the system "
               "IRQ level to prevent meltdown.\n",
               irq_id_);
        mask_interrupt(irq_id_);
        return;
    }

    PcieDevice* dev;
    list_for_every_entry(&device_handler_list_,
                         dev,
                         PcieDevice,
                         irq_.legacy.shared_handler_node) {
        uint16_t command, status;
        auto cfg = dev->config();

        {
            AutoSpinLockNoIrqSave cmd_reg_lock(&dev->cmd_reg_lock_);
            command = cfg->Read(PciConfig::kCommand);
            status  = cfg->Read(PciConfig::kStatus);
        }

        if ((status & PCIE_CFG_STATUS_INT_STS) && !(command & PCIE_CFG_COMMAND_INT_DISABLE)) {
            DEBUG_ASSERT(dev);
            pcie_irq_handler_state_t* hstate  = &dev->irq_.handlers[0];

            if (hstate) {
                pcie_irq_handler_retval_t irq_ret = PCIE_IRQRET_MASK;
                AutoSpinLockNoIrqSave device_handler_lock(&hstate->lock);

                if (hstate->handler) {
                    if (!hstate->masked)
                        irq_ret = hstate->handler(*dev, 0, hstate->ctx);
                } else {
                    TRACEF("Received legacy PCI INT (system IRQ %u) for %02x:%02x.%02x, but no irq_ "
                           "handler has been registered by the driver.  Force disabling the "
                           "interrupt.\n",
                           irq_id_, dev->bus_id_, dev->dev_id_, dev->func_id_);
                }

                if (irq_ret & PCIE_IRQRET_MASK) {
                    hstate->masked = true;
                    {
                        AutoSpinLockNoIrqSave cmd_reg_lock(&dev->cmd_reg_lock_);
                        command = cfg->Read(PciConfig::kCommand);
                        cfg->Write(PciConfig::kCommand, command | PCIE_CFG_COMMAND_INT_DISABLE);
                    }
                }
            } else {
                TRACEF("Received legacy PCI INT (system IRQ %u) for %02x:%02x.%02x , but no irq_ "
                       "handlers have been allocated!  Force disabling the interrupt.\n",
                       irq_id_, dev->bus_id_, dev->dev_id_, dev->func_id_);

                {
                    AutoSpinLockNoIrqSave cmd_reg_lock(&dev->cmd_reg_lock_);
                    command = cfg->Read(PciConfig::kCommand);
                    cfg->Write(PciConfig::kCommand, command | PCIE_CFG_COMMAND_INT_DISABLE);
                }
            }
        }
    }
}

void SharedLegacyIrqHandler::AddDevice(PcieDevice& dev) {
    DEBUG_ASSERT(dev.irq_.legacy.shared_handler.get() == this);
    DEBUG_ASSERT(!list_in_list(&dev.irq_.legacy.shared_handler_node));

    /* Make certain that the device's legacy IRQ has been masked at the PCI
     * device level.  Then add this dev to the handler's list.  If this was the
     * first device added to the handler list, unmask the handler IRQ at the top
     * level. */
    AutoSpinLock lock(&device_handler_list_lock_);

    dev.cfg_->Write(PciConfig::kCommand, dev.cfg_->Read(PciConfig::kCommand) |
                                          PCIE_CFG_COMMAND_INT_DISABLE);

    bool first_device = list_is_empty(&device_handler_list_);
    list_add_tail(&device_handler_list_, &dev.irq_.legacy.shared_handler_node);

    if (first_device)
        unmask_interrupt(irq_id_);
}

void SharedLegacyIrqHandler::RemoveDevice(PcieDevice& dev) {
    DEBUG_ASSERT(dev.irq_.legacy.shared_handler.get() == this);
    DEBUG_ASSERT(list_in_list(&dev.irq_.legacy.shared_handler_node));

    /* Make absolutely sure we have been masked at the PCIe config level, then
     * remove the device from the shared handler list.  If this was the last
     * device on the list, mask the top level IRQ */
    AutoSpinLock lock(&device_handler_list_lock_);

    dev.cfg_->Write(PciConfig::kCommand, dev.cfg_->Read(PciConfig::kCommand) |
                                          PCIE_CFG_COMMAND_INT_DISABLE);
    list_delete(&dev.irq_.legacy.shared_handler_node);

    if (list_is_empty(&device_handler_list_))
        mask_interrupt(irq_id_);
}

zx_status_t PcieDevice::MaskUnmaskLegacyIrq(bool mask) {
    if (!irq_.handlers || !irq_.handler_count)
        return ZX_ERR_INVALID_ARGS;

    pcie_irq_handler_state_t& hstate = irq_.handlers[0];

    {
        AutoSpinLock lock(&hstate.lock);

        if (mask) ModifyCmdLocked(0, PCIE_CFG_COMMAND_INT_DISABLE);
        else      ModifyCmdLocked(PCIE_CFG_COMMAND_INT_DISABLE, 0);
        hstate.masked = mask;
    }

    return ZX_OK;
}

zx_status_t PcieDevice::EnterLegacyIrqMode(uint requested_irqs) {
    DEBUG_ASSERT(requested_irqs);

    if (!irq_.legacy.pin || (requested_irqs > 1))
        return ZX_ERR_NOT_SUPPORTED;

    // Make absolutely certain we are masked.
    ModifyCmdLocked(0, PCIE_CFG_COMMAND_INT_DISABLE);

    // We can never fail to allocated a single handlers (since we are going to
    // use the pre-allocated singleton)
    __UNUSED zx_status_t res = AllocIrqHandlers(requested_irqs, true);
    DEBUG_ASSERT(res == ZX_OK);
    DEBUG_ASSERT(irq_.handlers == &irq_.singleton_handler);

    irq_.mode = PCIE_IRQ_MODE_LEGACY;
    irq_.legacy.shared_handler->AddDevice(*this);

    return ZX_OK;
}

void PcieDevice::LeaveLegacyIrqMode() {
    /* Disable legacy IRQs and unregister from the shared legacy handler */
    MaskUnmaskLegacyIrq(true);
    irq_.legacy.shared_handler->RemoveDevice(*this);

    /* Release any handler storage and reset all of our bookkeeping */
    ResetCommonIrqBookkeeping();
}

/******************************************************************************
 *
 * MSI IRQ mode routines.
 *
 ******************************************************************************/
bool PcieDevice::MaskUnmaskMsiIrqLocked(uint irq_id, bool mask) {
    DEBUG_ASSERT(irq_.mode == PCIE_IRQ_MODE_MSI);
    DEBUG_ASSERT(irq_id < irq_.handler_count);
    DEBUG_ASSERT(irq_.handlers);

    pcie_irq_handler_state_t& hstate = irq_.handlers[irq_id];
    DEBUG_ASSERT(hstate.lock.IsHeld());

    /* Internal code should not be calling this function if they want to mask
     * the interrupt, but it is not possible to do so. */
    DEBUG_ASSERT(!mask ||
                 bus_drv_.platform().supports_msi_masking() ||
                 irq_.msi->has_pvm());

    /* If we can mask at the PCI device level, do so. */
    if (irq_.msi->has_pvm()) {
        DEBUG_ASSERT(irq_id < PCIE_MAX_MSI_IRQS);
        uint32_t  val  = cfg_->Read(irq_.msi->mask_bits_reg());
        if (mask) val |=  (static_cast<uint32_t>(1u) << irq_id);
        else      val &= ~(static_cast<uint32_t>(1u) << irq_id);
        cfg_->Write(irq_.msi->mask_bits_reg(), val);
    }

    /* If we can mask at the platform interrupt controller level, do so. */
    DEBUG_ASSERT(irq_.msi->irq_block_.allocated);
    DEBUG_ASSERT(irq_id < irq_.msi->irq_block_.num_irq);
    if (bus_drv_.platform().supports_msi_masking())
        bus_drv_.platform().MaskUnmaskMsi(&irq_.msi->irq_block_, irq_id, mask);

    bool ret = hstate.masked;
    hstate.masked = mask;
    return ret;
}

zx_status_t PcieDevice::MaskUnmaskMsiIrq(uint irq_id, bool mask) {
    if (irq_id >= irq_.handler_count)
        return ZX_ERR_INVALID_ARGS;

    /* If a mask is being requested, and we cannot mask at either the platform
     * interrupt controller or the PCI device level, tell the caller that the
     * operation is unsupported. */
    if (mask && !bus_drv_.platform().supports_msi_masking() && !irq_.msi->has_pvm())
        return ZX_ERR_NOT_SUPPORTED;

    DEBUG_ASSERT(irq_.handlers);

    {
        AutoSpinLock handler_lock(&irq_.handlers[irq_id].lock);
        MaskUnmaskMsiIrqLocked(irq_id, mask);
    }

    return ZX_OK;
}

void PcieDevice::MaskAllMsiVectors() {
    DEBUG_ASSERT(irq_.msi);
    DEBUG_ASSERT(irq_.msi->is_valid());

    for (uint i = 0; i < irq_.handler_count; i++)
        MaskUnmaskMsiIrq(i, true);

    /* In theory, this should not be needed as all of the relevant bits should
     * have already been masked during the calls to MaskUnmaskMsiIrq.  Just to
     * be careful, however, we explicitly mask all of the upper bits as well. */
    if (irq_.msi->has_pvm())
        cfg_->Write(irq_.msi->mask_bits_reg(), 0xFFFFFFFF);
}

void PcieDevice::SetMsiTarget(uint64_t tgt_addr, uint32_t tgt_data) {
    DEBUG_ASSERT(irq_.msi);
    DEBUG_ASSERT(irq_.msi->is_valid());
    DEBUG_ASSERT(irq_.msi->is64Bit() || !(tgt_addr >> 32));
    DEBUG_ASSERT(!(tgt_data >> 16));

    /* Make sure MSI is disabled_ and all vectors masked (if possible) before
     * changing the target address and data. */
    SetMsiEnb(false);
    MaskAllMsiVectors();

    /* lower bits of the address register are common to all forms of the MSI
     * capability structure.  Upper address bits and data position depend on
     * whether this is a 64 bit or 32 bit version */
    cfg_->Write(irq_.msi->addr_reg(), static_cast<uint32_t>(tgt_addr & 0xFFFFFFFF));
    if (irq_.msi->is64Bit()) {
        cfg_->Write(irq_.msi->addr_upper_reg(), static_cast<uint32_t>(tgt_addr >> 32));
    }
    cfg_->Write(irq_.msi->data_reg(), static_cast<uint16_t>(tgt_data & 0xFFFF));
}

void PcieDevice::FreeMsiBlock() {
    /* If no block has been allocated, there is nothing to do */
    if (!irq_.msi->irq_block_.allocated)
        return;

    DEBUG_ASSERT(bus_drv_.platform().supports_msi());

    /* Mask the IRQ at the platform interrupt controller level if we can, and
     * unregister any registered handler. */
    const msi_block_t* b = &irq_.msi->irq_block_;
    for (uint i = 0; i < b->num_irq; i++) {
        if (bus_drv_.platform().supports_msi_masking()) {
            bus_drv_.platform().MaskUnmaskMsi(b, i, true);
        }
        bus_drv_.platform().RegisterMsiHandler(b, i, nullptr, nullptr);
    }

    /* Give the block of IRQs back to the plaform */
    bus_drv_.platform().FreeMsiBlock(&irq_.msi->irq_block_);
    DEBUG_ASSERT(!irq_.msi->irq_block_.allocated);
}

void PcieDevice::SetMsiMultiMessageEnb(uint requested_irqs) {
    DEBUG_ASSERT(irq_.msi);
    DEBUG_ASSERT(irq_.msi->is_valid());
    DEBUG_ASSERT((requested_irqs >= 1) && (requested_irqs <= PCIE_MAX_MSI_IRQS));

    uint log2 = log2_uint_ceil(requested_irqs);

    DEBUG_ASSERT(log2 <= 5);
    DEBUG_ASSERT(!log2 || ((0x1u << (log2 - 1)) < requested_irqs));
    DEBUG_ASSERT((0x1u << log2) >= requested_irqs);

    cfg_->Write(irq_.msi->ctrl_reg(),
            PCIE_CAP_MSI_CTRL_SET_MME(log2, cfg_->Read(irq_.msi->ctrl_reg())));
}

void PcieDevice::LeaveMsiIrqMode() {
    /* Disable MSI, mask all vectors and zero out the target */
    SetMsiTarget(0x0, 0x0);

    /* Return any allocated irq_ block to the platform, unregistering with
     * the interrupt controller and synchronizing with the dispatchers in
     * the process. */
    FreeMsiBlock();

    /* Reset our common state, free any allocated handlers */
    ResetCommonIrqBookkeeping();
}

zx_status_t PcieDevice::EnterMsiIrqMode(uint requested_irqs) {
    DEBUG_ASSERT(requested_irqs);

    zx_status_t res = ZX_OK;

    // We cannot go into MSI mode if we don't support MSI at all, or we don't
    // support the number of IRQs requested
    if (!irq_.msi                             ||
        !irq_.msi->is_valid()                 ||
        !bus_drv_.platform().supports_msi()   ||
        (requested_irqs > irq_.msi->max_irqs()))
        return ZX_ERR_NOT_SUPPORTED;

    // If we support PVM, make sure that we are completely masked before
    // attempting to allocate the block of IRQs.
    bool initially_masked;
    if (irq_.msi->has_pvm()) {
        cfg_->Write(irq_.msi->mask_bits_reg(), 0xFFFFFFFF);
        initially_masked = true;
    } else {
        // If we cannot mask at the PCI level, then our IRQs will be initially
        // masked only if the platform supports masking at the interrupt
        // controller level.
        initially_masked = bus_drv_.platform().supports_msi_masking();
    }

    /* Ask the platform for a chunk of MSI compatible IRQs */
    DEBUG_ASSERT(!irq_.msi->irq_block_.allocated);
    res = bus_drv_.platform().AllocMsiBlock(requested_irqs,
                                            irq_.msi->is64Bit(),
                                            false,  /* is_msix == false */
                                            &irq_.msi->irq_block_);
    if (res != ZX_OK) {
        LTRACEF("Failed to allocate a block of %u MSI IRQs for device "
                "%02x:%02x.%01x (res %d)\n",
                requested_irqs, bus_id_, dev_id_, func_id_, res);
        goto bailout;
    }

    /* Allocate our handler table */
    res = AllocIrqHandlers(requested_irqs, initially_masked);
    if (res != ZX_OK)
        goto bailout;

    /* Record our new IRQ mode */
    irq_.mode = PCIE_IRQ_MODE_MSI;

    /* Program the target write transaction into the MSI registers.  As a side
     * effect, this will ensure that...
     *
     * 1) MSI mode has been disabled_ at the top level
     * 2) Each IRQ has been masked at system level (if supported)
     * 3) Each IRQ has been masked at the PCI PVM level (if supported)
     */
    DEBUG_ASSERT(irq_.msi->irq_block_.allocated);
    SetMsiTarget(irq_.msi->irq_block_.tgt_addr, irq_.msi->irq_block_.tgt_data);

    /* Properly program the multi-message enable field in the control register */
    SetMsiMultiMessageEnb(requested_irqs);

    /* Register each IRQ with the dispatcher */
    DEBUG_ASSERT(irq_.handler_count <= irq_.msi->irq_block_.num_irq);
    for (uint i = 0; i < irq_.handler_count; ++i) {
        bus_drv_.platform().RegisterMsiHandler(&irq_.msi->irq_block_,
                                               i,
                                               PcieDevice::MsiIrqHandlerThunk,
                                               irq_.handlers + i);
    }

    /* Enable MSI at the top level */
    SetMsiEnb(true);

bailout:
    if (res != ZX_OK)
        LeaveMsiIrqMode();

    return res;
}

void PcieDevice::MsiIrqHandler(pcie_irq_handler_state_t& hstate) {
    DEBUG_ASSERT(irq_.msi);
    /* No need to save IRQ state; we are in an IRQ handler at the moment. */
    AutoSpinLockNoIrqSave handler_lock(&hstate.lock);

    /* Mask our IRQ if we can. */
    bool was_masked;
    if (bus_drv_.platform().supports_msi_masking() || irq_.msi->has_pvm()) {
        was_masked = MaskUnmaskMsiIrqLocked(hstate.pci_irq_id, true);
    } else {
        DEBUG_ASSERT(!hstate.masked);
        was_masked = false;
    }

    /* If the IRQ was masked or the handler removed by the time we got here,
     * leave the IRQ masked, unlock and get out. */
    if (was_masked || !hstate.handler)
        return;

    /* Dispatch */
    pcie_irq_handler_retval_t irq_ret = hstate.handler(*this, hstate.pci_irq_id, hstate.ctx);

    /* Re-enable the IRQ if asked to do so */
    if (!(irq_ret & PCIE_IRQRET_MASK))
        MaskUnmaskMsiIrqLocked(hstate.pci_irq_id, false);
}

interrupt_eoi PcieDevice::MsiIrqHandlerThunk(void *arg) {
    DEBUG_ASSERT(arg);
    auto& hstate = *(reinterpret_cast<pcie_irq_handler_state_t*>(arg));
    DEBUG_ASSERT(hstate.dev);
    hstate.dev->MsiIrqHandler(hstate);
    return IRQ_EOI_DEACTIVATE;
}

/******************************************************************************
 *
 * Internal implementation of the Kernel facing API.
 *
 ******************************************************************************/
zx_status_t PcieDevice::QueryIrqModeCapabilitiesLocked(pcie_irq_mode_t mode,
                                                       pcie_irq_mode_caps_t* out_caps) const {
    DEBUG_ASSERT(plugged_in_);
    DEBUG_ASSERT(dev_lock_.IsHeld());
    DEBUG_ASSERT(out_caps);

    memset(out_caps, 0, sizeof(*out_caps));

    switch (mode) {
    // All devices always support "DISABLED".  No need to set the max_irqs to
    // zero or the PVM supported flag to false, the memset has taken care of
    // this for us already.
    case PCIE_IRQ_MODE_DISABLED:
        return ZX_OK;

    case PCIE_IRQ_MODE_LEGACY:
        if (!irq_.legacy.pin)
            return ZX_ERR_NOT_SUPPORTED;

        out_caps->max_irqs = 1;
        out_caps->per_vector_masking_supported = true;
        break;

    case PCIE_IRQ_MODE_MSI:
        /* If the platform does not support MSI, then we don't support MSI,
         * even if the device does. */
        if (!bus_drv_.platform().supports_msi())
            return ZX_ERR_NOT_SUPPORTED;

        /* If the device supports MSI, it will have a pointer to the control
         * structure in config. */
        if (!irq_.msi || !irq_.msi->is_valid())
            return ZX_ERR_NOT_SUPPORTED;

        /* We support PVM if either the device does, or if the platform is
         * capable of masking and unmasking individual IRQs from an MSI block
         * allocation. */
        out_caps->max_irqs = irq_.msi->max_irqs();
        out_caps->per_vector_masking_supported = irq_.msi->has_pvm()
                                              || (bus_drv_.platform().supports_msi_masking());
        break;

    case PCIE_IRQ_MODE_MSI_X:
        /* If the platform does not support MSI, then we don't support MSI,
         * even if the device does. */
        if (!bus_drv_.platform().supports_msi())
            return ZX_ERR_NOT_SUPPORTED;

        /* TODO(johngro) : finish MSI-X implementation. */
        return ZX_ERR_NOT_SUPPORTED;

    default:
        return ZX_ERR_INVALID_ARGS;
    }

    return ZX_OK;
}

zx_status_t PcieDevice::GetIrqModeLocked(pcie_irq_mode_info_t* out_info) const {
    DEBUG_ASSERT(plugged_in_);
    DEBUG_ASSERT(dev_lock_.IsHeld());
    DEBUG_ASSERT(out_info);

    out_info->mode                = irq_.mode;
    out_info->max_handlers        = irq_.handler_count;
    out_info->registered_handlers = irq_.registered_handler_count;

    return ZX_OK;
}

zx_status_t PcieDevice::SetIrqModeLocked(pcie_irq_mode_t mode, uint requested_irqs) {
    DEBUG_ASSERT(plugged_in_);
    DEBUG_ASSERT(dev_lock_.IsHeld());

    // Ensure the mode selection is valid
    if (mode >= PCIE_IRQ_MODE_COUNT) {
        return ZX_ERR_INVALID_ARGS;
    }

    // We need a valid number of IRQs for any mode that isn't strictly
    // disabling the device's IRQs.
    if (mode != PCIE_IRQ_MODE_DISABLED && requested_irqs < 1) {
        return ZX_ERR_INVALID_ARGS;
    }

    // Leave our present IRQ mode
    switch (irq_.mode) {
    case PCIE_IRQ_MODE_LEGACY:
        DEBUG_ASSERT(list_in_list(&irq_.legacy.shared_handler_node));
        LeaveLegacyIrqMode();
        break;

    case PCIE_IRQ_MODE_MSI:
        DEBUG_ASSERT(irq_.msi);
        DEBUG_ASSERT(irq_.msi->is_valid());
        DEBUG_ASSERT(irq_.msi->irq_block_.allocated);
        LeaveMsiIrqMode();
        break;

        // Right now, there should be no way to get into MSI-X mode
    case PCIE_IRQ_MODE_MSI_X:
        return ZX_ERR_NOT_SUPPORTED;

    // If we're disabled we have no work to do besides some sanity checks
    case PCIE_IRQ_MODE_DISABLED:
    default:
        DEBUG_ASSERT(!irq_.handlers);
        DEBUG_ASSERT(!irq_.handler_count);
        break;
    }

    // At this point we should be in the disabled state and can enable
    // the requested IRQ mode.
    switch (mode) {
    case PCIE_IRQ_MODE_DISABLED: return ZX_OK;
    case PCIE_IRQ_MODE_LEGACY: return EnterLegacyIrqMode(requested_irqs);
    case PCIE_IRQ_MODE_MSI:    return EnterMsiIrqMode   (requested_irqs);
    default:                   return ZX_ERR_NOT_SUPPORTED;
    }
}

zx_status_t PcieDevice::RegisterIrqHandlerLocked(uint irq_id,
                                              pcie_irq_handler_fn_t handler,
                                              void* ctx) {
    DEBUG_ASSERT(plugged_in_);
    DEBUG_ASSERT(dev_lock_.IsHeld());

    /* Cannot register a handler if we are currently disabled_ */
    if (irq_.mode == PCIE_IRQ_MODE_DISABLED)
        return ZX_ERR_BAD_STATE;

    DEBUG_ASSERT(irq_.handlers);
    DEBUG_ASSERT(irq_.handler_count);

    /* Make sure that the IRQ ID is within range */
    if (irq_id >= irq_.handler_count)
        return ZX_ERR_INVALID_ARGS;

    /* Looks good, register (or unregister the handler) and we are done. */
    pcie_irq_handler_state_t& hstate = irq_.handlers[irq_id];

    /* Update our registered handler bookkeeping.  Perform some sanity checks as we do so */
    if (hstate.handler) {
        DEBUG_ASSERT(irq_.registered_handler_count);
        if (!handler)
            irq_.registered_handler_count--;
    } else {
        if (handler)
            irq_.registered_handler_count++;
    }
    DEBUG_ASSERT(irq_.registered_handler_count <= irq_.handler_count);

    {
        AutoSpinLock handler_lock(&hstate.lock);
        hstate.handler = handler;
        hstate.ctx     = handler ? ctx : nullptr;
    }

    return ZX_OK;
}

zx_status_t PcieDevice::MaskUnmaskIrqLocked(uint irq_id, bool mask) {
    DEBUG_ASSERT(plugged_in_);
    DEBUG_ASSERT(dev_lock_.IsHeld());

    /* Cannot manipulate mask status while in the DISABLED state */
    if (irq_.mode == PCIE_IRQ_MODE_DISABLED)
        return ZX_ERR_BAD_STATE;

    DEBUG_ASSERT(irq_.handlers);
    DEBUG_ASSERT(irq_.handler_count);

    /* Make sure that the IRQ ID is within range */
    if (irq_id >= irq_.handler_count)
        return ZX_ERR_INVALID_ARGS;

    /* If we are unmasking (enabling), then we need to make sure that there is a
     * handler in place for the IRQ we are enabling. */
    pcie_irq_handler_state_t& hstate = irq_.handlers[irq_id];
    if (!mask && !hstate.handler)
        return ZX_ERR_BAD_STATE;

    /* OK, everything looks good.  Go ahead and make the change based on the
     * mode we are currently in. */
    switch (irq_.mode) {
    case PCIE_IRQ_MODE_LEGACY: return MaskUnmaskLegacyIrq(mask);
    case PCIE_IRQ_MODE_MSI:    return MaskUnmaskMsiIrq(irq_id, mask);
    case PCIE_IRQ_MODE_MSI_X:  return ZX_ERR_NOT_SUPPORTED;
    default:
        DEBUG_ASSERT(false); /* This should be un-possible! */
        return ZX_ERR_INTERNAL;
    }

    return ZX_OK;
}

/******************************************************************************
 *
 * Kernel API; prototypes in dev/pcie_irqs.h
 *
 ******************************************************************************/
zx_status_t PcieDevice::QueryIrqModeCapabilities(pcie_irq_mode_t mode,
                                              pcie_irq_mode_caps_t* out_caps) const {
    if (!out_caps)
        return ZX_ERR_INVALID_ARGS;

    AutoLock dev_lock(&dev_lock_);

    return (plugged_in_ && !disabled_)
        ? QueryIrqModeCapabilitiesLocked(mode, out_caps)
        : ZX_ERR_BAD_STATE;
}

zx_status_t PcieDevice::GetIrqMode(pcie_irq_mode_info_t* out_info) const {
    if (!out_info)
        return ZX_ERR_INVALID_ARGS;

    AutoLock dev_lock(&dev_lock_);

    return (plugged_in_ && !disabled_)
        ? GetIrqModeLocked(out_info)
        : ZX_ERR_BAD_STATE;
}

zx_status_t PcieDevice::SetIrqMode(pcie_irq_mode_t mode, uint requested_irqs) {
    AutoLock dev_lock(&dev_lock_);

    return ((mode == PCIE_IRQ_MODE_DISABLED) || (plugged_in_ && !disabled_))
        ? SetIrqModeLocked(mode, requested_irqs)
        : ZX_ERR_BAD_STATE;
}

zx_status_t PcieDevice::RegisterIrqHandler(uint irq_id, pcie_irq_handler_fn_t handler, void* ctx) {
    AutoLock dev_lock(&dev_lock_);

    return (plugged_in_ && !disabled_)
        ? RegisterIrqHandlerLocked(irq_id, handler, ctx)
        : ZX_ERR_BAD_STATE;
}

zx_status_t PcieDevice::MaskUnmaskIrq(uint irq_id, bool mask) {
    AutoLock dev_lock(&dev_lock_);

    return (mask || (plugged_in_ && !disabled_))
        ? MaskUnmaskIrqLocked(irq_id, mask)
        : ZX_ERR_BAD_STATE;
}


// Map from a device's interrupt pin ID to the proper system IRQ ID.  Follow the
// PCIe graph up to the root, swizzling as we traverse PCIe switches,
// PCIe-to-PCI bridges, and native PCI-to-PCI bridges.  Once we hit the root,
// perform the final remapping using the platform supplied remapping routine.
//
// Platform independent swizzling behavior is documented in the PCIe base
// specification in section 2.2.8.1 and Table 2-20.
//
// Platform dependent remapping is an exercise for the reader.  FWIW: PC
// architectures use the _PRT tables in ACPI to perform the remapping.
//
zx_status_t PcieDevice::MapPinToIrqLocked(fbl::RefPtr<PcieUpstreamNode>&& upstream) {
    DEBUG_ASSERT(dev_lock_.IsHeld());

    if (!legacy_irq_pin() || (legacy_irq_pin() > PCIE_MAX_LEGACY_IRQ_PINS))
        return ZX_ERR_BAD_STATE;

    auto dev = fbl::WrapRefPtr(this);
    uint pin = legacy_irq_pin() - 1;  // Change to 0s indexing

    // Walk up the PCI/PCIe tree, applying the swizzling rules as we go.  Stop
    // when we reach the device which is hanging off of the root bus/root
    // complex.  At this point, platform specific swizzling takes over.
    while ((upstream != nullptr) &&
           (upstream->type() == PcieUpstreamNode::Type::BRIDGE)) {
        // TODO(johngro) : Eliminate the null-check of bridge below.  Currently,
        // it is needed because we have gcc/g++'s "null-dereference" warning
        // turned on, and because of the potentially offsetting nature of static
        // casting, the compiler cannot be sure that bridge is non-null, just
        // because upstream was non-null (check in the while predicate, above).
        // Even adding explicit checks to the Downcast method in RefPtr<> does
        // not seem to satisfy it.
        //
        // Some potential future options include...
        // 1) Change this to DEBUG_ASSERT and turn off the null-dereference
        //    warning in release builds.
        // 2) Wait until GCC becomes smart enough to figure this out.
        // 3) Switch completely to clang (assuming that clang does not have
        //    similar problems).
        auto bridge = fbl::RefPtr<PcieBridge>::Downcast(ktl::move(upstream));
        if (bridge == nullptr)
            return ZX_ERR_INTERNAL;

        // We need to swizzle every time we pass through...
        // 1) A PCI-to-PCI bridge (real or virtual)
        // 2) A PCIe-to-PCI bridge
        // 3) The Upstream port of a switch.
        //
        // We do NOT swizzle when we pass through...
        // 1) A root port hanging off the root complex. (any swizzling here is up
        //    to the platform implementation)
        // 2) A Downstream switch port.  Since downstream PCIe switch ports are
        //    only permitted to have a single device located at position 0 on
        //    their "bus", it does not really matter if we do the swizzle or
        //    not, since it would turn out to be an identity transformation
        //    anyway.
        switch (bridge->pcie_device_type()) {
            // UNKNOWN devices are devices which did not have a PCI Express
            // Capabilities structure in their capabilities list.  Since every
            // device we pass through on the way up the tree should be a device
            // with a Type 1 header, these should be PCI-to-PCI bridges (real or
            // virtual)
            case PCIE_DEVTYPE_UNKNOWN:
            case PCIE_DEVTYPE_SWITCH_UPSTREAM_PORT:
            case PCIE_DEVTYPE_PCIE_TO_PCI_BRIDGE:
            case PCIE_DEVTYPE_PCI_TO_PCIE_BRIDGE:
                pin = (pin + dev->dev_id()) % PCIE_MAX_LEGACY_IRQ_PINS;
                break;

            default:
                break;
        }

        // Climb one branch higher up the tree
        dev = ktl::move(bridge);
        upstream = dev->GetUpstream();
    }

    // If our upstream is ever null as we climb the tree, then something must
    // have been unplugged as we were climbing.
    if (upstream == nullptr)
        return ZX_ERR_BAD_STATE;

    // We have hit root of the tree.  Something is very wrong if our
    // UpstreamNode is not, in fact, a root.
    if (upstream->type() != PcieUpstreamNode::Type::ROOT) {
        TRACEF("Failed to map legacy pin to platform IRQ ID for dev "
               "%02x:%02x.%01x (pin %u).  Top of the device tree "
               "(managed bus ID 0x%02x) does not appear to be either a root or a "
               "bridge! (type %u)\n",
               bus_id_, dev_id_, func_id_, irq_.legacy.pin,
               upstream->managed_bus_id(), static_cast<uint>(upstream->type()));
        return ZX_ERR_BAD_STATE;
    }

    // TODO(johngro) : Eliminate the null-check of root below.  See the TODO for
    // the downcast of upstream -> bridge above for details.
    auto root = fbl::RefPtr<PcieRoot>::Downcast(ktl::move(upstream));
    if (root == nullptr)
        return ZX_ERR_INTERNAL;
    return root->Swizzle(dev->dev_id(), dev->func_id(), pin, &irq_.legacy.irq_id);
}

zx_status_t PcieDevice::InitLegacyIrqStateLocked(PcieUpstreamNode& upstream) {
    DEBUG_ASSERT(dev_lock_.IsHeld());
    DEBUG_ASSERT(cfg_);
    DEBUG_ASSERT(irq_.legacy.shared_handler == nullptr);

    // Make certain that the device's legacy IRQ (if any) has been disabled.
    ModifyCmdLocked(0u, PCIE_CFG_COMMAND_INT_DISABLE);

    // Does config say that we have a legacy IRQ pin?  If so use the bus driver
    // to map it to the system IRQ ID, then grab a hold of the shared legacy IRQ
    // handler.
    irq_.legacy.pin = cfg_->Read(PciConfig::kInterruptPin);
    if (irq_.legacy.pin) {
        zx_status_t res = MapPinToIrqLocked(fbl::RefPtr<PcieUpstreamNode>(&upstream));
        if (res != ZX_OK) {
            TRACEF("Failed to map legacy pin to platform IRQ ID for "
                   "dev %02x:%02x.%01x (pin %u)\n",
                   bus_id_, dev_id_, func_id_,
                   irq_.legacy.pin);
            return res;
        }

        irq_.legacy.shared_handler = bus_drv_.FindLegacyIrqHandler(irq_.legacy.irq_id);
        if (irq_.legacy.shared_handler == nullptr) {
            TRACEF("Failed to find or create shared legacy IRQ handler for "
                   "dev %02x:%02x.%01x (pin %u, irq_id %u)\n",
                   bus_id_, dev_id_, func_id_,
                   irq_.legacy.pin, irq_.legacy.irq_id);
            return ZX_ERR_NO_RESOURCES;
        }
    }

    return ZX_OK;
}

void PcieBusDriver::ShutdownIrqs() {
    /* Shut off all of our legacy IRQs and free all of our bookkeeping */
    AutoLock lock(&legacy_irq_list_lock_);
    legacy_irq_list_.clear();
}

fbl::RefPtr<SharedLegacyIrqHandler> PcieBusDriver::FindLegacyIrqHandler(uint irq_id) {
    /* Search to see if we have already created a shared handler for this system
     * level IRQ id already */
    AutoLock lock(&legacy_irq_list_lock_);

    auto iter = legacy_irq_list_.begin();
    while (iter != legacy_irq_list_.end()) {
        if (irq_id == iter->irq_id())
            return iter.CopyPointer();
        ++iter;
    }

    auto handler = SharedLegacyIrqHandler::Create(irq_id);
    if (handler != nullptr)
        legacy_irq_list_.push_front(handler);

    return handler;
}