xref: /system/dev/usb/xhci/xdc-transfer.cpp

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <ddk/debug.h>

#include "xdc.h"
#include "xdc-transfer.h"

// Reads a range of bits from an integer.
#define READ_FIELD(i, start, bits) (((i) >> (start)) & ((1 << (bits)) - 1))

static void xdc_ring_doorbell(xdc_t* xdc, xdc_endpoint_t* ep) {
    uint8_t doorbell_val = ep->direction == USB_DIR_IN ? DCDB_DB_EP_IN : DCDB_DB_EP_OUT;
    XHCI_SET_BITS32(&xdc->debug_cap_regs->dcdb, DCDB_DB_START, DCDB_DB_BITS, doorbell_val);
}

// Stores the value of the Dequeue Pointer into out_dequeue.
// Returns ZX_OK if successful, or ZX_ERR_BAD_STATE if the endpoint was not in the Stopped state.
static zx_status_t xdc_get_dequeue_ptr_locked(xdc_t* xdc, xdc_endpoint_t* ep,
                                              uint64_t* out_dequeue) __TA_REQUIRES(xdc->lock) {
    if (ep->state != XDC_EP_STATE_STOPPED) {
        zxlogf(ERROR, "tried to read dequeue pointer of %s EP while not stopped, state is: %d\n",
               ep->name, ep->state);
        return ZX_ERR_BAD_STATE;
    }
    xdc_context_data_t* ctx = xdc->context_data;
    xhci_endpoint_context_t* epc = ep->direction == USB_DIR_OUT ? &ctx->out_epc : &ctx->in_epc;

    uint64_t dequeue_ptr_hi = XHCI_READ32(&epc->tr_dequeue_hi);
    uint32_t dequeue_ptr_lo = XHCI_READ32(&epc->epc2) & EP_CTX_TR_DEQUEUE_LO_MASK;
    *out_dequeue = (dequeue_ptr_hi << 32 | dequeue_ptr_lo);
    return ZX_OK;
}

// Returns ZX_OK if the request was scheduled successfully, or ZX_ERR_SHOULD_WAIT
// if we ran out of TRBs.
static zx_status_t xdc_schedule_transfer_locked(xdc_t* xdc, xdc_endpoint_t* ep,
                                                usb_request_t* req) __TA_REQUIRES(xdc->lock) {
    xhci_transfer_ring_t* ring = &ep->transfer_ring;

    // Need to clean the cache for both IN and OUT transfers, invalidate only for IN.
    if (ep->direction == USB_DIR_IN) {
        usb_request_cache_flush_invalidate(req, 0, req->header.length);
    } else {
        usb_request_cache_flush(req, 0, req->header.length);
    }

    zx_status_t status = xhci_queue_data_trbs(ring, &ep->transfer_state, req,
                                              0 /* interrupter */, false /* isochronous */);
    if (status != ZX_OK) {
        return status;
    }

    // If we get here, then we are ready to ring the doorbell.
    // Save the ring position so we can update the ring dequeue ptr once the transfer completes.
    xdc_req_internal_t* req_int = USB_REQ_TO_XDC_INTERNAL(req, sizeof(usb_request_t));
    req_int->context = ring->current;
    xdc_ring_doorbell(xdc, ep);

    return ZX_OK;
}

// Schedules any queued requests on the endpoint's transfer ring, until we fill our
// transfer ring or have no more requests.
void xdc_process_transactions_locked(xdc_t* xdc, xdc_endpoint_t* ep) __TA_REQUIRES(xdc->lock) {
    uint64_t usb_req_size = sizeof(usb_request_t);
    zx_status_t status;
    while (1) {
        if (xhci_transfer_ring_free_trbs(&ep->transfer_ring) == 0) {
            // No available TRBs - need to wait for some to complete.
            return;
        }

        if (!ep->current_req) {
            // Start the next transaction in the queue.
            usb_request_t* req = xdc_req_list_remove_head(&ep->queued_reqs, usb_req_size);
            if (!req) {
                // No requests waiting.
                return;
            }
            xhci_transfer_state_init(&ep->transfer_state, req,
                                     USB_ENDPOINT_BULK, EP_CTX_MAX_PACKET_SIZE);
            status = xdc_req_list_add_tail(&ep->pending_reqs, req, usb_req_size);
            ZX_DEBUG_ASSERT(status == ZX_OK);
            ep->current_req = req;
        }

        usb_request_t* req = ep->current_req;
        status = xdc_schedule_transfer_locked(xdc, ep, req);
        if (status == ZX_ERR_SHOULD_WAIT) {
            // No available TRBs - need to wait for some to complete.
            return;
        } else {
            ep->current_req = nullptr;
        }
    }
}

zx_status_t xdc_queue_transfer(xdc_t* xdc, usb_request_t* req, bool in, bool is_ctrl_msg) {
    xdc_endpoint_t* ep = in ? &xdc->eps[IN_EP_IDX] : &xdc->eps[OUT_EP_IDX];

    mtx_lock(&xdc->lock);

    // We should always queue control messages unless there is an unrecoverable error.
    if (!is_ctrl_msg && (!xdc->configured || ep->state == XDC_EP_STATE_DEAD)) {
        mtx_unlock(&xdc->lock);
        return ZX_ERR_IO_NOT_PRESENT;
    }

    if (req->header.length > 0) {
        zx_status_t status = usb_request_physmap(req, xdc->bti_handle);
        if (status != ZX_OK) {
            zxlogf(ERROR, "%s: usb_request_physmap failed: %d\n", __FUNCTION__, status);
            mtx_unlock(&xdc->lock);
            return status;
        }
    }

    xdc_req_internal_t* req_int = USB_REQ_TO_XDC_INTERNAL(req, sizeof(usb_request_t));
    req_int->complete_cb.callback = in ? xdc_read_complete : xdc_write_complete;
    req_int->complete_cb.ctx = xdc;

    list_add_tail(&ep->queued_reqs, &req_int->node);

    // We can still queue requests for later while waiting for the xdc device to be configured,
    // or while the endpoint is halted. Before scheduling the TRBs however, we should wait
    // for the device to be configured, and/or the halt is cleared by DbC and we've cleaned
    // up the transfer ring.
    if (xdc->configured && ep->state == XDC_EP_STATE_RUNNING) {
        xdc_process_transactions_locked(xdc, ep);
    }

    mtx_unlock(&xdc->lock);

    return ZX_OK;
}

bool xdc_has_free_trbs(xdc_t* xdc, bool in) {
    mtx_lock(&xdc->lock);

    xdc_endpoint_t* ep = in ? &xdc->eps[IN_EP_IDX] : &xdc->eps[OUT_EP_IDX];
    bool has_trbs = xhci_transfer_ring_free_trbs(&ep->transfer_ring) > 0;

    mtx_unlock(&xdc->lock);
    return has_trbs;
}

zx_status_t xdc_restart_transfer_ring_locked(xdc_t* xdc, xdc_endpoint_t* ep) {
    // Once the DbC clears the halt flag for the endpoint, the address stored in the
    // TR Dequeue Pointer field is the next TRB to be executed (see XHCI Spec 7.6.4.3).
    // There seems to be no guarantee which TRB this will point to.
    //
    // The easiest way to deal with this is to convert all scheduled TRBs to NO-OPs,
    // and reschedule pending requests.

    uint64_t dequeue_ptr;
    zx_status_t status = xdc_get_dequeue_ptr_locked(xdc, ep, &dequeue_ptr);
    if (status != ZX_OK) {
        return status;
    }
    xhci_transfer_ring_t* ring = &ep->transfer_ring;
    xhci_trb_t* trb = xhci_transfer_ring_phys_to_trb(ring, dequeue_ptr);
    if (!trb) {
        zxlogf(ERROR, "no valid TRB corresponding to dequeue_ptr: %lu\n", dequeue_ptr);
        return ZX_ERR_BAD_STATE;
    }

    // Reset our copy of the dequeue pointer.
    xhci_set_dequeue_ptr(ring, trb);

    // Convert all pending TRBs on the transfer ring into NO-OPs TRBs.
    // ring->current is just after our last queued TRB.
    xhci_trb_t* last_trb = nullptr;
    while (trb != ring->current) {
        xhci_set_transfer_noop_trb(trb);
        last_trb = trb;
        trb = xhci_get_next_trb(ring, trb);
    }
    if (last_trb) {
        // Set IOC (Interrupt on Completion) on the last NO-OP TRB, so we know
        // when we can overwrite them in the transfer ring.
        uint32_t control = XHCI_READ32(&last_trb->control);
        XHCI_WRITE32(&last_trb->control, control | XFER_TRB_IOC);
    }
    // Restart the transfer ring.
    xdc_ring_doorbell(xdc, ep);
    ep->state = XDC_EP_STATE_RUNNING;

    // Requeue and reschedule the requests.
    usb_request_t* req;
    uint64_t usb_req_size = sizeof(usb_request_t);
    while ((req = xdc_req_list_remove_tail(&ep->pending_reqs, usb_req_size)) != nullptr) {
        status = xdc_req_list_add_head(&ep->queued_reqs, req, usb_req_size);
        ZX_DEBUG_ASSERT(status == ZX_OK);
    }
    xdc_process_transactions_locked(xdc, ep);
    return ZX_OK;
}

void xdc_handle_transfer_event_locked(xdc_t* xdc, xdc_poll_state_t* poll_state, xhci_trb_t* trb) {
    uint32_t control = XHCI_READ32(&trb->control);
    uint32_t status = XHCI_READ32(&trb->status);
    uint32_t ep_dev_ctx_idx = READ_FIELD(control, TRB_ENDPOINT_ID_START, TRB_ENDPOINT_ID_BITS);
    uint8_t xdc_ep_idx = ep_dev_ctx_idx == EP_IN_DEV_CTX_IDX ? IN_EP_IDX : OUT_EP_IDX;
    xdc_endpoint_t* ep = &xdc->eps[xdc_ep_idx];
    xhci_transfer_ring_t* ring = &ep->transfer_ring;
    uint64_t usb_req_size = sizeof(usb_request_t);

    uint32_t cc = READ_FIELD(status, EVT_TRB_CC_START, EVT_TRB_CC_BITS);
    uint32_t length = READ_FIELD(status, EVT_TRB_XFER_LENGTH_START, EVT_TRB_XFER_LENGTH_BITS);
    usb_request_t* req = nullptr;
    bool error = false;

    switch (cc) {
        case TRB_CC_SUCCESS:
        case TRB_CC_SHORT_PACKET:
            break;
        case TRB_CC_BABBLE_DETECTED_ERROR:
        case TRB_CC_USB_TRANSACTION_ERROR:
        case TRB_CC_TRB_ERROR:
        case TRB_CC_STALL_ERROR:
            zxlogf(ERROR, "xdc_handle_transfer_event: error condition code: %d\n", cc);
            error = true;
            break;
        default:
            zxlogf(ERROR, "xdc_handle_transfer_event: unexpected condition code %d\n", cc);
            error = true;
            break;
    }

    // Even though the main poll loop checks for changes in the halt registers,
    // it's possible we missed the halt register being set if the halt was cleared fast enough.
    if (error) {
        if (ep->state == XDC_EP_STATE_RUNNING) {
             xdc_endpoint_set_halt_locked(xdc, poll_state, ep);
        }
        ep->got_err_event = true;
        // We're going to requeue the transfer when we restart the transfer ring,
        // so nothing else to do.
        return;
    }

    if (control & EVT_TRB_ED) {
        // An Event Data TRB generated the completion event, so the TRB Pointer field
        // will contain the usb request pointer we previously stored.
        req = reinterpret_cast<usb_request_t*>(trb->ptr);
    } else {
        // Get the pointer to the TRB that generated the event.
        trb = xhci_read_trb_ptr(ring, trb);
        if (trb_get_type(trb) == TRB_TRANSFER_NOOP) {
            // If it's the NO-OP TRB we queued when dealing with the halt condition,
            // there won't be a corresponding usb request.
            zxlogf(TRACE, "xdc_handle_transfer_event: got a NO-OP TRB\n");
            xhci_set_dequeue_ptr(ring, xhci_get_next_trb(ring, trb));
            xdc_process_transactions_locked(xdc, ep);
            return;
        }

        // Look for the Event Data TRB which will have the usb request pointer.
        for (uint i = 0; i < TRANSFER_RING_SIZE && trb; i++) {
            if (trb_get_type(trb) == TRB_TRANSFER_EVENT_DATA) {
                req = reinterpret_cast<usb_request_t*>(trb->ptr);
                break;
            }
            trb = xhci_get_next_trb(ring, trb);
        }
    }

    if (!req) {
        zxlogf(ERROR, "xdc_handle_transfer_event: unable to find request to complete\n");
        return;
    }

    // Find the usb request in the pending list.
    bool found_req = false;
    usb_request_t* test;
    xdc_req_internal_t* test_int;
    list_for_every_entry(&ep->pending_reqs, test_int, xdc_req_internal_t, node) {
        test = XDC_INTERNAL_TO_USB_REQ(test_int, usb_req_size);
        if (test == req) {
            found_req = true;
            break;
        }
    }
    if (!found_req) {
        zxlogf(ERROR, "xdc_handle_transfer_event: ignoring event for completed transfer\n");
        return;
    }
    // Remove request from pending_reqs.
    list_delete(&test_int->node);

    // Update our copy of the dequeue_ptr to the TRB following this transaction.
    xhci_set_dequeue_ptr(ring, static_cast<xhci_trb_t*>(test_int->context));
    xdc_process_transactions_locked(xdc, ep);

    // Save the request to be completed later out of the lock.
    req->response.status = ZX_OK;
    req->response.actual = length;
    status = xdc_req_list_add_tail(&poll_state->completed_reqs, req, usb_req_size);
    ZX_DEBUG_ASSERT(status == ZX_OK);
}