bus/virtio/ethernet.cpp

// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "ethernet.h"

#include <assert.h>
#include <limits.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>

#include <ddk/debug.h>
#include <ddk/io-buffer.h>
#include <ddk/protocol/ethernet.h>
#include <fbl/algorithm.h>
#include <fbl/alloc_checker.h>
#include <fbl/auto_call.h>
#include <fbl/auto_lock.h>
#include <fbl/unique_ptr.h>
#include <pretty/hexdump.h>
#include <virtio/net.h>
#include <virtio/virtio.h>
#include <zircon/assert.h>
#include <zircon/status.h>
#include <zircon/types.h>

#include <utility>

#include "ring.h"
#include "trace.h"

// Enables/disables debugging info
#define LOCAL_TRACE 0

namespace virtio {

namespace {

// Specifies how many packets can fit in each of the receive and transmit
// backlogs.
const size_t kBacklog = 32;

// Specifies the maximum transfer unit we support and the maximum layer 1
// Ethernet packet header length.
const size_t kVirtioMtu = 1500;
const size_t kEthHeaderSizeBytes = 14;
const size_t kEthFrameSize = kVirtioMtu + kEthHeaderSizeBytes;

// Other constants determined by the values above and the memory architecture.
// The goal here is to allocate single-page I/O buffers.
const size_t kFrameSize = sizeof(virtio_net_hdr_t) + kEthFrameSize;
const size_t kFramesInBuf = PAGE_SIZE / kFrameSize;
const size_t kNumIoBufs = fbl::round_up(kBacklog * 2, kFramesInBuf) / kFramesInBuf;

const uint16_t kRxId = 0u;
const uint16_t kTxId = 1u;

// Strictly for convenience...
typedef struct vring_desc desc_t;

// Device bridge helpers
void virtio_net_unbind(void* ctx) {
    virtio::EthernetDevice* eth = static_cast<virtio::EthernetDevice*>(ctx);
    eth->Unbind();
}

void virtio_net_release(void* ctx) {
    fbl::unique_ptr<virtio::EthernetDevice> eth(static_cast<virtio::EthernetDevice*>(ctx));
    eth->Release();
}

zx_protocol_device_t kDeviceOps = {
    DEVICE_OPS_VERSION,
    nullptr, // get_protocol
    nullptr, // open
    nullptr, // openat
    nullptr, // close
    virtio_net_unbind,
    virtio_net_release,
    nullptr, // read
    nullptr, // write
    nullptr, // get_size
    nullptr, // ioctl
    nullptr, // suspend
    nullptr, // resume
    nullptr, // rxrpc
    nullptr, // rxmsg
};

// Protocol bridge helpers
zx_status_t virtio_net_query(void* ctx, uint32_t options, ethmac_info_t* info) {
    virtio::EthernetDevice* eth = static_cast<virtio::EthernetDevice*>(ctx);
    return eth->Query(options, info);
}

void virtio_net_stop(void* ctx) {
    virtio::EthernetDevice* eth = static_cast<virtio::EthernetDevice*>(ctx);
    eth->Stop();
}

zx_status_t virtio_net_start(void* ctx, const ethmac_ifc_t* ifc) {
    virtio::EthernetDevice* eth = static_cast<virtio::EthernetDevice*>(ctx);
    return eth->Start(ifc);
}

zx_status_t virtio_net_queue_tx(void* ctx, uint32_t options, ethmac_netbuf_t* netbuf) {
    virtio::EthernetDevice* eth = static_cast<virtio::EthernetDevice*>(ctx);
    return eth->QueueTx(options, netbuf);
}

static zx_status_t virtio_set_param(void* ctx, uint32_t param, int32_t value, const void* data,
                                    size_t data_size) {
    return ZX_ERR_NOT_SUPPORTED;
}

ethmac_protocol_ops_t kProtoOps = {
    virtio_net_query,
    virtio_net_stop,
    virtio_net_start,
    virtio_net_queue_tx,
    virtio_set_param,
    NULL, // get_bti not implemented because we don't have FEATURE_DMA
};

// I/O buffer helpers
zx_status_t InitBuffers(const zx::bti& bti, fbl::unique_ptr<io_buffer_t[]>* out) {
    zx_status_t rc;
    fbl::AllocChecker ac;
    fbl::unique_ptr<io_buffer_t[]> bufs(new (&ac) io_buffer_t[kNumIoBufs]);
    if (!ac.check()) {
        zxlogf(ERROR, "out of memory!\n");
        return ZX_ERR_NO_MEMORY;
    }
    memset(bufs.get(), 0, sizeof(io_buffer_t) * kNumIoBufs);
    size_t buf_size = kFrameSize * kFramesInBuf;
    for (uint16_t id = 0; id < kNumIoBufs; ++id) {
        if ((rc = io_buffer_init(&bufs[id], bti.get(), buf_size,
                                 IO_BUFFER_RW | IO_BUFFER_CONTIG)) != ZX_OK) {
            zxlogf(ERROR, "failed to allocate I/O buffers: %s\n", zx_status_get_string(rc));
            return rc;
        }
    }
    *out = std::move(bufs);
    return ZX_OK;
}

void ReleaseBuffers(fbl::unique_ptr<io_buffer_t[]> bufs) {
    if (!bufs) {
        return;
    }
    for (size_t i = 0; i < kNumIoBufs; ++i) {
        if (io_buffer_is_valid(&bufs[i])) {
            io_buffer_release(&bufs[i]);
        }
    }
}

// Frame access helpers
zx_off_t GetFrame(io_buffer_t** bufs, uint16_t ring_id, uint16_t desc_id) {
    uint16_t i = static_cast<uint16_t>(desc_id + ring_id * kBacklog);
    *bufs = &((*bufs)[i / kFramesInBuf]);
    return (i % kFramesInBuf) * kFrameSize;
}

void* GetFrameVirt(io_buffer_t* bufs, uint16_t ring_id, uint16_t desc_id) {
    zx_off_t offset = GetFrame(&bufs, ring_id, desc_id);
    uintptr_t vaddr = reinterpret_cast<uintptr_t>(io_buffer_virt(bufs));
    return reinterpret_cast<void*>(vaddr + offset);
}

zx_paddr_t GetFramePhys(io_buffer_t* bufs, uint16_t ring_id, uint16_t desc_id) {
    zx_off_t offset = GetFrame(&bufs, ring_id, desc_id);
    return io_buffer_phys(bufs) + offset;
}

virtio_net_hdr_t* GetFrameHdr(io_buffer_t* bufs, uint16_t ring_id, uint16_t desc_id) {
    return reinterpret_cast<virtio_net_hdr_t*>(GetFrameVirt(bufs, ring_id, desc_id));
}

uint8_t* GetFrameData(io_buffer_t* bufs, uint16_t ring_id, uint16_t desc_id, size_t hdr_size) {
    uintptr_t vaddr = reinterpret_cast<uintptr_t>(GetFrameHdr(bufs, ring_id, desc_id));
    return reinterpret_cast<uint8_t*>(vaddr + hdr_size);
}

} // namespace

EthernetDevice::EthernetDevice(zx_device_t* bus_device, zx::bti bti, fbl::unique_ptr<Backend> backend)
    : Device(bus_device, std::move(bti), std::move(backend)), rx_(this), tx_(this), bufs_(nullptr),
      unkicked_(0), ifc_({nullptr, nullptr}) {
}

EthernetDevice::~EthernetDevice() {
    LTRACE_ENTRY;
}

zx_status_t EthernetDevice::Init() {
    LTRACE_ENTRY;
    zx_status_t rc;
    if (mtx_init(&state_lock_, mtx_plain) != thrd_success ||
        mtx_init(&tx_lock_, mtx_plain) != thrd_success) {
        return ZX_ERR_NO_RESOURCES;
    }
    fbl::AutoLock lock(&state_lock_);

    // Reset the device and read our configuration
    DeviceReset();
    CopyDeviceConfig(&config_, sizeof(config_));
    LTRACEF("mac %02x:%02x:%02x:%02x:%02x:%02x\n", config_.mac[0], config_.mac[1], config_.mac[2],
            config_.mac[3], config_.mac[4], config_.mac[5]);
    LTRACEF("status %u\n", config_.status);
    LTRACEF("max_virtqueue_pairs  %u\n", config_.max_virtqueue_pairs);

    // Ack and set the driver status bit
    DriverStatusAck();

    virtio_hdr_len_ = sizeof(virtio_net_hdr_t);
    if (DeviceFeatureSupported(VIRTIO_F_VERSION_1)) {
      DriverFeatureAck(VIRTIO_F_VERSION_1);
    } else {
      // 5.1.6.1 Legacy Interface: Device Operation
      //
      // The legacy driver only presented num_buffers in the struct
      // virtio_net_hdr when VIRTIO_NET_F_MRG_RXBUF was negotiated; without
      // that feature the structure was 2 bytes shorter.
      virtio_hdr_len_ -= 2;
    }

    // TODO(aarongreen): Check additional features bits and ack/nak them
    rc = DeviceStatusFeaturesOk();
    if (rc != ZX_OK) {
        zxlogf(ERROR, "%s: Feature negotiation failed (%d)\n", tag(), rc);
        return rc;
    }

    // Plan to clean up unless everything goes right.
    auto cleanup = fbl::MakeAutoCall([this]() { Release(); });

    // Allocate I/O buffers and virtqueues.
    uint16_t num_descs = static_cast<uint16_t>(kBacklog & 0xffff);
    if ((rc = InitBuffers(bti_, &bufs_)) != ZX_OK || (rc = rx_.Init(kRxId, num_descs)) != ZX_OK ||
        (rc = tx_.Init(kTxId, num_descs)) != ZX_OK) {
        zxlogf(ERROR, "failed to allocate virtqueue: %s\n", zx_status_get_string(rc));
        return rc;
    }

    // Associate the I/O buffers with the virtqueue descriptors
    desc_t* desc = nullptr;
    uint16_t id;

    // For rx buffers, we queue a bunch of "reads" from the network that
    // complete when packets arrive.
    for (uint16_t i = 0; i < num_descs; ++i) {
        desc = rx_.AllocDescChain(1, &id);
        desc->addr = GetFramePhys(bufs_.get(), kRxId, id);
        desc->len = kFrameSize;
        desc->flags |= VRING_DESC_F_WRITE;
        LTRACE_DO(virtio_dump_desc(desc));
        rx_.SubmitChain(id);
    }

    // For tx buffers, we hold onto them until we need to send a packet.
    for (uint16_t id = 0; id < num_descs; ++id) {
        desc = tx_.DescFromIndex(id);
        desc->addr = GetFramePhys(bufs_.get(), kTxId, id);
        desc->len = 0;
        desc->flags &= static_cast<uint16_t>(~VRING_DESC_F_WRITE);
        LTRACE_DO(virtio_dump_desc(desc));
    }

    // Start the interrupt thread and set the driver OK status
    StartIrqThread();

    // Initialize the zx_device and publish us
    device_add_args_t args;
    memset(&args, 0, sizeof(args));
    args.version = DEVICE_ADD_ARGS_VERSION;
    args.name = "virtio-net";
    args.ctx = this;
    args.ops = &kDeviceOps;
    args.proto_id = ZX_PROTOCOL_ETHMAC;
    args.proto_ops = &kProtoOps;
    if ((rc = device_add(bus_device_, &args, &device_)) != ZX_OK) {
        zxlogf(ERROR, "failed to add device: %s\n", zx_status_get_string(rc));
        return rc;
    }
    // Give the rx buffers to the host
    rx_.Kick();

    // Woohoo! Driver should be ready.
    cleanup.cancel();
    DriverStatusOk();
    return ZX_OK;
}

void EthernetDevice::Release() {
    LTRACE_ENTRY;
    fbl::AutoLock lock(&state_lock_);
    ReleaseLocked();
}

void EthernetDevice::ReleaseLocked() {
    ifc_.ops = nullptr;
    ReleaseBuffers(std::move(bufs_));
    Device::Release();
}

void EthernetDevice::IrqRingUpdate() {
    LTRACE_ENTRY;
    // Lock to prevent changes to ifc_.
    {
        fbl::AutoLock lock(&state_lock_);
        if (!ifc_.ops) {
            return;
        }
        // Ring::IrqRingUpdate will call this lambda on each rx buffer filled by
        // the underlying device since the last IRQ.
        // Thread safety analysis is explicitly disabled as clang isn't able to determine that the
        // state_lock_ is  held when the lambda invoked.
        rx_.IrqRingUpdate([this](vring_used_elem* used_elem) TA_NO_THREAD_SAFETY_ANALYSIS {
            uint16_t id = static_cast<uint16_t>(used_elem->id & 0xffff);
            desc_t* desc = rx_.DescFromIndex(id);

            // Transitional driver does not merge rx buffers.
            if ((desc->flags & VRING_DESC_F_NEXT) != 0) {
                zxlogf(ERROR, "dropping rx packet; do not support descriptor chaining");
                while((desc->flags & VRING_DESC_F_NEXT)) {
                    uint16_t next_id = desc->next;
                    rx_.FreeDesc(id);
                    id = next_id;
                    desc = rx_.DescFromIndex(id);
                }
                return;
            }
            assert(used_elem->len <= desc->len);
            uint8_t* data = GetFrameData(bufs_.get(), kRxId, id, virtio_hdr_len_);
            size_t len = used_elem->len - virtio_hdr_len_;
            LTRACEF("Receiving %zu bytes:\n", len);
            LTRACE_DO(hexdump8_ex(data, len, 0));

            // Pass the data up the stack to the generic Ethernet driver
            ethmac_ifc_recv(&ifc_, data, len, 0);
            LTRACE_DO(virtio_dump_desc(desc));
            rx_.FreeDesc(id);
        });
    }

    // Now recycle the rx buffers.  As in Init(), this means queuing a bunch of
    // "reads" from the network that will complete when packets arrive.
    desc_t* desc = nullptr;
    uint16_t id;
    bool need_kick = false;
    while ((desc = rx_.AllocDescChain(1, &id))) {
        desc->len = kFrameSize;
        rx_.SubmitChain(id);
        need_kick = true;
    }

    // If we have re-queued any rx buffers, poke the virtqueue to pick them up.
    if (need_kick) {
        rx_.Kick();
    }
}

void EthernetDevice::IrqConfigChange() {
    LTRACE_ENTRY;
    fbl::AutoLock lock(&state_lock_);
    if (!ifc_.ops) {
        return;
    }

    // Re-read our configuration
    CopyDeviceConfig(&config_, sizeof(config_));
    ethmac_ifc_status(&ifc_, (config_.status & VIRTIO_NET_S_LINK_UP) ? ETHMAC_STATUS_ONLINE : 0);
}

zx_status_t EthernetDevice::Query(uint32_t options, ethmac_info_t* info) {
    LTRACE_ENTRY;
    if (options) {
        return ZX_ERR_INVALID_ARGS;
    }
    fbl::AutoLock lock(&state_lock_);
    if (info) {
        // TODO(aarongreen): Add info->features = GetFeatures();
        info->mtu = kVirtioMtu;
        info->netbuf_size = sizeof(ethmac_netbuf_t);
        memcpy(info->mac, config_.mac, sizeof(info->mac));
    }
    return ZX_OK;
}

void EthernetDevice::Stop() {
    LTRACE_ENTRY;
    fbl::AutoLock lock(&state_lock_);
    ifc_.ops = nullptr;
}

zx_status_t EthernetDevice::Start(const ethmac_ifc_t* ifc) {
    LTRACE_ENTRY;
    if (!ifc) {
        return ZX_ERR_INVALID_ARGS;
    }
    fbl::AutoLock lock(&state_lock_);
    if (!bufs_ || ifc_.ops) {
        return ZX_ERR_BAD_STATE;
    }
    ifc_ = *ifc;
    ethmac_ifc_status(&ifc_, (config_.status & VIRTIO_NET_S_LINK_UP) ? ETHMAC_STATUS_ONLINE : 0);
    return ZX_OK;
}

zx_status_t EthernetDevice::QueueTx(uint32_t options, ethmac_netbuf_t* netbuf) {
    LTRACE_ENTRY;
    const void* data = netbuf->data_buffer;
    size_t length = netbuf->data_size;
    // First, validate the packet
    if (!data || length > kEthFrameSize) {
        zxlogf(ERROR, "dropping packet; invalid packet\n");
        return ZX_ERR_INVALID_ARGS;
    }

    fbl::AutoLock lock(&tx_lock_);

    // Flush outstanding descriptors.  Ring::IrqRingUpdate will call this lambda
    // on each sent tx_buffer, allowing us to reclaim them.
    auto flush = [this](vring_used_elem* used_elem) {
        uint16_t id = static_cast<uint16_t>(used_elem->id & 0xffff);
        desc_t* desc = tx_.DescFromIndex(id);
        assert((desc->flags & VRING_DESC_F_NEXT) == 0);
        LTRACE_DO(virtio_dump_desc(desc));
        tx_.FreeDesc(id);
    };

    // Grab a free descriptor
    uint16_t id;
    desc_t* desc = tx_.AllocDescChain(1, &id);
    if (!desc) {
        tx_.IrqRingUpdate(flush);
        desc = tx_.AllocDescChain(1, &id);
    }
    if (!desc) {
        zxlogf(ERROR, "dropping packet; out of descriptors\n");
        return ZX_ERR_NO_RESOURCES;
    }

    // Add the data to be sent
    virtio_net_hdr_t* tx_hdr = GetFrameHdr(bufs_.get(), kTxId, id);
    memset(tx_hdr, 0, virtio_hdr_len_);

    // 5.1.6.2.1 Driver Requirements: Packet Transmission
    //
    // The driver MUST set num_buffers to zero.
    //
    // Implementation note: This field doesn't exist if neither
    // |VIRTIO_F_VERSION_1| or |VIRTIO_F_MRG_RXBUF| have been negotiated. Since
    // this field will be part of the payload without these features we elide
    // the check as we know the memory is valid and will soon be overwritten
    // with packet data.
    tx_hdr->num_buffers = 0;

    // If VIRTIO_NET_F_CSUM is not negotiated, the driver MUST set flags to
    // zero and SHOULD supply a fully checksummed packet to the device.
    tx_hdr->flags = 0;

    // If none of the VIRTIO_NET_F_HOST_TSO4, TSO6 or UFO options have been
    // negotiated, the driver MUST set gso_type to VIRTIO_NET_HDR_GSO_NONE.
    tx_hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;

    void* tx_buf = GetFrameData(bufs_.get(), kTxId, id, virtio_hdr_len_);
    memcpy(tx_buf, data, length);
    desc->len = static_cast<uint32_t>(virtio_hdr_len_ + length);

    // Submit the descriptor and notify the back-end.
    LTRACE_DO(virtio_dump_desc(desc));
    LTRACEF("Sending %zu bytes:\n", length);
    LTRACE_DO(hexdump8_ex(tx_buf, length, 0));
    tx_.SubmitChain(id);
    ++unkicked_;
    if ((options & ETHMAC_TX_OPT_MORE) == 0 || unkicked_ > kBacklog / 2) {
        tx_.Kick();
        unkicked_ = 0;
    }
    return ZX_OK;
}

} // namespace virtio