xref: /kernel/platform/pc/memory.cpp

// Copyright 2016 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#include <arch/x86/bootstrap16.h>
#include <arch/x86/feature.h>
#include <arch/x86/mmu.h>
#include <assert.h>
#include <dev/interrupt.h>
#include <efi/boot-services.h>
#include <err.h>
#include <fbl/algorithm.h>
#include <inttypes.h>
#include <lib/memory_limit.h>
#include <lk/init.h>
#include <platform.h>
#include <platform/pc/bootloader.h>
#include <string.h>
#include <trace.h>
#include <vm/vm.h>
#include <zircon/types.h>
#include <zircon/boot/e820.h>
#include <object/resource_dispatcher.h>

#include "platform_p.h"

#define LOCAL_TRACE 0

struct addr_range {
    uint64_t base;
    uint64_t size;
    bool is_mem;
};

/* Values that will store the largest low-memory contiguous address space
 * that we can let the PCIe bus driver use for allocations */
paddr_t pcie_mem_lo_base;
size_t pcie_mem_lo_size;

// These are used to track memory arenas found during boot so they can
// be exclusively reserved within the resource system after the heap
// has been initialized.
constexpr uint8_t kMaxReservedMmioEntries = 64;
typedef struct reserved_mmio_space {
    uint64_t base;
    size_t len;
    ResourceDispatcher::RefPtr dispatcher;;
} reserved_mmio_space_t;
reserved_mmio_space_t reserved_mmio_entries[kMaxReservedMmioEntries];
static uint8_t reserved_mmio_count = 0;

static void mark_mmio_region_to_reserve(uint64_t base, size_t len) {
    reserved_mmio_entries[reserved_mmio_count].base = base;
    reserved_mmio_entries[reserved_mmio_count].len = len;
    reserved_mmio_count++;
}

#define DEFAULT_MEMEND (16 * 1024 * 1024)

/* boot_addr_range_t is an iterator which iterates over address ranges from
 * the boot loader
 */
struct boot_addr_range;

typedef void (*boot_addr_range_advance_func)(
    struct boot_addr_range* range_struct);
typedef void (*boot_addr_range_reset_func)(
    struct boot_addr_range* range_struct);

typedef struct boot_addr_range {
    /* the base of the current address range */
    uint64_t base;
    /* the size of the current address range */
    uint64_t size;
    /* whether this range contains memory */
    int is_mem;
    /* whether this range is currently reset and invalid */
    int is_reset;

    /* private information for the advance function to keep its place */
    void* seq;
    /* a function which advances this iterator to the next address range */
    boot_addr_range_advance_func advance;
    /* a function which resets this range and its sequencing information */
    boot_addr_range_reset_func reset;
} boot_addr_range_t;

/* a utility function to reset the common parts of a boot_addr_range_t */
static void boot_addr_range_reset(boot_addr_range_t* range) {
    range->base = 0;
    range->size = 0;
    range->is_mem = 0;
    range->is_reset = 1;
}

/* this function uses the boot_addr_range_t iterator to walk through address
 * ranges described by the boot loader. it fills in the mem_arenas global
 * array with the ranges of memory it finds, compacted to the start of the
 * array. it returns the total count of arenas which have been populated.
 */
static zx_status_t mem_arena_init(boot_addr_range_t* range) {
    bool have_limit = (memory_limit_init() == ZX_OK);
    // Create the kernel's singleton for address space management
    // Set up a base arena template to use
    pmm_arena_info_t base_arena;
    snprintf(base_arena.name, sizeof(base_arena.name), "%s", "memory");
    base_arena.priority = 1;
    base_arena.flags = 0;

    zx_status_t status;
    for (range->reset(range), range->advance(range); !range->is_reset; range->advance(range)) {
        LTRACEF("Range at %#" PRIx64 " of %#" PRIx64 " bytes is %smemory.\n",
                range->base, range->size, range->is_mem ? "" : "not ");

        if (!range->is_mem) {
            continue;
        }

        // trim off parts of memory ranges that are smaller than a page
        uint64_t base = ROUNDUP(range->base, PAGE_SIZE);
        uint64_t size = ROUNDDOWN(range->base + range->size, PAGE_SIZE) -
                        base;

        // trim any memory below 1MB for safety and SMP booting purposes
        if (base < 1 * MB) {
            uint64_t adjust = 1 * MB - base;
            if (adjust >= size)
                continue;

            base += adjust;
            size -= adjust;
        }

        mark_mmio_region_to_reserve(base, static_cast<size_t>(size));
        if (have_limit) {
            status = memory_limit_add_range(base, size, base_arena);
        }

        // If there is no limit, or we failed to add arenas from processing
        // ranges then add the original range.
        if (!have_limit || status != ZX_OK) {
            auto arena = base_arena;
            arena.base = base;
            arena.size = size;

            LTRACEF("Adding pmm range at %#" PRIxPTR " of %#zx bytes.\n", arena.base, arena.size);
            status = pmm_add_arena(&arena);

            // print a warning and continue
            if (status != ZX_OK) {
                printf("MEM: Failed to add pmm range at %#" PRIxPTR " size %#zx\n", arena.base, arena.size);
            }
        }
    }

    if (have_limit) {
        memory_limit_add_arenas(base_arena);
    }

    return ZX_OK;
}

typedef struct e820_range_seq {
    e820entry_t* map;
    int index;
    int count;
} e820_range_seq_t;

static void e820_range_reset(boot_addr_range_t* range) {
    boot_addr_range_reset(range);

    e820_range_seq_t* seq = (e820_range_seq_t*)(range->seq);
    seq->index = -1;
}

static void e820_range_advance(boot_addr_range_t* range) {
    e820_range_seq_t* seq = (e820_range_seq_t*)(range->seq);

    seq->index++;

    if (seq->index == seq->count) {
        /* reset range to signal that we're at the end of the map */
        e820_range_reset(range);
        return;
    }

    e820entry_t* entry = &seq->map[seq->index];
    range->base = entry->addr;
    range->size = entry->size;
    range->is_mem = (entry->type == E820_RAM) ? 1 : 0;
    range->is_reset = 0;
}

static zx_status_t e820_range_init(boot_addr_range_t* range, e820_range_seq_t* seq) {
    range->seq = seq;
    range->advance = &e820_range_advance;
    range->reset = &e820_range_reset;

    if (bootloader.e820_count) {
        seq->count = static_cast<int>(bootloader.e820_count);
        seq->map = static_cast<e820entry_t*>(bootloader.e820_table);
        range->reset(range);
        return ZX_OK;
    }

    return ZX_ERR_NO_MEMORY;
}

typedef struct efi_range_seq {
    void* base;
    size_t entrysz;
    int index;
    int count;
} efi_range_seq_t;

static void efi_range_reset(boot_addr_range_t* range) {
    boot_addr_range_reset(range);

    efi_range_seq_t* seq = (efi_range_seq_t*)(range->seq);
    seq->index = -1;
}

static int efi_is_mem(uint32_t type) {
    switch (type) {
    case EfiLoaderCode:
    case EfiLoaderData:
    case EfiBootServicesCode:
    case EfiBootServicesData:
    case EfiConventionalMemory:
        return 1;
    default:
        return 0;
    }
}

static void efi_print(const char* tag, efi_memory_descriptor* e) {
    bool mb = e->NumberOfPages > 256;
    LTRACEF("%s%016lx %08x %lu%s\n",
            tag, e->PhysicalStart, e->Type,
            mb ? e->NumberOfPages / 256 : e->NumberOfPages * 4,
            mb ? "MB" : "KB");
}

static void efi_range_advance(boot_addr_range_t* range) {
    efi_range_seq_t* seq = (efi_range_seq_t*)(range->seq);

    seq->index++;

    if (seq->index == seq->count) {
        /* reset range to signal that we're at the end of the map */
        efi_range_reset(range);
        return;
    }

    const uintptr_t addr = reinterpret_cast<uintptr_t>(seq->base) + (seq->index * seq->entrysz);
    efi_memory_descriptor* entry = reinterpret_cast<efi_memory_descriptor*>(addr);
    efi_print("EFI: ", entry);
    range->base = entry->PhysicalStart;
    range->size = entry->NumberOfPages * PAGE_SIZE;
    range->is_reset = 0;
    range->is_mem = efi_is_mem(entry->Type);

    // coalesce adjacent memory ranges
    while ((seq->index + 1) < seq->count) {
        const uintptr_t addr = reinterpret_cast<uintptr_t>(seq->base) +
                               ((seq->index + 1) * seq->entrysz);
        efi_memory_descriptor* next = reinterpret_cast<efi_memory_descriptor*>(addr);
        if ((range->base + range->size) != next->PhysicalStart) {
            break;
        }
        if (efi_is_mem(next->Type) != range->is_mem) {
            break;
        }
        efi_print("EFI+ ", next);
        range->size += next->NumberOfPages * PAGE_SIZE;
        seq->index++;
    }
}

static zx_status_t efi_range_init(boot_addr_range_t* range, efi_range_seq_t* seq) {
    range->seq = seq;
    range->advance = &efi_range_advance;
    range->reset = &efi_range_reset;

    if (bootloader.efi_mmap &&
        (bootloader.efi_mmap_size > sizeof(uint64_t))) {
        seq->entrysz = *((uint64_t*)bootloader.efi_mmap);
        if (seq->entrysz < sizeof(efi_memory_descriptor)) {
            return ZX_ERR_NO_MEMORY;
        }

        seq->count = static_cast<int>((bootloader.efi_mmap_size - sizeof(uint64_t)) / seq->entrysz);
        seq->base = reinterpret_cast<void*>(
            reinterpret_cast<uintptr_t>(bootloader.efi_mmap) + sizeof(uint64_t));
        range->reset(range);
        return ZX_OK;
    } else {
        return ZX_ERR_NO_MEMORY;
    }
}

static int addr_range_cmp(const void* p1, const void* p2) {
    const struct addr_range* a1 = static_cast<const struct addr_range*>(p1);
    const struct addr_range* a2 = static_cast<const struct addr_range*>(p2);

    if (a1->base < a2->base)
        return -1;
    else if (a1->base == a2->base)
        return 0;
    return 1;
}

static zx_status_t platform_mem_range_init(void) {
    zx_status_t status;
    boot_addr_range_t range;

    /* first try the efi memory table */
    efi_range_seq_t efi_seq;
    status = efi_range_init(&range, &efi_seq);
    if (status == ZX_OK) {
        status = mem_arena_init(&range);
        if (status != ZX_OK) {
            printf("MEM: failure while adding EFI memory ranges\n");
        }
        return ZX_OK;
    }

    /* then try getting range info from e820 */
    e820_range_seq_t e820_seq;
    status = e820_range_init(&range, &e820_seq);
    if (status == ZX_OK) {
        status = mem_arena_init(&range);
        if (status != ZX_OK) {
            printf("MEM: failure while adding e820 memory ranges\n");
        }
        return ZX_OK;
    }

    /* if still no ranges were found, make a safe guess */
    printf("MEM: no arena range source: falling back to fixed size\n");
    e820_range_init(&range, &e820_seq);
    e820entry_t entry = {
        .addr = 0,
        .size = DEFAULT_MEMEND,
        .type = E820_RAM,
    };
    e820_seq.map = &entry;
    e820_seq.count = 1;
    return mem_arena_init(&range);
}

static size_t cached_e820_entry_count;
static struct addr_range cached_e820_entries[64];

zx_status_t enumerate_e820(enumerate_e820_callback callback, void* ctx) {
    if (callback == NULL)
        return ZX_ERR_INVALID_ARGS;

    if (!cached_e820_entry_count)
        return ZX_ERR_BAD_STATE;

    DEBUG_ASSERT(cached_e820_entry_count <= fbl::count_of(cached_e820_entries));
    for (size_t i = 0; i < cached_e820_entry_count; ++i)
        callback(cached_e820_entries[i].base, cached_e820_entries[i].size,
                 cached_e820_entries[i].is_mem, ctx);

    return ZX_OK;
}

/* Discover the basic memory map */
void pc_mem_init(void) {
    if (platform_mem_range_init() != ZX_OK) {
        TRACEF("Error adding arenas from provided memory tables.\n");
    }

    // Cache the e820 entries so that they will be available for enumeration
    // later in the boot.
    //
    // TODO(teisenbe, johngro): do not hardcode a limit on the number of
    // entries we may have.  Find some other way to make this information
    // available at any point in time after we boot.
    boot_addr_range_t range;
    efi_range_seq_t efi_seq;
    e820_range_seq_t e820_seq;
    bool initialized_bootstrap16 = false;

    cached_e820_entry_count = 0;
    if ((efi_range_init(&range, &efi_seq) == ZX_OK) ||
        (e820_range_init(&range, &e820_seq) == ZX_OK)) {
        for (range.reset(&range),
             range.advance(&range);
             !range.is_reset; range.advance(&range)) {
            if (cached_e820_entry_count >= fbl::count_of(cached_e820_entries)) {
                TRACEF("ERROR - Too many e820 entries to hold in the cache!\n");
                cached_e820_entry_count = 0;
                break;
            }

            struct addr_range* entry = &cached_e820_entries[cached_e820_entry_count++];
            entry->base = range.base;
            entry->size = range.size;
            entry->is_mem = range.is_mem ? true : false;

            const uint64_t alloc_size = 2 * PAGE_SIZE;
            const uint64_t min_base = 2 * PAGE_SIZE;
            if (!initialized_bootstrap16 && entry->is_mem &&
                entry->base <= 1 * MB - alloc_size && entry->size >= alloc_size) {

                uint64_t adj_base = entry->base;
                if (entry->base < min_base) {
                    uint64_t size_adj = min_base - entry->base;
                    if (entry->size < size_adj + alloc_size) {
                        continue;
                    }
                    adj_base = min_base;
                }

                LTRACEF("Selected %" PRIxPTR " as bootstrap16 region\n", adj_base);
                x86_bootstrap16_init(adj_base);
                initialized_bootstrap16 = true;
            }
        }
    } else {
        TRACEF("ERROR - No e820 range entries found!  This is going to end badly for everyone.\n");
    }

    if (!initialized_bootstrap16) {
        TRACEF("WARNING - Failed to assign bootstrap16 region, SMP won't work\n");
    }
}



// Initialize the higher level PhysicalAspaceManager after the heap is initialized.
static void x86_resource_init_hook(unsigned int rl) {
    // An error is likely fatal if the bookkeeping is broken and driver
    ResourceDispatcher::InitializeAllocator(ZX_RSRC_KIND_MMIO,
                                            0,
                                            (1ull << (x86_physical_address_width())) - 1);
    ResourceDispatcher::InitializeAllocator(ZX_RSRC_KIND_IOPORT,
                                            0,
                                            UINT16_MAX);
    ResourceDispatcher::InitializeAllocator(ZX_RSRC_KIND_IRQ,
                                            interrupt_get_base_vector(),
                                            interrupt_get_max_vector());

    // Exclusively reserve the regions marked as memory earlier so that physical
    // vmos cannot be created against them.
    for (uint8_t i = 0; i < reserved_mmio_count; i++) {
        zx_rights_t rights;
        auto& entry = reserved_mmio_entries[i];
        zx_status_t st = ResourceDispatcher::Create(&entry.dispatcher, &rights, ZX_RSRC_KIND_MMIO,
                                                    entry.base, entry.len, ZX_RSRC_FLAG_EXCLUSIVE,
                                                    "platform_memory");
        if (st == ZX_OK) {
        } else {
            TRACEF("failed to create backing resource for boot memory region %#lx - %#lx: %d\n",
                   entry.base, entry.base + entry.len, st);
        }

    }
}

LK_INIT_HOOK(x86_resource_init, x86_resource_init_hook, LK_INIT_LEVEL_HEAP);