1 // Copyright 2016 The Fuchsia Authors
2 // Copyright (c) 2016 Travis Geiselbrecht
3 //
4 // Use of this source code is governed by a MIT-style
5 // license that can be found in the LICENSE file or at
6 // https://opensource.org/licenses/MIT
7
8 #pragma once
9
10 /* describes the per cpu structure pointed to by gs: in the kernel */
11
12 /* offsets into this structure, used by assembly */
13 #define PERCPU_DIRECT_OFFSET 0x0
14 #define PERCPU_CURRENT_THREAD_OFFSET 0x8
15 // ZX_TLS_STACK_GUARD_OFFSET 0x10
16 // ZX_TLS_UNSAFE_SP_OFFSET 0x18
17 #define PERCPU_SAVED_USER_SP_OFFSET 0x20
18 #define PERCPU_GPF_RETURN_OFFSET 0x40
19 #define PERCPU_CPU_NUM_OFFSET 0x48
20 #define PERCPU_DEFAULT_TSS_OFFSET 0x50
21
22 /* offset of default_tss.rsp0 */
23 #define PERCPU_KERNEL_SP_OFFSET (PERCPU_DEFAULT_TSS_OFFSET + 4)
24
25 #ifndef __ASSEMBLER__
26
27 #include <arch/x86.h>
28 #include <arch/x86/idt.h>
29 #include <assert.h>
30 #include <kernel/align.h>
31 #include <kernel/cpu.h>
32 #include <stdint.h>
33 #include <zircon/compiler.h>
34 #include <zircon/tls.h>
35 #include <zircon/types.h>
36
37 __BEGIN_CDECLS
38
39 struct thread;
40
41 struct x86_percpu {
42 /* a direct pointer to ourselves */
43 struct x86_percpu *direct;
44
45 /* the current thread */
46 struct thread *current_thread;
47
48 // The offsets of these two slots are published in
49 // system/public/zircon/tls.h and known to the compiler.
50 uintptr_t stack_guard;
51 uintptr_t kernel_unsafe_sp;
52
53 /* temporarily saved during a syscall */
54 uintptr_t saved_user_sp;
55
56 /* Whether blocking is disallowed. See arch_blocking_disallowed(). */
57 uint32_t blocking_disallowed;
58
59 /* Memory for IPI-free rescheduling of idle CPUs with monitor/mwait. */
60 volatile uint8_t* monitor;
61
62 /* local APIC id */
63 uint32_t apic_id;
64
65 /* If nonzero and we receive a GPF, change the return IP to this value. */
66 uintptr_t gpf_return_target;
67
68 /* CPU number */
69 cpu_num_t cpu_num;
70
71 /* This CPU's default TSS */
72 tss_t default_tss __ALIGNED(16);
73
74 /* Reserved space for interrupt stacks */
75 uint8_t interrupt_stacks[NUM_ASSIGNED_IST_ENTRIES][PAGE_SIZE] __ALIGNED(16);
76 } __CPU_ALIGN;
77
78 static_assert(__offsetof(struct x86_percpu, direct) == PERCPU_DIRECT_OFFSET, "");
79 static_assert(__offsetof(struct x86_percpu, current_thread) == PERCPU_CURRENT_THREAD_OFFSET, "");
80 static_assert(__offsetof(struct x86_percpu, stack_guard) == ZX_TLS_STACK_GUARD_OFFSET, "");
81 static_assert(__offsetof(struct x86_percpu, kernel_unsafe_sp) == ZX_TLS_UNSAFE_SP_OFFSET, "");
82 static_assert(__offsetof(struct x86_percpu, saved_user_sp) == PERCPU_SAVED_USER_SP_OFFSET, "");
83 static_assert(__offsetof(struct x86_percpu, gpf_return_target) == PERCPU_GPF_RETURN_OFFSET, "");
84 static_assert(__offsetof(struct x86_percpu, cpu_num) == PERCPU_CPU_NUM_OFFSET, "");
85 static_assert(__offsetof(struct x86_percpu, default_tss) == PERCPU_DEFAULT_TSS_OFFSET, "");
86 static_assert(__offsetof(struct x86_percpu, default_tss.rsp0) == PERCPU_KERNEL_SP_OFFSET, "");
87
88 extern struct x86_percpu bp_percpu;
89 extern struct x86_percpu *ap_percpus;
90
91 // This needs to be run very early in the boot process from start.S and as
92 // each CPU is brought up.
93 void x86_init_percpu(uint cpu_num);
94
95 /* used to set the bootstrap processor's apic_id once the APIC is initialized */
96 void x86_set_local_apic_id(uint32_t apic_id);
97
98 int x86_apic_id_to_cpu_num(uint32_t apic_id);
99
100 // Allocate all of the necessary structures for all of the APs to run.
101 zx_status_t x86_allocate_ap_structures(uint32_t *apic_ids, uint8_t cpu_count);
102
x86_get_percpu(void)103 static inline struct x86_percpu *x86_get_percpu(void)
104 {
105 return (struct x86_percpu *)x86_read_gs_offset64(PERCPU_DIRECT_OFFSET);
106 }
107
arch_curr_cpu_num(void)108 static inline cpu_num_t arch_curr_cpu_num(void)
109 {
110 return x86_get_percpu()->cpu_num;
111 }
112
113 extern uint8_t x86_num_cpus;
arch_max_num_cpus(void)114 static uint arch_max_num_cpus(void)
115 {
116 return x86_num_cpus;
117 }
118
119 #define READ_PERCPU_FIELD32(field) \
120 x86_read_gs_offset32(offsetof(struct x86_percpu, field))
121
122 #define WRITE_PERCPU_FIELD32(field, value) \
123 x86_write_gs_offset32(offsetof(struct x86_percpu, field), (value))
124
125 void x86_ipi_halt_handler(void*) __NO_RETURN;
126 void x86_secondary_entry(volatile int *aps_still_booting, thread_t *thread);
127 void x86_force_halt_all_but_local_and_bsp(void);
128
129 __END_CDECLS
130
131 #endif // !__ASSEMBLER__
132