1 // © 2021 Qualcomm Innovation Center, Inc. All rights reserved.
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4
5 // rootvm_init() is allowed to call partition_get_root().
6 #define ROOTVM_INIT 1
7
8 #include <assert.h>
9 #include <hyptypes.h>
10 #include <string.h>
11
12 #include <attributes.h>
13 #include <bitmap.h>
14 #include <cpulocal.h>
15 #include <cspace.h>
16 #include <memdb.h>
17 #include <object.h>
18 #include <panic.h>
19 #include <partition.h>
20 #include <partition_alloc.h>
21 #include <partition_init.h>
22 #include <platform_mem.h>
23 #include <qcbor.h>
24 #include <scheduler.h>
25 #include <spinlock.h>
26 #include <thread.h>
27 #include <util.h>
28 #include <vcpu.h>
29
30 #include <events/object.h>
31 #include <events/rootvm.h>
32
33 #include <asm/cache.h>
34 #include <asm/cpu.h>
35
36 #include "boot_init.h"
37 #include "event_handlers.h"
38
39 // FIXME: remove when we have a device tree where to read it from
40 // dummy value.
41 #define MAX_CAPS 2048
42
43 static void
copy_rm_env_data_to_rootvm_mem(hyp_env_data_t hyp_env,const rm_env_data_hdr_t * rm_env_data,rt_env_data_t * crt_env,uint32_t env_data_size)44 copy_rm_env_data_to_rootvm_mem(hyp_env_data_t hyp_env,
45 const rm_env_data_hdr_t *rm_env_data,
46 rt_env_data_t *crt_env, uint32_t env_data_size)
47 {
48 paddr_t hyp_env_phys = hyp_env.env_ipa - hyp_env.me_ipa_base +
49 PLATFORM_ROOTVM_LMA_BASE;
50 assert(util_is_baligned(hyp_env_phys, PGTABLE_VM_PAGE_SIZE));
51
52 void *va = partition_phys_map(hyp_env_phys, env_data_size);
53 partition_phys_access_enable(va);
54
55 (void)memcpy(va, (void *)crt_env,
56 (rm_env_data->data_payload_size + sizeof(*rm_env_data) +
57 sizeof(*crt_env)));
58 CACHE_CLEAN_RANGE((rm_env_data_hdr_t *)va,
59 (rm_env_data->data_payload_size +
60 sizeof(*rm_env_data) + sizeof(*crt_env)));
61
62 partition_phys_access_disable(va);
63 partition_phys_unmap(va, hyp_env_phys, env_data_size);
64 }
65
66 static void
rootvm_close_env_data(qcbor_enc_ctxt_t * qcbor_enc_ctxt,rm_env_data_hdr_t * rm_env_data)67 rootvm_close_env_data(qcbor_enc_ctxt_t *qcbor_enc_ctxt,
68 rm_env_data_hdr_t *rm_env_data)
69 {
70 qcbor_err_t cb_err;
71 const_useful_buff_t payload_out_buff;
72
73 payload_out_buff.ptr = NULL;
74 payload_out_buff.len = 0;
75
76 cb_err = QCBOREncode_Finish(qcbor_enc_ctxt, &payload_out_buff);
77
78 if (cb_err != QCBOR_SUCCESS) {
79 panic("Env data encoding error, increase the buffer size");
80 }
81
82 rm_env_data->data_payload_size = (uint32_t)payload_out_buff.len;
83 }
84
85 typedef struct {
86 hyp_env_data_t hyp_env;
87 qcbor_enc_ctxt_t *qcbor_enc_ctxt;
88 rm_env_data_hdr_t *rm_env_data;
89 rt_env_data_t *crt_env;
90 } rootvm_init_env_info;
91
92 static rootvm_init_env_info
rootvm_init_env_data(partition_t * root_partition,uint32_t env_data_size)93 rootvm_init_env_data(partition_t *root_partition, uint32_t env_data_size)
94 {
95 void_ptr_result_t alloc_ret;
96 hyp_env_data_t hyp_env; // Local on stack used as context
97 qcbor_enc_ctxt_t *qcbor_enc_ctxt;
98
99 rm_env_data_hdr_t *rm_env_data;
100 rt_env_data_t *crt_env;
101 uint32_t remaining_size;
102
103 alloc_ret = partition_alloc(root_partition, env_data_size,
104 PGTABLE_VM_PAGE_SIZE);
105 if (alloc_ret.e != OK) {
106 panic("Allocate env_data failed");
107 }
108 crt_env = (rt_env_data_t *)alloc_ret.r;
109 (void)memset_s(crt_env, env_data_size, 0, env_data_size);
110
111 alloc_ret = partition_alloc(root_partition, sizeof(*qcbor_enc_ctxt),
112 alignof(*qcbor_enc_ctxt));
113 if (alloc_ret.e != OK) {
114 panic("Allocate cbor_ctxt failed");
115 }
116
117 qcbor_enc_ctxt = (qcbor_enc_ctxt_t *)alloc_ret.r;
118
119 (void)memset_s(qcbor_enc_ctxt, sizeof(*qcbor_enc_ctxt), 0,
120 sizeof(*qcbor_enc_ctxt));
121
122 (void)memset_s(&hyp_env, sizeof(hyp_env), 0, sizeof(hyp_env));
123
124 hyp_env.env_data_size = env_data_size;
125 remaining_size = env_data_size;
126
127 crt_env->signature = ROOTVM_ENV_DATA_SIGNATURE;
128 crt_env->version = 1;
129
130 size_t rm_config_offset =
131 util_balign_up(sizeof(*crt_env), alignof(*rm_env_data));
132 assert(remaining_size >= (rm_config_offset + sizeof(*rm_env_data)));
133
134 remaining_size -= rm_config_offset;
135 rm_env_data =
136 (rm_env_data_hdr_t *)((uintptr_t)crt_env + rm_config_offset);
137
138 crt_env->rm_config_offset = rm_config_offset;
139 crt_env->rm_config_size = remaining_size;
140
141 rm_env_data->signature = RM_ENV_DATA_SIGNATURE;
142 rm_env_data->version = 1;
143 rm_env_data->data_payload_offset = sizeof(*rm_env_data);
144 rm_env_data->data_payload_size = 0U;
145
146 remaining_size -= sizeof(*rm_env_data);
147
148 useful_buff_t qcbor_data_buff;
149 qcbor_data_buff.ptr =
150 (((uint8_t *)rm_env_data) + rm_env_data->data_payload_offset);
151 qcbor_data_buff.len = remaining_size;
152
153 QCBOREncode_Init(qcbor_enc_ctxt, qcbor_data_buff);
154
155 return (rootvm_init_env_info){
156 .crt_env = crt_env,
157 .hyp_env = hyp_env,
158 .qcbor_enc_ctxt = qcbor_enc_ctxt,
159 .rm_env_data = rm_env_data,
160 };
161 }
162
163 void NOINLINE
rootvm_init(void)164 rootvm_init(void)
165 {
166 static_assert(SCHEDULER_NUM_PRIORITIES >= (priority_t)3U,
167 "unexpected scheduler configuration");
168 static_assert(ROOTVM_PRIORITY <= VCPU_MAX_PRIORITY,
169 "unexpected scheduler configuration");
170
171 thread_create_t params = {
172 .scheduler_affinity = cpulocal_get_index(),
173 .scheduler_affinity_valid = true,
174 .scheduler_priority = ROOTVM_PRIORITY,
175 .scheduler_priority_valid = true,
176 };
177
178 partition_t *root_partition = partition_get_root();
179
180 assert(root_partition != NULL);
181
182 platform_add_root_heap(root_partition);
183
184 // Create cspace for root partition
185 cspace_create_t cs_params = { NULL };
186
187 cspace_ptr_result_t cspace_ret =
188 partition_allocate_cspace(root_partition, cs_params);
189 if (cspace_ret.e != OK) {
190 goto cspace_fail;
191 }
192 cspace_t *root_cspace = cspace_ret.r;
193
194 spinlock_acquire_nopreempt(&root_cspace->header.lock);
195 if (cspace_configure(root_cspace, MAX_CAPS) != OK) {
196 spinlock_release_nopreempt(&root_cspace->header.lock);
197 goto cspace_fail;
198 }
199 spinlock_release_nopreempt(&root_cspace->header.lock);
200
201 if (object_activate_cspace(root_cspace) != OK) {
202 goto cspace_fail;
203 }
204
205 trigger_object_get_defaults_thread_event(¶ms);
206
207 // Allocate and setup the root thread
208 thread_ptr_result_t thd_ret =
209 partition_allocate_thread(root_partition, params);
210 if (thd_ret.e != OK) {
211 panic("Error allocating root thread");
212 }
213 thread_t *root_thread = (thread_t *)thd_ret.r;
214
215 vcpu_option_flags_t vcpu_options = vcpu_option_flags_default();
216
217 vcpu_option_flags_set_critical(&vcpu_options, true);
218
219 if (vcpu_configure(root_thread, vcpu_options) != OK) {
220 panic("Error configuring vcpu");
221 }
222
223 // Attach root cspace to root thread
224 if (cspace_attach_thread(root_cspace, root_thread) != OK) {
225 panic("Error attaching cspace to root thread");
226 }
227
228 // Give the root cspace a cap to itself
229 object_ptr_t obj_ptr;
230
231 obj_ptr.cspace = root_cspace;
232 cap_id_result_t capid_ret = cspace_create_master_cap(
233 root_cspace, obj_ptr, OBJECT_TYPE_CSPACE);
234 if (capid_ret.e != OK) {
235 goto cspace_fail;
236 }
237
238 uint32_t env_data_size = QCBOR_ENV_CONFIG_SIZE;
239
240 rootvm_init_env_info info =
241 rootvm_init_env_data(root_partition, env_data_size);
242
243 hyp_env_data_t hyp_env = info.hyp_env;
244 qcbor_enc_ctxt_t *qcbor_enc_ctxt = info.qcbor_enc_ctxt;
245
246 rm_env_data_hdr_t *rm_env_data = info.rm_env_data;
247 rt_env_data_t *crt_env = info.crt_env;
248
249 QCBOREncode_OpenMap(qcbor_enc_ctxt);
250
251 QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "cspace_capid", capid_ret.r);
252
253 // Take extra reference so that the deletion of the master cap does not
254 // accidentally destroy the partition.
255 root_partition = object_get_partition_additional(root_partition);
256
257 // Create caps for the root partition and thread
258 obj_ptr.partition = root_partition;
259 capid_ret = cspace_create_master_cap(root_cspace, obj_ptr,
260 OBJECT_TYPE_PARTITION);
261 if (capid_ret.e != OK) {
262 panic("Error creating root partition cap");
263 }
264 QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "partition_capid",
265 capid_ret.r);
266
267 obj_ptr.thread = root_thread;
268 capid_ret = cspace_create_master_cap(root_cspace, obj_ptr,
269 OBJECT_TYPE_THREAD);
270 if (capid_ret.e != OK) {
271 panic("Error creating root partition cap");
272 }
273 QCBOREncode_AddUInt64ToMap(qcbor_enc_ctxt, "vcpu_capid", capid_ret.r);
274 crt_env->vcpu_capid = capid_ret.r;
275
276 // Do a memdb walk to get all the available memory ranges of the root
277 // partition and save in the rm_env_data
278 if (boot_add_free_range((uintptr_t)root_partition, MEMDB_TYPE_PARTITION,
279 qcbor_enc_ctxt) != OK) {
280 panic("Error doing the memory database walk");
281 }
282
283 trigger_rootvm_init_event(root_partition, root_thread, root_cspace,
284 &hyp_env, qcbor_enc_ctxt);
285
286 QCBOREncode_CloseMap(qcbor_enc_ctxt);
287
288 rootvm_close_env_data(qcbor_enc_ctxt, rm_env_data);
289
290 crt_env->runtime_ipa = hyp_env.runtime_ipa;
291 crt_env->app_ipa = hyp_env.app_ipa;
292 crt_env->app_heap_ipa = hyp_env.app_heap_ipa;
293 crt_env->app_heap_size = hyp_env.app_heap_size;
294 crt_env->timer_freq = hyp_env.timer_freq;
295 crt_env->gicd_base = hyp_env.gicd_base;
296 crt_env->gicr_base = hyp_env.gicr_base;
297
298 // Copy the rm_env_data to the root VM memory
299 copy_rm_env_data_to_rootvm_mem(hyp_env, rm_env_data, crt_env,
300 env_data_size);
301
302 // Setup the root VM thread
303 if (object_activate_thread(root_thread) != OK) {
304 panic("Error activating root thread");
305 }
306
307 trigger_rootvm_init_late_event(root_partition, root_thread, root_cspace,
308 &hyp_env);
309
310 scheduler_lock_nopreempt(root_thread);
311 // FIXME: eventually pass as dtb, for now the rm_env_data ipa is passed
312 // directly.
313 bool_result_t power_ret =
314 vcpu_poweron(root_thread, vmaddr_result_ok(hyp_env.entry_ipa),
315 register_result_ok(hyp_env.env_ipa));
316 if (power_ret.e != OK) {
317 panic("Error vcpu poweron");
318 }
319
320 // Allow other modules to clean up after root VM creation.
321 trigger_rootvm_started_event(root_thread);
322 scheduler_unlock_nopreempt(root_thread);
323 (void)partition_free(root_partition, crt_env, env_data_size);
324 rm_env_data = NULL;
325
326 return;
327
328 cspace_fail:
329 panic("Error creating root cspace cap");
330 }
331