1 // © 2021 Qualcomm Innovation Center, Inc. All rights reserved.
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4
5 #include <assert.h>
6 #include <hyptypes.h>
7
8 #include <hypconstants.h>
9 #include <hypregisters.h>
10
11 #include <compiler.h>
12 #include <cpulocal.h>
13 #include <irq.h>
14 #include <object.h>
15 #include <panic.h>
16 #include <partition.h>
17 #include <partition_alloc.h>
18 #include <preempt.h>
19 #include <scheduler.h>
20 #include <trace.h>
21
22 #include <asm/barrier.h>
23
24 #include "arm_vm_timer.h"
25 #include "event_handlers.h"
26
27 #if defined(VERBOSE) && VERBOSE
28 #define VM_TIMER_DEBUG 1
29 #else
30 #define VM_TIMER_DEBUG 0
31 #endif
32
33 #define ARM_VM_TIMER_TYPE_NUM (ENUM_ARM_VM_TIMER_TYPE_MAX_VALUE + 1)
34
35 static hwirq_t *arm_vm_timer_hwirq[ARM_VM_TIMER_TYPE_NUM];
36 CPULOCAL_DECLARE_STATIC(bool, arm_vm_timer_irq_active)[ARM_VM_TIMER_TYPE_NUM];
37
38 void
arm_vm_timer_init(arm_vm_timer_type_t tt)39 arm_vm_timer_init(arm_vm_timer_type_t tt)
40 {
41 CNT_CTL_t cnt_ctl;
42
43 CNT_CTL_init(&cnt_ctl);
44 CNT_CTL_set_IMASK(&cnt_ctl, true);
45
46 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
47 register_CNTV_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering);
48 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
49 register_CNTP_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering);
50 } else {
51 panic("Invalid timer");
52 }
53 }
54
55 bool
arm_vm_timer_is_irq_enabled(arm_vm_timer_type_t tt)56 arm_vm_timer_is_irq_enabled(arm_vm_timer_type_t tt)
57 {
58 CNT_CTL_t cnt_ctl;
59
60 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
61 cnt_ctl = register_CNTV_CTL_EL0_read_volatile_ordered(
62 &asm_ordering);
63 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
64 cnt_ctl = register_CNTP_CTL_EL0_read_volatile_ordered(
65 &asm_ordering);
66 } else {
67 panic("Invalid timer");
68 }
69
70 return (CNT_CTL_get_ENABLE(&cnt_ctl) && !CNT_CTL_get_IMASK(&cnt_ctl));
71 }
72
73 bool
arm_vm_timer_is_irq_pending(arm_vm_timer_type_t tt)74 arm_vm_timer_is_irq_pending(arm_vm_timer_type_t tt)
75 {
76 CNT_CTL_t cnt_ctl;
77
78 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
79 cnt_ctl = register_CNTV_CTL_EL0_read_volatile_ordered(
80 &asm_ordering);
81 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
82 cnt_ctl = register_CNTP_CTL_EL0_read_volatile_ordered(
83 &asm_ordering);
84 } else {
85 panic("Invalid timer");
86 }
87
88 return (CNT_CTL_get_ENABLE(&cnt_ctl) && !CNT_CTL_get_IMASK(&cnt_ctl) &&
89 CNT_CTL_get_ISTATUS(&cnt_ctl));
90 }
91
92 void
arm_vm_timer_cancel_timeout(arm_vm_timer_type_t tt)93 arm_vm_timer_cancel_timeout(arm_vm_timer_type_t tt)
94 {
95 CNT_CTL_t cnt_ctl;
96
97 CNT_CTL_init(&cnt_ctl);
98 CNT_CTL_set_ENABLE(&cnt_ctl, false);
99
100 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
101 register_CNTV_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering);
102 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
103 register_CNTP_CTL_EL0_write_ordered(cnt_ctl, &asm_ordering);
104 } else {
105 panic("Invalid timer");
106 }
107 }
108
109 bool
arm_vm_timer_get_is_expired(arm_vm_timer_type_t tt)110 arm_vm_timer_get_is_expired(arm_vm_timer_type_t tt)
111 {
112 CNT_CTL_t cnt_ctl;
113
114 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
115 cnt_ctl = register_CNTV_CTL_EL0_read_volatile_ordered(
116 &asm_ordering);
117 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
118 cnt_ctl = register_CNTP_CTL_EL0_read_volatile_ordered(
119 &asm_ordering);
120 } else {
121 panic("Invalid timer");
122 }
123
124 assert(CNT_CTL_get_ENABLE(&cnt_ctl));
125 return CNT_CTL_get_ISTATUS(&cnt_ctl);
126 }
127
128 uint32_t
arm_vm_timer_get_freqeuncy(void)129 arm_vm_timer_get_freqeuncy(void)
130 {
131 CNTFRQ_EL0_t cntfrq = register_CNTFRQ_EL0_read();
132
133 return CNTFRQ_EL0_get_ClockFrequency(&cntfrq);
134 }
135
136 uint64_t
arm_vm_timer_get_ticks(void)137 arm_vm_timer_get_ticks(void)
138 {
139 // This register read below is allowed to occur speculatively at any
140 // time after the most recent context sync event. If caller the wants
141 // it to actually reflect the exact current time, it must execute an
142 // ordered ISB before calling this function.
143 CNTPCT_EL0_t cntpct =
144 register_CNTPCT_EL0_read_volatile_ordered(&asm_ordering);
145
146 return CNTPCT_EL0_get_CountValue(&cntpct);
147 }
148
149 uint64_t
arm_vm_timer_get_timeout(arm_vm_timer_type_t tt)150 arm_vm_timer_get_timeout(arm_vm_timer_type_t tt)
151 {
152 CNT_CVAL_t cnt_cval;
153
154 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
155 cnt_cval = register_CNTV_CVAL_EL0_read_volatile();
156 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
157 cnt_cval = register_CNTP_CVAL_EL0_read_volatile();
158 } else {
159 panic("Invalid timer");
160 }
161
162 return CNT_CVAL_get_CompareValue(&cnt_cval);
163 }
164
165 void
arm_vm_timer_arch_timer_hw_irq_activated(arm_vm_timer_type_t tt)166 arm_vm_timer_arch_timer_hw_irq_activated(arm_vm_timer_type_t tt)
167 {
168 if ((tt == ARM_VM_TIMER_TYPE_PHYSICAL) ||
169 (tt == ARM_VM_TIMER_TYPE_VIRTUAL)) {
170 CPULOCAL(arm_vm_timer_irq_active)[tt] = true;
171 } else {
172 panic("Invalid timer");
173 }
174 }
175
176 void
arm_vm_timer_arch_timer_hw_irq_deactivate(arm_vm_timer_type_t tt)177 arm_vm_timer_arch_timer_hw_irq_deactivate(arm_vm_timer_type_t tt)
178 {
179 if ((tt == ARM_VM_TIMER_TYPE_PHYSICAL) ||
180 (tt == ARM_VM_TIMER_TYPE_VIRTUAL)) {
181 if (CPULOCAL(arm_vm_timer_irq_active)[tt]) {
182 CPULOCAL(arm_vm_timer_irq_active)[tt] = false;
183 irq_deactivate(arm_vm_timer_hwirq[tt]);
184 }
185 } else {
186 panic("Invalid timer");
187 }
188 }
189
190 void
arm_vm_timer_handle_boot_cpu_cold_init(void)191 arm_vm_timer_handle_boot_cpu_cold_init(void)
192 {
193 for (int tt = ENUM_ARM_VM_TIMER_TYPE_MIN_VALUE;
194 tt < ARM_VM_TIMER_TYPE_NUM; tt++) {
195 CPULOCAL(arm_vm_timer_irq_active)[tt] = false;
196 }
197 }
198
199 void
arm_vm_timer_handle_boot_hypervisor_start(void)200 arm_vm_timer_handle_boot_hypervisor_start(void)
201 {
202 hwirq_ptr_result_t ret;
203 hwirq_create_t params[] = {
204 {
205 .irq = PLATFORM_VM_ARCH_VIRTUAL_TIMER_IRQ,
206 .action = HWIRQ_ACTION_VM_TIMER,
207 },
208 {
209 .irq = PLATFORM_VM_ARCH_PHYSICAL_TIMER_IRQ,
210 .action = HWIRQ_ACTION_VM_TIMER,
211 }
212 };
213
214 for (int tt = ENUM_ARM_VM_TIMER_TYPE_MIN_VALUE;
215 tt < ARM_VM_TIMER_TYPE_NUM; tt++) {
216 ret = partition_allocate_hwirq(partition_get_private(),
217 params[tt]);
218
219 if ((ret.e != OK) || (object_activate_hwirq(ret.r) != OK)) {
220 panic("Failed to enable VM Timer IRQ");
221 }
222
223 arm_vm_timer_hwirq[tt] = ret.r;
224 irq_enable_local(arm_vm_timer_hwirq[tt]);
225 }
226 }
227
228 error_t
arm_vm_timer_handle_power_cpu_suspend(void)229 arm_vm_timer_handle_power_cpu_suspend(void)
230 {
231 arm_vm_timer_arch_timer_hw_irq_deactivate(ARM_VM_TIMER_TYPE_VIRTUAL);
232 arm_vm_timer_arch_timer_hw_irq_deactivate(ARM_VM_TIMER_TYPE_PHYSICAL);
233
234 return OK;
235 }
236
237 void
arm_vm_timer_handle_boot_cpu_warm_init(void)238 arm_vm_timer_handle_boot_cpu_warm_init(void)
239 {
240 arm_vm_timer_init(ARM_VM_TIMER_TYPE_VIRTUAL);
241 arm_vm_timer_init(ARM_VM_TIMER_TYPE_PHYSICAL);
242
243 #if defined(ARCH_ARM_FEAT_VHE)
244 CNTHCTL_EL2_E2H1_t cnthctl;
245 CNTHCTL_EL2_E2H1_init(&cnthctl);
246
247 CNTHCTL_EL2_E2H1_set_EL1PTEN(&cnthctl, true);
248 CNTHCTL_EL2_E2H1_set_EL1PCTEN(&cnthctl, true);
249
250 // TODO: Determine correct setting for EVNTI
251 CNTHCTL_EL2_E2H1_set_EVNTI(&cnthctl, 5);
252 CNTHCTL_EL2_E2H1_set_EVNTDIR(&cnthctl, false);
253 CNTHCTL_EL2_E2H1_set_EVNTEN(&cnthctl, false);
254
255 // These four are here for completeness and are not strictly necessary.
256 CNTHCTL_EL2_E2H1_set_EL0PTEN(&cnthctl, true);
257 CNTHCTL_EL2_E2H1_set_EL0VTEN(&cnthctl, true);
258 CNTHCTL_EL2_E2H1_set_EL0VCTEN(&cnthctl, true);
259 CNTHCTL_EL2_E2H1_set_EL0PCTEN(&cnthctl, true);
260
261 #if defined(ARCH_ARM_FEAT_ECV)
262 // Explicitly disable the ECV feature and the access traps for the
263 // virtual timer and counter registers.
264 CNTHCTL_EL2_E2H1_set_ECV(&cnthctl, false);
265 CNTHCTL_EL2_E2H1_set_EL1TVT(&cnthctl, false);
266 CNTHCTL_EL2_E2H1_set_EL1TVCT(&cnthctl, false);
267 #endif
268
269 register_CNTHCTL_EL2_E2H1_write(cnthctl);
270 #else
271 CNTHCTL_EL2_E2H0_t cnthctl;
272 CNTHCTL_EL2_E2H0_init(&cnthctl);
273
274 // In order to disable the physical timer at EL0 and EL1 we trap the
275 // accesses to the physical timer registers but do not provide a handler
276 // for the trap, causing a synchronous data abort to be injected to the
277 // guest.
278 CNTHCTL_EL2_E2H0_set_EL1PCEN(&cnthctl, true);
279 CNTHCTL_EL2_E2H0_set_EL1PCTEN(&cnthctl, true);
280
281 // TODO: Determine correct setting for EVNTI
282 CNTHCTL_EL2_E2H0_set_EVNTI(&cnthctl, 5);
283 CNTHCTL_EL2_E2H0_set_EVNTDIR(&cnthctl, false);
284 CNTHCTL_EL2_E2H0_set_EVNTEN(&cnthctl, false);
285
286 #if defined(ARCH_ARM_FEAT_ECV)
287 // Explicitly disable the ECV feature and the access traps for the
288 // virtual timer and counter registers.
289 CNTHCTL_EL2_E2H0_set_ECV(&cnthctl, false);
290 CNTHCTL_EL2_E2H0_set_EL1TVT(&cnthctl, false);
291 CNTHCTL_EL2_E2H0_set_EL1TVCT(&cnthctl, false);
292 #endif
293
294 register_CNTHCTL_EL2_E2H0_write(cnthctl);
295 #endif
296
297 #if VM_TIMER_DEBUG
298 TRACE_LOCAL(
299 DEBUG, INFO,
300 "arm_vm_timer warm boot pcnt {:#x} vctl {:#x} vact {:d} pact {:d}",
301 CNTPCT_EL0_raw(register_CNTPCT_EL0_read_volatile_ordered(
302 &asm_ordering)),
303 CNT_CTL_raw(register_CNTV_CTL_EL0_read_ordered(&asm_ordering)),
304 (register_t)CPULOCAL(
305 arm_vm_timer_irq_active)[ARM_VM_TIMER_TYPE_VIRTUAL],
306 (register_t)CPULOCAL(
307 arm_vm_timer_irq_active)[ARM_VM_TIMER_TYPE_PHYSICAL]);
308 #endif
309
310 register_CNTVOFF_EL2_write(CNTVOFF_EL2_cast(0U));
311
312 for (int tt = ENUM_ARM_VM_TIMER_TYPE_MIN_VALUE;
313 tt < ARM_VM_TIMER_TYPE_NUM; tt++) {
314 if (arm_vm_timer_hwirq[tt] != NULL) {
315 irq_enable_local(arm_vm_timer_hwirq[tt]);
316 }
317 }
318 }
319
320 bool
arm_vm_timer_is_irq_enabled_thread(thread_t * thread,arm_vm_timer_type_t tt)321 arm_vm_timer_is_irq_enabled_thread(thread_t *thread, arm_vm_timer_type_t tt)
322 {
323 CNT_CTL_t cnt_ctl;
324
325 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
326 cnt_ctl = thread->vcpu_regs_el1.cntv_ctl_el0;
327 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
328 cnt_ctl = thread->vcpu_regs_el1.cntp_ctl_el0;
329 } else {
330 panic("Invalid timer");
331 }
332
333 return (CNT_CTL_get_ENABLE(&cnt_ctl) && !CNT_CTL_get_IMASK(&cnt_ctl));
334 }
335
336 ticks_t
arm_vm_timer_get_timeout_thread(thread_t * thread,arm_vm_timer_type_t tt)337 arm_vm_timer_get_timeout_thread(thread_t *thread, arm_vm_timer_type_t tt)
338 {
339 CNT_CVAL_t cnt_cval;
340
341 if (tt == ARM_VM_TIMER_TYPE_VIRTUAL) {
342 cnt_cval = thread->vcpu_regs_el1.cntv_cval_el0;
343 } else if (tt == ARM_VM_TIMER_TYPE_PHYSICAL) {
344 cnt_cval = thread->vcpu_regs_el1.cntp_cval_el0;
345 } else {
346 panic("Invalid timer");
347 }
348
349 return CNT_CVAL_get_CompareValue(&cnt_cval);
350 }
351
352 void
arm_vm_timer_load_state(thread_t * thread)353 arm_vm_timer_load_state(thread_t *thread)
354 {
355 register_CNTKCTL_EL1_write_ordered(thread->vcpu_regs_el1.cntkctl_el1,
356 &asm_ordering);
357 register_CNTV_CTL_EL0_write_ordered(thread->vcpu_regs_el1.cntv_ctl_el0,
358 &asm_ordering);
359 register_CNTV_CVAL_EL0_write_ordered(
360 thread->vcpu_regs_el1.cntv_cval_el0, &asm_ordering);
361 register_CNTP_CTL_EL0_write_ordered(thread->vcpu_regs_el1.cntp_ctl_el0,
362 &asm_ordering);
363 register_CNTP_CVAL_EL0_write_ordered(
364 thread->vcpu_regs_el1.cntp_cval_el0, &asm_ordering);
365 }
366
367 void
arm_vm_timer_handle_thread_save_state(void)368 arm_vm_timer_handle_thread_save_state(void)
369 {
370 thread_t *thread = thread_get_self();
371
372 if ((compiler_expected(thread->kind == THREAD_KIND_VCPU)) &&
373 !scheduler_is_blocked(thread, SCHEDULER_BLOCK_VCPU_OFF)) {
374 thread->vcpu_regs_el1.cntkctl_el1 = register_CNTKCTL_EL1_read();
375 thread->vcpu_regs_el1.cntv_ctl_el0 =
376 register_CNTV_CTL_EL0_read();
377 thread->vcpu_regs_el1.cntv_cval_el0 =
378 register_CNTV_CVAL_EL0_read();
379 thread->vcpu_regs_el1.cntp_ctl_el0 =
380 register_CNTP_CTL_EL0_read();
381 thread->vcpu_regs_el1.cntp_cval_el0 =
382 register_CNTP_CVAL_EL0_read();
383 }
384 }
385