1 // Copyright 2016 The Fuchsia Authors
2 // Copyright (c) 2013, Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a MIT-style
5 // license that can be found in the LICENSE file or at
6 // https://opensource.org/licenses/MIT
7
8 #include <dev/timer/arm_generic.h>
9
10 #include <arch/ops.h>
11 #include <assert.h>
12 #include <dev/interrupt.h>
13 #include <inttypes.h>
14 #include <lib/fixed_point.h>
15 #include <lk/init.h>
16 #include <pdev/driver.h>
17 #include <platform.h>
18 #include <platform/timer.h>
19 #include <trace.h>
20 #include <zircon/boot/driver-config.h>
21 #include <zircon/types.h>
22
23 #define LOCAL_TRACE 0
24
25 /* CNTFRQ AArch64 register */
26 #define TIMER_REG_CNTFRQ "cntfrq_el0"
27
28 /* CNTP AArch64 registers */
29 #define TIMER_REG_CNTP_CTL "cntp_ctl_el0"
30 #define TIMER_REG_CNTP_CVAL "cntp_cval_el0"
31 #define TIMER_REG_CNTP_TVAL "cntp_tval_el0"
32 #define TIMER_REG_CNTPCT "cntpct_el0"
33
34 /* CNTPS "AArch64" registers */
35 #define TIMER_REG_CNTPS_CTL "cntps_ctl_el1"
36 #define TIMER_REG_CNTPS_CVAL "cntps_cval_el1"
37 #define TIMER_REG_CNTPS_TVAL "cntps_tval_el1"
38
39 /* CNTV "AArch64" registers */
40 #define TIMER_REG_CNTV_CTL "cntv_ctl_el0"
41 #define TIMER_REG_CNTV_CVAL "cntv_cval_el0"
42 #define TIMER_REG_CNTV_TVAL "cntv_tval_el0"
43 #define TIMER_REG_CNTVCT "cntvct_el0"
44
45 static int timer_irq;
46
47 struct fp_32_64 cntpct_per_ns;
48 struct fp_32_64 ns_per_cntpct;
49
zx_time_to_cntpct(zx_time_t zx_time)50 static uint64_t zx_time_to_cntpct(zx_time_t zx_time) {
51 DEBUG_ASSERT(zx_time >= 0);
52 return u64_mul_u64_fp32_64(zx_time, cntpct_per_ns);
53 }
54
cntpct_to_zx_time(uint64_t cntpct)55 zx_time_t cntpct_to_zx_time(uint64_t cntpct) {
56 return u64_mul_u64_fp32_64(cntpct, ns_per_cntpct);
57 }
58
read_cntfrq(void)59 static uint32_t read_cntfrq(void) {
60 uint32_t cntfrq;
61
62 cntfrq = __arm_rsr(TIMER_REG_CNTFRQ);
63 LTRACEF("cntfrq: 0x%08x, %u\n", cntfrq, cntfrq);
64 return cntfrq;
65 }
66
read_cntp_ctl(void)67 static uint32_t read_cntp_ctl(void) {
68 return __arm_rsr(TIMER_REG_CNTP_CTL);
69 }
70
read_cntv_ctl(void)71 static uint32_t read_cntv_ctl(void) {
72 return __arm_rsr(TIMER_REG_CNTV_CTL);
73 }
74
read_cntps_ctl(void)75 static uint32_t read_cntps_ctl(void) {
76 return __arm_rsr(TIMER_REG_CNTPS_CTL);
77 }
78
write_cntp_ctl(uint32_t val)79 static void write_cntp_ctl(uint32_t val) {
80 LTRACEF_LEVEL(3, "cntp_ctl: 0x%x %x\n", val, read_cntp_ctl());
81 __arm_wsr(TIMER_REG_CNTP_CTL, val);
82 __isb(ARM_MB_SY);
83 }
84
write_cntv_ctl(uint32_t val)85 static void write_cntv_ctl(uint32_t val) {
86 LTRACEF_LEVEL(3, "cntv_ctl: 0x%x %x\n", val, read_cntv_ctl());
87 __arm_wsr(TIMER_REG_CNTV_CTL, val);
88 __isb(ARM_MB_SY);
89 }
90
write_cntps_ctl(uint32_t val)91 static void write_cntps_ctl(uint32_t val) {
92 LTRACEF_LEVEL(3, "cntps_ctl: 0x%x %x\n", val, read_cntps_ctl());
93 __arm_wsr(TIMER_REG_CNTPS_CTL, val);
94 __isb(ARM_MB_SY);
95 }
96
write_cntp_cval(uint64_t val)97 static void write_cntp_cval(uint64_t val) {
98 LTRACEF_LEVEL(3, "cntp_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
99 val, val);
100 __arm_wsr64(TIMER_REG_CNTP_CVAL, val);
101 __isb(ARM_MB_SY);
102 }
103
write_cntv_cval(uint64_t val)104 static void write_cntv_cval(uint64_t val) {
105 LTRACEF_LEVEL(3, "cntv_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
106 val, val);
107 __arm_wsr64(TIMER_REG_CNTV_CVAL, val);
108 __isb(ARM_MB_SY);
109 }
110
write_cntps_cval(uint64_t val)111 static void write_cntps_cval(uint64_t val) {
112 LTRACEF_LEVEL(3, "cntps_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
113 val, val);
114 __arm_wsr64(TIMER_REG_CNTPS_CVAL, val);
115 __isb(ARM_MB_SY);
116 }
117
write_cntp_tval(int32_t val)118 static void write_cntp_tval(int32_t val) {
119 LTRACEF_LEVEL(3, "cntp_tval: %d\n", val);
120 __arm_wsr(TIMER_REG_CNTP_TVAL, val);
121 __isb(ARM_MB_SY);
122 }
123
write_cntv_tval(int32_t val)124 static void write_cntv_tval(int32_t val) {
125 LTRACEF_LEVEL(3, "cntv_tval: %d\n", val);
126 __arm_wsr(TIMER_REG_CNTV_TVAL, val);
127 __isb(ARM_MB_SY);
128 }
129
write_cntps_tval(int32_t val)130 static void write_cntps_tval(int32_t val) {
131 LTRACEF_LEVEL(3, "cntps_tval: %d\n", val);
132 __arm_wsr(TIMER_REG_CNTPS_TVAL, val);
133 __isb(ARM_MB_SY);
134 }
135
read_cntpct(void)136 static uint64_t read_cntpct(void) {
137 return __arm_rsr64(TIMER_REG_CNTPCT);
138 }
139
read_cntvct(void)140 static uint64_t read_cntvct(void) {
141 return __arm_rsr64(TIMER_REG_CNTVCT);
142 }
143
144 struct timer_reg_procs {
145 void (*write_ctl)(uint32_t val);
146 void (*write_cval)(uint64_t val);
147 void (*write_tval)(int32_t val);
148 uint64_t (*read_ct)(void);
149 };
150
151 __UNUSED static const struct timer_reg_procs cntp_procs = {
152 .write_ctl = write_cntp_ctl,
153 .write_cval = write_cntp_cval,
154 .write_tval = write_cntp_tval,
155 .read_ct = read_cntpct,
156 };
157
158 __UNUSED static const struct timer_reg_procs cntv_procs = {
159 .write_ctl = write_cntv_ctl,
160 .write_cval = write_cntv_cval,
161 .write_tval = write_cntv_tval,
162 .read_ct = read_cntvct,
163 };
164
165 __UNUSED static const struct timer_reg_procs cntps_procs = {
166 .write_ctl = write_cntps_ctl,
167 .write_cval = write_cntps_cval,
168 .write_tval = write_cntps_tval,
169 .read_ct = read_cntpct,
170 };
171
172 #if (TIMER_ARM_GENERIC_SELECTED_CNTV)
173 static const struct timer_reg_procs* reg_procs = &cntv_procs;
174 #else
175 static const struct timer_reg_procs* reg_procs = &cntp_procs;
176 #endif
177
write_ctl(uint32_t val)178 static inline void write_ctl(uint32_t val) {
179 reg_procs->write_ctl(val);
180 }
181
write_cval(uint64_t val)182 static inline void write_cval(uint64_t val) {
183 reg_procs->write_cval(val);
184 }
185
write_tval(uint32_t val)186 static inline void write_tval(uint32_t val) {
187 reg_procs->write_tval(val);
188 }
189
read_ct(void)190 static uint64_t read_ct(void) {
191 uint64_t cntpct = reg_procs->read_ct();
192 LTRACEF_LEVEL(3, "cntpct: 0x%016" PRIx64 ", %" PRIu64 "\n",
193 cntpct, cntpct);
194 return cntpct;
195 }
196
platform_tick(void * arg)197 static interrupt_eoi platform_tick(void* arg) {
198 write_ctl(0);
199 timer_tick(current_time());
200 return IRQ_EOI_DEACTIVATE;
201 }
202
platform_set_oneshot_timer(zx_time_t deadline)203 zx_status_t platform_set_oneshot_timer(zx_time_t deadline) {
204 DEBUG_ASSERT(arch_ints_disabled());
205
206 if (deadline < 0) {
207 deadline = 0;
208 }
209
210 // Add one to the deadline, since with very high probability the deadline
211 // straddles a counter tick.
212 const uint64_t cntpct_deadline = zx_time_to_cntpct(deadline) + 1;
213
214 // Even if the deadline has already passed, the ARMv8-A timer will fire the
215 // interrupt.
216 write_cval(cntpct_deadline);
217 write_ctl(1);
218
219 return 0;
220 }
221
platform_stop_timer(void)222 void platform_stop_timer(void) {
223 write_ctl(0);
224 }
225
platform_shutdown_timer(void)226 void platform_shutdown_timer(void) {
227 DEBUG_ASSERT(arch_ints_disabled());
228 mask_interrupt(timer_irq);
229 }
230
current_time(void)231 zx_time_t current_time(void) {
232 return cntpct_to_zx_time(read_ct());
233 }
234
current_ticks(void)235 zx_ticks_t current_ticks(void) {
236 return read_ct();
237 }
238
ticks_per_second(void)239 zx_ticks_t ticks_per_second(void) {
240 return u64_mul_u32_fp32_64(1000 * 1000 * 1000, cntpct_per_ns);
241 }
242
abs_int64(int64_t a)243 static uint64_t abs_int64(int64_t a) {
244 return (a > 0) ? a : -a;
245 }
246
test_time_conversion_check_result(uint64_t a,uint64_t b,uint64_t limit)247 static void test_time_conversion_check_result(uint64_t a, uint64_t b, uint64_t limit) {
248 if (a != b) {
249 uint64_t diff = abs_int64(a - b);
250 if (diff <= limit)
251 LTRACEF("ROUNDED by %" PRIu64 " (up to %" PRIu64 " allowed)\n", diff, limit);
252 else
253 TRACEF("FAIL, off by %" PRIu64 "\n", diff);
254 }
255 }
256
test_zx_time_to_cntpct(uint32_t cntfrq,zx_time_t zx_time)257 static void test_zx_time_to_cntpct(uint32_t cntfrq, zx_time_t zx_time) {
258 uint64_t cntpct = zx_time_to_cntpct(zx_time);
259 const uint64_t nanos_per_sec = ZX_SEC(1);
260 uint64_t expected_cntpct = ((uint64_t)cntfrq * zx_time + nanos_per_sec / 2) / nanos_per_sec;
261
262 test_time_conversion_check_result(cntpct, expected_cntpct, 1);
263 LTRACEF_LEVEL(2, "zx_time_to_cntpct(%" PRIi64 "): got %" PRIu64
264 ", expect %" PRIu64 "\n",
265 zx_time, cntpct, expected_cntpct);
266 }
267
test_cntpct_to_zx_time(uint32_t cntfrq,uint64_t expected_s)268 static void test_cntpct_to_zx_time(uint32_t cntfrq, uint64_t expected_s) {
269 zx_time_t expected_zx_time = ZX_SEC(expected_s);
270 uint64_t cntpct = (uint64_t)cntfrq * expected_s;
271 zx_time_t zx_time = cntpct_to_zx_time(cntpct);
272
273 test_time_conversion_check_result(zx_time, expected_zx_time, (1000 * 1000 + cntfrq - 1) / cntfrq);
274 LTRACEF_LEVEL(2, "cntpct_to_zx_time(%" PRIu64
275 "): got %" PRIi64 ", expect %" PRIi64 "\n",
276 cntpct, zx_time, expected_zx_time);
277 }
278
test_time_conversions(uint32_t cntfrq)279 static void test_time_conversions(uint32_t cntfrq) {
280 test_zx_time_to_cntpct(cntfrq, 0);
281 test_zx_time_to_cntpct(cntfrq, 1);
282 test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24);
283 test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24 * 365);
284 test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
285 test_zx_time_to_cntpct(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
286 test_zx_time_to_cntpct(cntfrq, 1ULL << 60);
287 test_cntpct_to_zx_time(cntfrq, 0);
288 test_cntpct_to_zx_time(cntfrq, 1);
289 test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24);
290 test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24 * 365);
291 test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
292 test_cntpct_to_zx_time(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
293 }
294
arm_generic_timer_init_conversion_factors(uint32_t cntfrq)295 static void arm_generic_timer_init_conversion_factors(uint32_t cntfrq) {
296 fp_32_64_div_32_32(&cntpct_per_ns, cntfrq, ZX_SEC(1));
297 fp_32_64_div_32_32(&ns_per_cntpct, ZX_SEC(1), cntfrq);
298 dprintf(SPEW, "cntpct_per_ns: %08x.%08x%08x\n", cntpct_per_ns.l0, cntpct_per_ns.l32, cntpct_per_ns.l64);
299 dprintf(SPEW, "ns_per_cntpct: %08x.%08x%08x\n", ns_per_cntpct.l0, ns_per_cntpct.l32, ns_per_cntpct.l64);
300 }
301
arm_generic_timer_init(uint32_t freq_override)302 static void arm_generic_timer_init(uint32_t freq_override) {
303 uint32_t cntfrq;
304
305 if (freq_override == 0) {
306 cntfrq = read_cntfrq();
307
308 if (!cntfrq) {
309 TRACEF("Failed to initialize timer, frequency is 0\n");
310 return;
311 }
312 } else {
313 cntfrq = freq_override;
314 }
315
316 dprintf(INFO, "arm generic timer freq %u Hz\n", cntfrq);
317
318 #if LOCAL_TRACE
319 LTRACEF("Test min cntfrq\n");
320 arm_generic_timer_init_conversion_factors(1);
321 test_time_conversions(1);
322 LTRACEF("Test max cntfrq\n");
323 arm_generic_timer_init_conversion_factors(~0);
324 test_time_conversions(~0);
325 LTRACEF("Set actual cntfrq\n");
326 #endif
327 arm_generic_timer_init_conversion_factors(cntfrq);
328 test_time_conversions(cntfrq);
329
330 LTRACEF("register irq %d on cpu %u\n", timer_irq, arch_curr_cpu_num());
331 zx_status_t status = register_int_handler(timer_irq, &platform_tick, NULL);
332 DEBUG_ASSERT(status == ZX_OK);
333 unmask_interrupt(timer_irq);
334 }
335
arm_generic_timer_init_secondary_cpu(uint level)336 static void arm_generic_timer_init_secondary_cpu(uint level) {
337 LTRACEF("unmask irq %d on cpu %u\n", timer_irq, arch_curr_cpu_num());
338 unmask_interrupt(timer_irq);
339 }
340
341 /* secondary cpu initialize the timer just before the kernel starts with interrupts enabled */
342 LK_INIT_HOOK_FLAGS(arm_generic_timer_init_secondary_cpu,
343 arm_generic_timer_init_secondary_cpu,
344 LK_INIT_LEVEL_THREADING - 1, LK_INIT_FLAG_SECONDARY_CPUS);
345
arm_generic_timer_resume_cpu(uint level)346 static void arm_generic_timer_resume_cpu(uint level) {
347 /* Always trigger a timer interrupt on each cpu for now */
348 write_tval(0);
349 write_ctl(1);
350 }
351
352 LK_INIT_HOOK_FLAGS(arm_generic_timer_resume_cpu, arm_generic_timer_resume_cpu,
353 LK_INIT_LEVEL_PLATFORM, LK_INIT_FLAG_CPU_RESUME);
354
arm_generic_timer_pdev_init(const void * driver_data,uint32_t length)355 static void arm_generic_timer_pdev_init(const void* driver_data, uint32_t length) {
356 ASSERT(length >= sizeof(dcfg_arm_generic_timer_driver_t));
357 auto driver = static_cast<const dcfg_arm_generic_timer_driver_t*>(driver_data);
358 uint32_t irq_phys = driver->irq_phys;
359 uint32_t irq_virt = driver->irq_virt;
360 uint32_t irq_sphys = driver->irq_sphys;
361
362 if (irq_phys && irq_virt && arm64_get_boot_el() < 2) {
363 // If we did not boot at EL2 or above, prefer the virtual timer.
364 irq_phys = 0;
365 }
366 if (irq_phys) {
367 timer_irq = irq_phys;
368 reg_procs = &cntp_procs;
369 } else if (irq_virt) {
370 timer_irq = irq_virt;
371 reg_procs = &cntv_procs;
372 } else if (irq_sphys) {
373 timer_irq = irq_sphys;
374 reg_procs = &cntps_procs;
375 } else {
376 panic("no irqs set in arm_generic_timer_pdev_init\n");
377 }
378 smp_mb();
379
380 arm_generic_timer_init(driver->freq_override);
381 }
382
383 LK_PDEV_INIT(arm_generic_timer_pdev_init, KDRV_ARM_GENERIC_TIMER, arm_generic_timer_pdev_init, LK_INIT_LEVEL_PLATFORM_EARLY);
384