1 // Copyright 2016 The Fuchsia Authors
2 // Copyright (c) 2013, Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a MIT-style
5 // license that can be found in the LICENSE file or at
6 // https://opensource.org/licenses/MIT
7 
8 #include <dev/timer/arm_generic.h>
9 
10 #include <arch/ops.h>
11 #include <assert.h>
12 #include <dev/interrupt.h>
13 #include <inttypes.h>
14 #include <lib/fixed_point.h>
15 #include <lk/init.h>
16 #include <pdev/driver.h>
17 #include <platform.h>
18 #include <platform/timer.h>
19 #include <trace.h>
20 #include <zircon/boot/driver-config.h>
21 #include <zircon/types.h>
22 
23 #define LOCAL_TRACE 0
24 
25 /* CNTFRQ AArch64 register */
26 #define TIMER_REG_CNTFRQ "cntfrq_el0"
27 
28 /* CNTP AArch64 registers */
29 #define TIMER_REG_CNTP_CTL "cntp_ctl_el0"
30 #define TIMER_REG_CNTP_CVAL "cntp_cval_el0"
31 #define TIMER_REG_CNTP_TVAL "cntp_tval_el0"
32 #define TIMER_REG_CNTPCT "cntpct_el0"
33 
34 /* CNTPS "AArch64" registers */
35 #define TIMER_REG_CNTPS_CTL "cntps_ctl_el1"
36 #define TIMER_REG_CNTPS_CVAL "cntps_cval_el1"
37 #define TIMER_REG_CNTPS_TVAL "cntps_tval_el1"
38 
39 /* CNTV "AArch64" registers */
40 #define TIMER_REG_CNTV_CTL "cntv_ctl_el0"
41 #define TIMER_REG_CNTV_CVAL "cntv_cval_el0"
42 #define TIMER_REG_CNTV_TVAL "cntv_tval_el0"
43 #define TIMER_REG_CNTVCT "cntvct_el0"
44 
45 static int timer_irq;
46 
47 struct fp_32_64 cntpct_per_ns;
48 struct fp_32_64 ns_per_cntpct;
49 
zx_time_to_cntpct(zx_time_t zx_time)50 static uint64_t zx_time_to_cntpct(zx_time_t zx_time) {
51     DEBUG_ASSERT(zx_time >= 0);
52     return u64_mul_u64_fp32_64(zx_time, cntpct_per_ns);
53 }
54 
cntpct_to_zx_time(uint64_t cntpct)55 zx_time_t cntpct_to_zx_time(uint64_t cntpct) {
56     return u64_mul_u64_fp32_64(cntpct, ns_per_cntpct);
57 }
58 
read_cntfrq(void)59 static uint32_t read_cntfrq(void) {
60     uint32_t cntfrq;
61 
62     cntfrq = __arm_rsr(TIMER_REG_CNTFRQ);
63     LTRACEF("cntfrq: 0x%08x, %u\n", cntfrq, cntfrq);
64     return cntfrq;
65 }
66 
read_cntp_ctl(void)67 static uint32_t read_cntp_ctl(void) {
68     return __arm_rsr(TIMER_REG_CNTP_CTL);
69 }
70 
read_cntv_ctl(void)71 static uint32_t read_cntv_ctl(void) {
72     return __arm_rsr(TIMER_REG_CNTV_CTL);
73 }
74 
read_cntps_ctl(void)75 static uint32_t read_cntps_ctl(void) {
76     return __arm_rsr(TIMER_REG_CNTPS_CTL);
77 }
78 
write_cntp_ctl(uint32_t val)79 static void write_cntp_ctl(uint32_t val) {
80     LTRACEF_LEVEL(3, "cntp_ctl: 0x%x %x\n", val, read_cntp_ctl());
81     __arm_wsr(TIMER_REG_CNTP_CTL, val);
82     __isb(ARM_MB_SY);
83 }
84 
write_cntv_ctl(uint32_t val)85 static void write_cntv_ctl(uint32_t val) {
86     LTRACEF_LEVEL(3, "cntv_ctl: 0x%x %x\n", val, read_cntv_ctl());
87     __arm_wsr(TIMER_REG_CNTV_CTL, val);
88     __isb(ARM_MB_SY);
89 }
90 
write_cntps_ctl(uint32_t val)91 static void write_cntps_ctl(uint32_t val) {
92     LTRACEF_LEVEL(3, "cntps_ctl: 0x%x %x\n", val, read_cntps_ctl());
93     __arm_wsr(TIMER_REG_CNTPS_CTL, val);
94     __isb(ARM_MB_SY);
95 }
96 
write_cntp_cval(uint64_t val)97 static void write_cntp_cval(uint64_t val) {
98     LTRACEF_LEVEL(3, "cntp_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
99                   val, val);
100     __arm_wsr64(TIMER_REG_CNTP_CVAL, val);
101     __isb(ARM_MB_SY);
102 }
103 
write_cntv_cval(uint64_t val)104 static void write_cntv_cval(uint64_t val) {
105     LTRACEF_LEVEL(3, "cntv_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
106                   val, val);
107     __arm_wsr64(TIMER_REG_CNTV_CVAL, val);
108     __isb(ARM_MB_SY);
109 }
110 
write_cntps_cval(uint64_t val)111 static void write_cntps_cval(uint64_t val) {
112     LTRACEF_LEVEL(3, "cntps_cval: 0x%016" PRIx64 ", %" PRIu64 "\n",
113                   val, val);
114     __arm_wsr64(TIMER_REG_CNTPS_CVAL, val);
115     __isb(ARM_MB_SY);
116 }
117 
write_cntp_tval(int32_t val)118 static void write_cntp_tval(int32_t val) {
119     LTRACEF_LEVEL(3, "cntp_tval: %d\n", val);
120     __arm_wsr(TIMER_REG_CNTP_TVAL, val);
121     __isb(ARM_MB_SY);
122 }
123 
write_cntv_tval(int32_t val)124 static void write_cntv_tval(int32_t val) {
125     LTRACEF_LEVEL(3, "cntv_tval: %d\n", val);
126     __arm_wsr(TIMER_REG_CNTV_TVAL, val);
127     __isb(ARM_MB_SY);
128 }
129 
write_cntps_tval(int32_t val)130 static void write_cntps_tval(int32_t val) {
131     LTRACEF_LEVEL(3, "cntps_tval: %d\n", val);
132     __arm_wsr(TIMER_REG_CNTPS_TVAL, val);
133     __isb(ARM_MB_SY);
134 }
135 
read_cntpct(void)136 static uint64_t read_cntpct(void) {
137     return __arm_rsr64(TIMER_REG_CNTPCT);
138 }
139 
read_cntvct(void)140 static uint64_t read_cntvct(void) {
141     return __arm_rsr64(TIMER_REG_CNTVCT);
142 }
143 
144 struct timer_reg_procs {
145     void (*write_ctl)(uint32_t val);
146     void (*write_cval)(uint64_t val);
147     void (*write_tval)(int32_t val);
148     uint64_t (*read_ct)(void);
149 };
150 
151 __UNUSED static const struct timer_reg_procs cntp_procs = {
152     .write_ctl = write_cntp_ctl,
153     .write_cval = write_cntp_cval,
154     .write_tval = write_cntp_tval,
155     .read_ct = read_cntpct,
156 };
157 
158 __UNUSED static const struct timer_reg_procs cntv_procs = {
159     .write_ctl = write_cntv_ctl,
160     .write_cval = write_cntv_cval,
161     .write_tval = write_cntv_tval,
162     .read_ct = read_cntvct,
163 };
164 
165 __UNUSED static const struct timer_reg_procs cntps_procs = {
166     .write_ctl = write_cntps_ctl,
167     .write_cval = write_cntps_cval,
168     .write_tval = write_cntps_tval,
169     .read_ct = read_cntpct,
170 };
171 
172 #if (TIMER_ARM_GENERIC_SELECTED_CNTV)
173 static const struct timer_reg_procs* reg_procs = &cntv_procs;
174 #else
175 static const struct timer_reg_procs* reg_procs = &cntp_procs;
176 #endif
177 
write_ctl(uint32_t val)178 static inline void write_ctl(uint32_t val) {
179     reg_procs->write_ctl(val);
180 }
181 
write_cval(uint64_t val)182 static inline void write_cval(uint64_t val) {
183     reg_procs->write_cval(val);
184 }
185 
write_tval(uint32_t val)186 static inline void write_tval(uint32_t val) {
187     reg_procs->write_tval(val);
188 }
189 
read_ct(void)190 static uint64_t read_ct(void) {
191     uint64_t cntpct = reg_procs->read_ct();
192     LTRACEF_LEVEL(3, "cntpct: 0x%016" PRIx64 ", %" PRIu64 "\n",
193                   cntpct, cntpct);
194     return cntpct;
195 }
196 
platform_tick(void * arg)197 static interrupt_eoi platform_tick(void* arg) {
198     write_ctl(0);
199     timer_tick(current_time());
200     return IRQ_EOI_DEACTIVATE;
201 }
202 
platform_set_oneshot_timer(zx_time_t deadline)203 zx_status_t platform_set_oneshot_timer(zx_time_t deadline) {
204     DEBUG_ASSERT(arch_ints_disabled());
205 
206     if (deadline < 0) {
207         deadline = 0;
208     }
209 
210     // Add one to the deadline, since with very high probability the deadline
211     // straddles a counter tick.
212     const uint64_t cntpct_deadline = zx_time_to_cntpct(deadline) + 1;
213 
214     // Even if the deadline has already passed, the ARMv8-A timer will fire the
215     // interrupt.
216     write_cval(cntpct_deadline);
217     write_ctl(1);
218 
219     return 0;
220 }
221 
platform_stop_timer(void)222 void platform_stop_timer(void) {
223     write_ctl(0);
224 }
225 
platform_shutdown_timer(void)226 void platform_shutdown_timer(void) {
227     DEBUG_ASSERT(arch_ints_disabled());
228     mask_interrupt(timer_irq);
229 }
230 
current_time(void)231 zx_time_t current_time(void) {
232     return cntpct_to_zx_time(read_ct());
233 }
234 
current_ticks(void)235 zx_ticks_t current_ticks(void) {
236     return read_ct();
237 }
238 
ticks_per_second(void)239 zx_ticks_t ticks_per_second(void) {
240     return u64_mul_u32_fp32_64(1000 * 1000 * 1000, cntpct_per_ns);
241 }
242 
abs_int64(int64_t a)243 static uint64_t abs_int64(int64_t a) {
244     return (a > 0) ? a : -a;
245 }
246 
test_time_conversion_check_result(uint64_t a,uint64_t b,uint64_t limit)247 static void test_time_conversion_check_result(uint64_t a, uint64_t b, uint64_t limit) {
248     if (a != b) {
249         uint64_t diff = abs_int64(a - b);
250         if (diff <= limit)
251             LTRACEF("ROUNDED by %" PRIu64 " (up to %" PRIu64 " allowed)\n", diff, limit);
252         else
253             TRACEF("FAIL, off by %" PRIu64 "\n", diff);
254     }
255 }
256 
test_zx_time_to_cntpct(uint32_t cntfrq,zx_time_t zx_time)257 static void test_zx_time_to_cntpct(uint32_t cntfrq, zx_time_t zx_time) {
258     uint64_t cntpct = zx_time_to_cntpct(zx_time);
259     const uint64_t nanos_per_sec = ZX_SEC(1);
260     uint64_t expected_cntpct = ((uint64_t)cntfrq * zx_time + nanos_per_sec / 2) / nanos_per_sec;
261 
262     test_time_conversion_check_result(cntpct, expected_cntpct, 1);
263     LTRACEF_LEVEL(2, "zx_time_to_cntpct(%" PRIi64 "): got %" PRIu64
264                      ", expect %" PRIu64 "\n",
265                   zx_time, cntpct, expected_cntpct);
266 }
267 
test_cntpct_to_zx_time(uint32_t cntfrq,uint64_t expected_s)268 static void test_cntpct_to_zx_time(uint32_t cntfrq, uint64_t expected_s) {
269     zx_time_t expected_zx_time = ZX_SEC(expected_s);
270     uint64_t cntpct = (uint64_t)cntfrq * expected_s;
271     zx_time_t zx_time = cntpct_to_zx_time(cntpct);
272 
273     test_time_conversion_check_result(zx_time, expected_zx_time, (1000 * 1000 + cntfrq - 1) / cntfrq);
274     LTRACEF_LEVEL(2, "cntpct_to_zx_time(%" PRIu64
275                      "): got %" PRIi64 ", expect %" PRIi64 "\n",
276                   cntpct, zx_time, expected_zx_time);
277 }
278 
test_time_conversions(uint32_t cntfrq)279 static void test_time_conversions(uint32_t cntfrq) {
280     test_zx_time_to_cntpct(cntfrq, 0);
281     test_zx_time_to_cntpct(cntfrq, 1);
282     test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24);
283     test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24 * 365);
284     test_zx_time_to_cntpct(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
285     test_zx_time_to_cntpct(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
286     test_zx_time_to_cntpct(cntfrq, 1ULL << 60);
287     test_cntpct_to_zx_time(cntfrq, 0);
288     test_cntpct_to_zx_time(cntfrq, 1);
289     test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24);
290     test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24 * 365);
291     test_cntpct_to_zx_time(cntfrq, 60 * 60 * 24 * (365 * 10 + 2));
292     test_cntpct_to_zx_time(cntfrq, 60ULL * 60 * 24 * (365 * 100 + 2));
293 }
294 
arm_generic_timer_init_conversion_factors(uint32_t cntfrq)295 static void arm_generic_timer_init_conversion_factors(uint32_t cntfrq) {
296     fp_32_64_div_32_32(&cntpct_per_ns, cntfrq, ZX_SEC(1));
297     fp_32_64_div_32_32(&ns_per_cntpct, ZX_SEC(1), cntfrq);
298     dprintf(SPEW, "cntpct_per_ns: %08x.%08x%08x\n", cntpct_per_ns.l0, cntpct_per_ns.l32, cntpct_per_ns.l64);
299     dprintf(SPEW, "ns_per_cntpct: %08x.%08x%08x\n", ns_per_cntpct.l0, ns_per_cntpct.l32, ns_per_cntpct.l64);
300 }
301 
arm_generic_timer_init(uint32_t freq_override)302 static void arm_generic_timer_init(uint32_t freq_override) {
303     uint32_t cntfrq;
304 
305     if (freq_override == 0) {
306         cntfrq = read_cntfrq();
307 
308         if (!cntfrq) {
309             TRACEF("Failed to initialize timer, frequency is 0\n");
310             return;
311         }
312     } else {
313         cntfrq = freq_override;
314     }
315 
316     dprintf(INFO, "arm generic timer freq %u Hz\n", cntfrq);
317 
318 #if LOCAL_TRACE
319     LTRACEF("Test min cntfrq\n");
320     arm_generic_timer_init_conversion_factors(1);
321     test_time_conversions(1);
322     LTRACEF("Test max cntfrq\n");
323     arm_generic_timer_init_conversion_factors(~0);
324     test_time_conversions(~0);
325     LTRACEF("Set actual cntfrq\n");
326 #endif
327     arm_generic_timer_init_conversion_factors(cntfrq);
328     test_time_conversions(cntfrq);
329 
330     LTRACEF("register irq %d on cpu %u\n", timer_irq, arch_curr_cpu_num());
331     zx_status_t status = register_int_handler(timer_irq, &platform_tick, NULL);
332     DEBUG_ASSERT(status == ZX_OK);
333     unmask_interrupt(timer_irq);
334 }
335 
arm_generic_timer_init_secondary_cpu(uint level)336 static void arm_generic_timer_init_secondary_cpu(uint level) {
337     LTRACEF("unmask irq %d on cpu %u\n", timer_irq, arch_curr_cpu_num());
338     unmask_interrupt(timer_irq);
339 }
340 
341 /* secondary cpu initialize the timer just before the kernel starts with interrupts enabled */
342 LK_INIT_HOOK_FLAGS(arm_generic_timer_init_secondary_cpu,
343                    arm_generic_timer_init_secondary_cpu,
344                    LK_INIT_LEVEL_THREADING - 1, LK_INIT_FLAG_SECONDARY_CPUS);
345 
arm_generic_timer_resume_cpu(uint level)346 static void arm_generic_timer_resume_cpu(uint level) {
347     /* Always trigger a timer interrupt on each cpu for now */
348     write_tval(0);
349     write_ctl(1);
350 }
351 
352 LK_INIT_HOOK_FLAGS(arm_generic_timer_resume_cpu, arm_generic_timer_resume_cpu,
353                    LK_INIT_LEVEL_PLATFORM, LK_INIT_FLAG_CPU_RESUME);
354 
arm_generic_timer_pdev_init(const void * driver_data,uint32_t length)355 static void arm_generic_timer_pdev_init(const void* driver_data, uint32_t length) {
356     ASSERT(length >= sizeof(dcfg_arm_generic_timer_driver_t));
357     auto driver = static_cast<const dcfg_arm_generic_timer_driver_t*>(driver_data);
358     uint32_t irq_phys = driver->irq_phys;
359     uint32_t irq_virt = driver->irq_virt;
360     uint32_t irq_sphys = driver->irq_sphys;
361 
362     if (irq_phys && irq_virt && arm64_get_boot_el() < 2) {
363         // If we did not boot at EL2 or above, prefer the virtual timer.
364         irq_phys = 0;
365     }
366     if (irq_phys) {
367         timer_irq = irq_phys;
368         reg_procs = &cntp_procs;
369     } else if (irq_virt) {
370         timer_irq = irq_virt;
371         reg_procs = &cntv_procs;
372     } else if (irq_sphys) {
373         timer_irq = irq_sphys;
374         reg_procs = &cntps_procs;
375     } else {
376         panic("no irqs set in arm_generic_timer_pdev_init\n");
377     }
378     smp_mb();
379 
380     arm_generic_timer_init(driver->freq_override);
381 }
382 
383 LK_PDEV_INIT(arm_generic_timer_pdev_init, KDRV_ARM_GENERIC_TIMER, arm_generic_timer_pdev_init, LK_INIT_LEVEL_PLATFORM_EARLY);
384