1 // © 2021 Qualcomm Innovation Center, Inc. All rights reserved.
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4
5 #include <assert.h>
6 #include <hyptypes.h>
7 #include <string.h>
8
9 #include <atomic.h>
10 #include <compiler.h>
11 #include <cpulocal.h>
12 #include <hyp_aspace.h>
13 #include <log.h>
14 #include <panic.h>
15 #include <partition.h>
16 #include <pgtable.h>
17 #include <platform_security.h>
18 #include <platform_timer.h>
19 #include <trace.h>
20 #include <util.h>
21
22 #include "etm.h"
23 #include "event_handlers.h"
24
25 #if defined(PLATFORM_ETM_REG_WRITE_WORKAROUND)
26 // this work around is for context save, since we are writing lots of registers
27 // back to back, it could block other master on NOC.
28 #define CTX_WRITE_WORKAROUND platform_timer_ndelay(20000)
29 #else
30 #define CTX_WRITE_WORKAROUND
31 #endif
32
33 #if !defined(ETM_USE_SOFTWARE_LOCK)
34 // Using or implementing TRCLAR is deprecated. Linux doesn't use it.
35 #define ETM_USE_SOFTWARE_LOCK 0
36 #endif
37
38 typedef struct {
39 size_t reg_offset;
40 size_t access_size;
41 size_t count;
42 size_t stride;
43 } context_register_info_t;
44
45 static etm_t *mapped_etms[PLATFORM_MAX_CORES];
46
47 static register_t *etm_contexts[PLATFORM_MAX_CORES];
48
49 static uint32_t etm_claim_tag[PLATFORM_MAX_CORES];
50
51 static uint32_t etm_cprgctlr[PLATFORM_MAX_CORES];
52
53 #define ETM_REGISTER(name) \
54 { \
55 .reg_offset = offsetof(etm_t, name), \
56 .access_size = util_sizeof_member(etm_t, name), .count = 1, \
57 .stride = 0 \
58 }
59
60 #define ETM_REGISTER_ARRAY(name) \
61 { \
62 .reg_offset = offsetof(etm_t, name), \
63 .access_size = util_sizeof_member(etm_t, name[0]), \
64 .count = util_sizeof_member(etm_t, name) / \
65 util_sizeof_member(etm_t, name[0]), \
66 .stride = util_sizeof_member(etm_t, name[0]) \
67 }
68
69 #define ETM_REGISTER_SPARSE_ARRAY(name) \
70 { \
71 .reg_offset = offsetof(etm_t, name), \
72 .access_size = util_sizeof_member(etm_t, name[0].value), \
73 .count = util_sizeof_member(etm_t, name) / \
74 util_sizeof_member(etm_t, name[0]), \
75 .stride = util_sizeof_member(etm_t, name[0]) \
76 }
77
78 // NOTE: registers are saved in the context memory region based on their
79 // index in context_register_list. Make sure the alignment is correct
80 static const context_register_info_t context_register_list[] = {
81 // main control & configuration regsters
82 ETM_REGISTER(trcprocselr),
83 ETM_REGISTER(trcconfigr),
84 ETM_REGISTER(trcauxctlr),
85 ETM_REGISTER(trceventctl0r),
86 ETM_REGISTER(trceventctl1r),
87 ETM_REGISTER(trcstallctlr),
88 ETM_REGISTER(trctsctlr),
89 ETM_REGISTER(trcsyncpr),
90 ETM_REGISTER(trcccctlr),
91 ETM_REGISTER(trcbbctlr),
92 ETM_REGISTER(trctraceidr),
93 ETM_REGISTER(trcqctlr),
94
95 // filtering control registers
96 ETM_REGISTER(trcvictlr),
97 ETM_REGISTER(trcviiectlr),
98 ETM_REGISTER(trcvissctlr),
99 ETM_REGISTER(trcvipcssctlr),
100 ETM_REGISTER(trcvdctlr),
101 ETM_REGISTER(trcvdsacctlr),
102 ETM_REGISTER(trcvdarcctlr),
103
104 // derived resources registers
105 ETM_REGISTER_ARRAY(trcseqevr),
106 ETM_REGISTER(trcseqrstevr),
107 ETM_REGISTER(trcseqstr),
108 ETM_REGISTER(trcextinselr),
109 ETM_REGISTER_ARRAY(trccntrldvr),
110 ETM_REGISTER_ARRAY(trccntctlr),
111 ETM_REGISTER_ARRAY(trccntvr),
112
113 // resource selection registers
114 ETM_REGISTER_ARRAY(trcrsctlr2),
115
116 // comparator registers
117 ETM_REGISTER_ARRAY(trcacvr),
118 ETM_REGISTER_ARRAY(trcacatr),
119
120 ETM_REGISTER_SPARSE_ARRAY(trcdvcvr),
121 ETM_REGISTER_SPARSE_ARRAY(trcdvcmr),
122
123 ETM_REGISTER_ARRAY(trccidcvr),
124 ETM_REGISTER_ARRAY(trccidcctlr),
125
126 ETM_REGISTER_ARRAY(trcvmidcvr),
127 ETM_REGISTER_ARRAY(trcvmidcctlr),
128
129 // single shot comparator registers
130 ETM_REGISTER_ARRAY(trcssccr),
131 ETM_REGISTER_ARRAY(trcsscsr),
132 ETM_REGISTER_ARRAY(trcsspcicr),
133 };
134
135 static size_t
etm_get_context_size_percpu(void)136 etm_get_context_size_percpu(void)
137 {
138 size_t ret = 0UL;
139
140 // can be optimized by read last entry offset + size, but need to
141 // restrict the timing to call this context
142 for (index_t i = 0; i < util_array_size(context_register_list); i++) {
143 const context_register_info_t *info = &context_register_list[i];
144 ret += sizeof(uint64_t) * info->count;
145 }
146 return ret;
147 }
148
149 void
etm_handle_boot_hypervisor_start(void)150 etm_handle_boot_hypervisor_start(void)
151 {
152 if (compiler_expected(platform_security_state_debug_disabled())) {
153 goto out;
154 }
155
156 partition_t *hyp_partition = partition_get_private();
157
158 // FIXME: remove when read from device tree
159 paddr_t etm_base = PLATFORM_ETM_BASE;
160 size_t etm_stride = PLATFORM_ETM_STRIDE;
161
162 for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) {
163 virt_range_result_t range =
164 hyp_aspace_allocate(PLATFORM_ETM_SIZE_PERCPU);
165 if (range.e != OK) {
166 panic("ETM: Address allocation failed.");
167 }
168
169 paddr_t cur_base = etm_base + i * etm_stride;
170
171 pgtable_hyp_start();
172
173 error_t ret = pgtable_hyp_map(
174 hyp_partition, range.r.base, PLATFORM_ETM_SIZE_PERCPU,
175 cur_base, PGTABLE_HYP_MEMTYPE_NOSPEC_NOCOMBINE,
176 PGTABLE_ACCESS_RW, VMSA_SHAREABILITY_NON_SHAREABLE);
177 if (ret != OK) {
178 panic("ETM: Mapping of etm register failed.");
179 }
180 mapped_etms[i] = (etm_t *)range.r.base;
181
182 pgtable_hyp_commit();
183 }
184
185 // allocate contexts
186 size_t context_size = etm_get_context_size_percpu();
187 for (cpu_index_t i = 0; cpulocal_index_valid(i); i++) {
188 void_ptr_result_t alloc_r = partition_alloc(
189 hyp_partition, context_size, alignof(uint64_t));
190 if (alloc_r.e != OK) {
191 panic("failed to allocate ETM context memory");
192 }
193
194 etm_contexts[i] = (register_t *)alloc_r.r;
195 (void)memset_s(etm_contexts[i], context_size, 0, context_size);
196 }
197
198 out:
199 return;
200 }
201
202 void
etm_set_reg(cpu_index_t cpu,size_t offset,register_t val,size_t access_size)203 etm_set_reg(cpu_index_t cpu, size_t offset, register_t val, size_t access_size)
204 {
205 (void)access_size;
206
207 assert(cpulocal_index_valid(cpu));
208
209 assert(offset < (sizeof(*mapped_etms[cpu]) - access_size));
210 uintptr_t base = (uintptr_t)mapped_etms[cpu];
211
212 if (access_size == sizeof(uint32_t)) {
213 _Atomic uint32_t *reg = (_Atomic uint32_t *)(base + offset);
214 atomic_store_relaxed(reg, val);
215 } else if (access_size == sizeof(uint64_t)) {
216 _Atomic uint64_t *reg = (_Atomic uint64_t *)(base + offset);
217 atomic_store_relaxed(reg, val);
218 } else {
219 panic("ETM: invalid access size");
220 }
221 }
222
223 void
etm_get_reg(cpu_index_t cpu,size_t offset,register_t * val,size_t access_size)224 etm_get_reg(cpu_index_t cpu, size_t offset, register_t *val, size_t access_size)
225 {
226 (void)access_size;
227
228 assert(cpulocal_index_valid(cpu));
229
230 uintptr_t base = (uintptr_t)mapped_etms[cpu];
231
232 if (access_size == sizeof(uint32_t)) {
233 // Regards the ETM v4 doc: HW implementation supports 32-bit
234 // accesses to access 32-bit registers or either half of a
235 // 64-bit register.
236 _Atomic uint32_t *reg = (_Atomic uint32_t *)(base + offset);
237
238 *val = atomic_load_relaxed(reg);
239 } else if (access_size == sizeof(uint64_t)) {
240 _Atomic uint64_t *reg = (_Atomic uint64_t *)(base + offset);
241
242 *val = atomic_load_relaxed(reg);
243 } else {
244 panic("ETM: invalid access size");
245 }
246 }
247
248 static void
etm_unlock_percpu(cpu_index_t cpu)249 etm_unlock_percpu(cpu_index_t cpu)
250 {
251 #if ETM_USE_SOFTWARE_LOCK
252 atomic_store_relaxed(&mapped_etms[cpu]->trclar, ETM_TRCLAR_UNLOCK);
253 CTX_WRITE_WORKAROUND;
254 #else
255 (void)cpu;
256 #endif
257 }
258
259 #if ETM_USE_SOFTWARE_LOCK
260 static void
etm_lock_percpu(cpu_index_t cpu)261 etm_lock_percpu(cpu_index_t cpu)
262 {
263 atomic_store_relaxed(&mapped_etms[cpu]->trclar, ETM_TRCLAR_LOCK);
264 CTX_WRITE_WORKAROUND;
265 }
266 #endif
267
268 static void
etm_os_unlock_percpu(cpu_index_t cpu)269 etm_os_unlock_percpu(cpu_index_t cpu)
270 {
271 atomic_store_relaxed(&mapped_etms[cpu]->trcoslar, ETM_TRCOSLAR_UNLOCK);
272
273 // Note: no write delay workaround for this register, to avoid delaying
274 // resume when the ETM is not being used. It is always written last
275 // in the sequence anyway, so a delay after it is useless.
276 }
277
278 static void
etm_os_lock_percpu(cpu_index_t cpu)279 etm_os_lock_percpu(cpu_index_t cpu)
280 {
281 atomic_store_relaxed(&mapped_etms[cpu]->trcoslar, ETM_TRCOSLAR_LOCK);
282
283 // Note: no write delay workaround for this register, to avoid delaying
284 // suspend when the ETM is not being used. The suspend sequence should
285 // start with a conditional CTX_WRITE_WORKAROUND as a substitute.
286 }
287
288 static index_t
etm_save_context_registers(cpu_index_t cpu,const context_register_info_t * info,index_t context_register_index)289 etm_save_context_registers(cpu_index_t cpu, const context_register_info_t *info,
290 index_t context_register_index)
291 {
292 register_t *context = etm_contexts[cpu];
293
294 index_t cur_register_index = context_register_index;
295
296 for (index_t i = 0; i < info->count; i++, cur_register_index++) {
297 size_t reg_offset = info->reg_offset + i * info->stride;
298
299 register_t *reg = context + cur_register_index;
300
301 etm_get_reg(cpu, reg_offset, reg, info->access_size);
302 }
303
304 return cur_register_index;
305 }
306
307 void
etm_save_context_percpu(cpu_index_t cpu)308 etm_save_context_percpu(cpu_index_t cpu)
309 {
310 // dsb isb
311 __asm__ volatile("dsb ish; isb" ::: "memory");
312
313 // Delay after taking the OS lock in the caller
314 CTX_WRITE_WORKAROUND;
315
316 // pull trcstatr.pmstable until it's stable
317 // wait up to 100us
318 ticks_t start = platform_timer_get_current_ticks();
319 ticks_t timeout =
320 start + platform_timer_convert_ns_to_ticks(100U * 1000U);
321 do {
322 ETM_TRCSTATR_t trcstatr =
323 atomic_load_relaxed(&mapped_etms[cpu]->trcstatr);
324 if (ETM_TRCSTATR_get_pmstable(&trcstatr)) {
325 break;
326 }
327
328 if (platform_timer_get_current_ticks() > timeout) {
329 TRACE_AND_LOG(ERROR, INFO,
330 "ETM: programmers model is not stable");
331 break;
332 }
333 } while (1);
334
335 etm_cprgctlr[cpu] = atomic_load_relaxed(&mapped_etms[cpu]->trcprgctlr);
336 if ((etm_cprgctlr[cpu] & ETM_TRCPRGCTLR_ENABLE) != 0U) {
337 // save all context registers
338 index_t context_register_index = 0U;
339 for (index_t i = 0; i < util_array_size(context_register_list);
340 i++) {
341 const context_register_info_t *info =
342 &context_register_list[i];
343
344 context_register_index = etm_save_context_registers(
345 cpu, info, context_register_index);
346 }
347
348 etm_claim_tag[cpu] =
349 atomic_load_relaxed(&mapped_etms[cpu]->trcclaimclr);
350
351 // poll until trcstatr.idle
352 count_t idle_count = 100U;
353 bool idle = false;
354 while (idle_count > 0U) {
355 // should be a volatile read
356 ETM_TRCSTATR_t trcstatr = atomic_load_relaxed(
357 &mapped_etms[cpu]->trcstatr);
358 idle = ETM_TRCSTATR_get_idle(&trcstatr);
359
360 if (idle) {
361 break;
362 }
363
364 idle_count--;
365 platform_timer_ndelay(1000);
366 }
367
368 if ((!idle) && (idle_count == 0)) {
369 LOG(ERROR, WARN,
370 "ETM: waiting idle timeout for context save");
371 }
372 }
373 return;
374 }
375
376 static index_t
etm_restore_context_registers(cpu_index_t cpu,const context_register_info_t * info,index_t context_register_index)377 etm_restore_context_registers(cpu_index_t cpu,
378 const context_register_info_t *info,
379 index_t context_register_index)
380 {
381 register_t *context = etm_contexts[cpu];
382
383 index_t cur_register_index = context_register_index;
384
385 for (index_t i = 0; i < info->count; i++, cur_register_index++) {
386 size_t reg_offset = info->reg_offset + i * info->stride;
387
388 register_t *reg = context + cur_register_index;
389
390 etm_set_reg(cpu, reg_offset, *reg, info->access_size);
391 CTX_WRITE_WORKAROUND;
392 }
393
394 return cur_register_index;
395 }
396
397 void
etm_restore_context_percpu(cpu_index_t cpu)398 etm_restore_context_percpu(cpu_index_t cpu)
399 {
400 if (((etm_cprgctlr[cpu]) & ETM_TRCPRGCTLR_ENABLE) != 0U) {
401 // restore all context registers
402 index_t context_register_index = 0U;
403 for (index_t i = 0; i < util_array_size(context_register_list);
404 i++) {
405 const context_register_info_t *info =
406 &context_register_list[i];
407
408 context_register_index = etm_restore_context_registers(
409 cpu, info, context_register_index);
410 }
411
412 // set claim tag
413 atomic_store_relaxed(&mapped_etms[cpu]->trcclaimset,
414 etm_claim_tag[cpu]);
415 CTX_WRITE_WORKAROUND;
416
417 atomic_store_relaxed(&mapped_etms[cpu]->trcprgctlr,
418 ETM_TRCPRGCTLR_ENABLE);
419 }
420
421 return;
422 }
423
424 void
etm_handle_power_cpu_online(void)425 etm_handle_power_cpu_online(void)
426 {
427 if (compiler_unexpected(!platform_security_state_debug_disabled())) {
428 cpu_index_t cpu = cpulocal_get_index();
429 etm_unlock_percpu(cpu);
430 etm_os_unlock_percpu(cpu);
431 }
432 }
433
434 void
etm_handle_power_cpu_offline(void)435 etm_handle_power_cpu_offline(void)
436 {
437 (void)etm_handle_power_cpu_suspend(true);
438 }
439
440 error_t
etm_handle_power_cpu_suspend(bool may_poweroff)441 etm_handle_power_cpu_suspend(bool may_poweroff)
442 {
443 if (may_poweroff &&
444 compiler_unexpected(!platform_security_state_debug_disabled())) {
445 cpu_index_t cpu = cpulocal_get_index();
446
447 etm_unlock_percpu(cpu);
448 etm_os_lock_percpu(cpu);
449
450 etm_save_context_percpu(cpu);
451 }
452
453 return OK;
454 }
455
456 void
etm_unwind_power_cpu_suspend(bool may_poweroff)457 etm_unwind_power_cpu_suspend(bool may_poweroff)
458 {
459 if (may_poweroff &&
460 compiler_unexpected(!platform_security_state_debug_disabled())) {
461 cpu_index_t cpu = cpulocal_get_index();
462 etm_os_unlock_percpu(cpu);
463
464 #if ETM_USE_SOFTWARE_LOCK
465 #error Restore software lock from before suspend (don't lock unconditionally)
466 #endif
467 }
468 }
469
470 void
471 etm_handle_power_cpu_resume(bool was_poweroff)
472 {
473 if (compiler_expected(platform_security_state_debug_disabled())) {
474 goto out;
475 }
476
477 cpu_index_t cpu = cpulocal_get_index();
478
479 if (was_poweroff) {
480 etm_unlock_percpu(cpu);
481
482 // check lock os los with trcoslsr.oslk == 1
483 ETM_TRCOSLSR_t trcoslsr =
484 atomic_load_relaxed(&mapped_etms[cpu]->trcoslsr);
485 if (!ETM_TRCOSLSR_get_oslk(&trcoslsr)) {
486 LOG(ERROR, WARN, "etm: os is not locked");
487 etm_os_lock_percpu(cpu);
488 }
489
490 etm_restore_context_percpu(cpu);
491 }
492
493 etm_os_unlock_percpu(cpu);
494
495 #if ETM_USE_SOFTWARE_LOCK
496 #error Restore software lock from before suspend (don't lock unconditionally)
497 #endif
498
499 out:
500 return;
501 }
502