1 // © 2021 Qualcomm Innovation Center, Inc. All rights reserved.
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4
5 #include <assert.h>
6 #include <hyptypes.h>
7
8 #include <hypregisters.h>
9
10 #include <compiler.h>
11 #include <cpulocal.h>
12 #include <preempt.h>
13 #include <scheduler.h>
14 #include <thread.h>
15 #include <vet.h>
16
17 #include <asm/barrier.h>
18
19 #include "event_handlers.h"
20 #include "trbe.h"
21
22 void
vtrbe_handle_boot_cpu_cold_init(void)23 vtrbe_handle_boot_cpu_cold_init(void)
24 {
25 ID_AA64DFR0_EL1_t id_aa64dfr0 = register_ID_AA64DFR0_EL1_read();
26 // NOTE: ID_AA64DFR0.TraceBuffer just indicates if trace buffer is
27 // implemented, so here we use equal for assertion.
28 assert(ID_AA64DFR0_EL1_get_TraceBuffer(&id_aa64dfr0) == 1U);
29 }
30
31 error_t
vtrbe_handle_object_create_thread(thread_create_t thread_create)32 vtrbe_handle_object_create_thread(thread_create_t thread_create)
33 {
34 thread_t *thread = thread_create.thread;
35
36 // MDCR_EL2.E2TB == 0b10 to prohibit trace EL2
37 MDCR_EL2_set_E2TB(&thread->vcpu_regs_el2.mdcr_el2, 0x2);
38
39 return OK;
40 }
41
42 void
vet_update_trace_buffer_status(thread_t * self)43 vet_update_trace_buffer_status(thread_t *self)
44 {
45 assert(self != NULL);
46
47 #if !DISABLE_TRBE
48 // check/set by reading TBRLIMITR.EN == 1
49 TRBLIMITR_EL1_t trb_limitr =
50 register_TRBLIMITR_EL1_read_ordered(&vet_ordering);
51 self->vet_trace_buffer_enabled = TRBLIMITR_EL1_get_E(&trb_limitr);
52 #endif
53 }
54
55 void
vet_flush_buffer(thread_t * self)56 vet_flush_buffer(thread_t *self)
57 {
58 assert(self != NULL);
59
60 if (compiler_unexpected(self->vet_trace_buffer_enabled)) {
61 __asm__ volatile("tsb csync" : "+m"(vet_ordering));
62 }
63 }
64
65 void
vet_disable_buffer(void)66 vet_disable_buffer(void)
67 {
68 TRBLIMITR_EL1_t trb_limitr =
69 register_TRBLIMITR_EL1_read_ordered(&vet_ordering);
70 TRBLIMITR_EL1_set_E(&trb_limitr, false);
71 register_TRBLIMITR_EL1_write_ordered(trb_limitr, &vet_ordering);
72 }
73
74 static void
vtrbe_prohibit_registers_access(thread_t * self,bool prohibit)75 vtrbe_prohibit_registers_access(thread_t *self, bool prohibit)
76 {
77 assert(self != NULL);
78
79 // MDCR_EL2.E2TB == 0b11 to enable access to TRBE
80 // MDCR_EL2.E2TB == 0b10 to disable access to TRBE
81 uint8_t expect = prohibit ? 0x2U : 0x3U;
82
83 MDCR_EL2_set_E2TB(&self->vcpu_regs_el2.mdcr_el2, expect);
84 register_MDCR_EL2_write_ordered(self->vcpu_regs_el2.mdcr_el2,
85 &vet_ordering);
86 }
87
88 void
vet_save_buffer_thread_context(thread_t * self)89 vet_save_buffer_thread_context(thread_t *self)
90 {
91 (void)self;
92 vtrbe_prohibit_registers_access(self, true);
93 }
94
95 void
vet_restore_buffer_thread_context(thread_t * self)96 vet_restore_buffer_thread_context(thread_t *self)
97 {
98 vtrbe_prohibit_registers_access(self, false);
99 }
100
101 void
vet_enable_buffer(void)102 vet_enable_buffer(void)
103 {
104 TRBLIMITR_EL1_t trb_limitr =
105 register_TRBLIMITR_EL1_read_ordered(&vet_ordering);
106 TRBLIMITR_EL1_set_E(&trb_limitr, true);
107 register_TRBLIMITR_EL1_write_ordered(trb_limitr, &vet_ordering);
108 }
109
110 void
vet_save_buffer_power_context(void)111 vet_save_buffer_power_context(void)
112 {
113 MDCR_EL2_t mdcr_el2;
114
115 // Enable E2TB access
116 mdcr_el2 = register_MDCR_EL2_read_ordered(&vet_ordering);
117 MDCR_EL2_set_E2TB(&mdcr_el2, 3);
118 register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering);
119
120 asm_context_sync_ordered(&vet_ordering);
121
122 trbe_save_context_percpu(cpulocal_get_index());
123
124 // Disable E2TB access
125 MDCR_EL2_set_E2TB(&mdcr_el2, 2);
126 register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering);
127 }
128
129 void
vet_restore_buffer_power_context(void)130 vet_restore_buffer_power_context(void)
131 {
132 MDCR_EL2_t mdcr_el2;
133
134 // Enable E2TB access
135 mdcr_el2 = register_MDCR_EL2_read_ordered(&vet_ordering);
136 MDCR_EL2_set_E2TB(&mdcr_el2, 3);
137 register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering);
138
139 asm_context_sync_ordered(&vet_ordering);
140
141 trbe_restore_context_percpu(cpulocal_get_index());
142
143 // Disable E2TB access
144 MDCR_EL2_set_E2TB(&mdcr_el2, 2);
145 register_MDCR_EL2_write_ordered(mdcr_el2, &vet_ordering);
146 }
147
148 vcpu_trap_result_t
vtrbe_handle_vcpu_trap_sysreg(ESR_EL2_ISS_MSR_MRS_t iss)149 vtrbe_handle_vcpu_trap_sysreg(ESR_EL2_ISS_MSR_MRS_t iss)
150 {
151 vcpu_trap_result_t ret;
152
153 #if DISABLE_TRBE
154 (void)iss;
155
156 ret = VCPU_TRAP_RESULT_UNHANDLED;
157 #else
158 thread_t *current = thread_get_self();
159
160 if (compiler_expected((ESR_EL2_ISS_MSR_MRS_get_Op0(&iss) != 3U) ||
161 (ESR_EL2_ISS_MSR_MRS_get_Op1(&iss) != 0U) ||
162 (ESR_EL2_ISS_MSR_MRS_get_CRn(&iss) != 9U) ||
163 (ESR_EL2_ISS_MSR_MRS_get_CRm(&iss) != 11U))) {
164 // Not a TRBE register access.
165 ret = VCPU_TRAP_RESULT_UNHANDLED;
166 } else if (!vcpu_option_flags_get_trace_allowed(
167 &thread->vcpu_options)) {
168 // This VCPU isn't allowed to trace. Fault immediately.
169 ret = VCPU_TRAP_RESULT_FAULT;
170 } else if (!current->vet_trace_buffer_enabled) {
171 // Lazily enable trace buffer register access and restore
172 // context.
173 current->vet_trace_buffer_enabled = true;
174
175 // only enable the register access
176 vtrbe_prohibit_registers_access(false);
177
178 ret = VCPU_TRAP_RESULT_RETRY;
179 } else {
180 // Probably an attempted OS lock; fall back to default RAZ/WI.
181 ret = VCPU_TRAP_RESULT_UNHANDLED;
182 }
183 #endif
184
185 return ret;
186 }
187