1 // SPDX-License-Identifier: BSD-2-Clause
2 /*
3 * Copyright (C) Foundries Ltd. 2022.
4 * Author: Jorge Ramirez <jorge@foundries.io>
5 */
6
7 #include <assert.h>
8 #include <crypto/crypto.h>
9 #include <crypto/crypto_impl.h>
10 #include <crypto/internal_aes-gcm.h>
11 #include <drvcrypt.h>
12 #include <drvcrypt_authenc.h>
13 #include <initcall.h>
14 #include <ipi.h>
15 #include <kernel/boot.h>
16 #include <kernel/dt.h>
17 #include <kernel/panic.h>
18 #include <kernel/spinlock.h>
19 #include <libfdt.h>
20 #include <mm/core_memprot.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <string_ext.h>
24 #include <tee_api_types.h>
25 #include <tee/cache.h>
26 #include <utee_defines.h>
27 #include <util.h>
28
29 /*
30 * This driver does not queue/pad non-aligned data.
31 *
32 * Allow debug information for future PLM work: if the PLM can not implement
33 * the required changes, we might be able to do it in OP-TEE.
34 */
35 #define DEBUG_VERSAL_AES 0
36
37 #define GCM_TAG_LEN 16
38
39 #define XSECURE_AES_KEY_SIZE_128 0 /* Key Length = 32 bytes = 256 bits */
40 #define XSECURE_AES_KEY_SIZE_256 2 /* Key Length = 16 bytes = 128 bits */
41
42 #define XSECURE_ENCRYPT 0
43 #define XSECURE_DECRYPT 1
44
45 enum versal_aes_err {
46 AES_GCM_TAG_MISMATCH = 0x40,
47 AES_KEY_CLEAR_ERROR,
48 AES_DPA_CM_NOT_SUPPORTED,
49 AES_KAT_WRITE_KEY_FAILED_ERROR,
50 AES_KAT_DECRYPT_INIT_FAILED_ERROR,
51 AES_KAT_GCM_TAG_MISMATCH_ERROR,
52 AES_KAT_DATA_MISMATCH_ERROR,
53 AES_KAT_FAILED_ERROR,
54 AESDPACM_KAT_WRITE_KEY_FAILED_ERROR,
55 AESDPACM_KAT_KEYLOAD_FAILED_ERROR,
56 AESDPACM_SSS_CFG_FAILED_ERROR,
57 AESDPACM_KAT_FAILED_ERROR,
58 AESDPACM_KAT_CHECK1_FAILED_ERROR,
59 AESDPACM_KAT_CHECK2_FAILED_ERROR,
60 AESDPACM_KAT_CHECK3_FAILED_ERROR,
61 AESDPACM_KAT_CHECK4_FAILED_ERROR,
62 AESDPACM_KAT_CHECK5_FAILED_ERROR,
63 AES_INVALID_PARAM,
64 AESKAT_INVALID_PARAM,
65 AES_STATE_MISMATCH_ERROR,
66 AES_DEVICE_KEY_NOT_ALLOWED,
67 };
68
69 #define VERSAL_AES_ERROR(m) { . error = (m), .name = TO_STR(m) }
70
versal_aes_error(uint8_t err)71 static const char *versal_aes_error(uint8_t err)
72 {
73 const struct {
74 enum versal_aes_err error;
75 const char *name;
76 } elist[] = {
77 VERSAL_AES_ERROR(AES_GCM_TAG_MISMATCH),
78 VERSAL_AES_ERROR(AES_KEY_CLEAR_ERROR),
79 VERSAL_AES_ERROR(AES_DPA_CM_NOT_SUPPORTED),
80 VERSAL_AES_ERROR(AES_KAT_WRITE_KEY_FAILED_ERROR),
81 VERSAL_AES_ERROR(AES_KAT_DECRYPT_INIT_FAILED_ERROR),
82 VERSAL_AES_ERROR(AES_KAT_GCM_TAG_MISMATCH_ERROR),
83 VERSAL_AES_ERROR(AES_KAT_DATA_MISMATCH_ERROR),
84 VERSAL_AES_ERROR(AES_KAT_FAILED_ERROR),
85 VERSAL_AES_ERROR(AESDPACM_KAT_WRITE_KEY_FAILED_ERROR),
86 VERSAL_AES_ERROR(AESDPACM_KAT_KEYLOAD_FAILED_ERROR),
87 VERSAL_AES_ERROR(AESDPACM_SSS_CFG_FAILED_ERROR),
88 VERSAL_AES_ERROR(AESDPACM_KAT_FAILED_ERROR),
89 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK1_FAILED_ERROR),
90 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK2_FAILED_ERROR),
91 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK3_FAILED_ERROR),
92 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK4_FAILED_ERROR),
93 VERSAL_AES_ERROR(AESDPACM_KAT_CHECK5_FAILED_ERROR),
94 VERSAL_AES_ERROR(AES_INVALID_PARAM),
95 VERSAL_AES_ERROR(AESKAT_INVALID_PARAM),
96 VERSAL_AES_ERROR(AES_STATE_MISMATCH_ERROR),
97 VERSAL_AES_ERROR(AES_DEVICE_KEY_NOT_ALLOWED),
98 };
99
100 if (err >= AES_GCM_TAG_MISMATCH && err <= AES_DEVICE_KEY_NOT_ALLOWED) {
101 if (elist[err - AES_GCM_TAG_MISMATCH].name)
102 return elist[err - AES_GCM_TAG_MISMATCH].name;
103
104 return "Invalid";
105 }
106
107 return "Unknown";
108 }
109
110 enum aes_key_src {
111 XSECURE_AES_BBRAM_KEY = 0, /* BBRAM Key */
112 XSECURE_AES_BBRAM_RED_KEY, /* BBRAM Red Key */
113 XSECURE_AES_BH_KEY, /* BH Key */
114 XSECURE_AES_BH_RED_KEY, /* BH Red Key */
115 XSECURE_AES_EFUSE_KEY, /* eFUSE Key */
116 XSECURE_AES_EFUSE_RED_KEY, /* eFUSE Red Key */
117 XSECURE_AES_EFUSE_USER_KEY_0, /* eFUSE User Key 0 */
118 XSECURE_AES_EFUSE_USER_KEY_1, /* eFUSE User Key 1 */
119 XSECURE_AES_EFUSE_USER_RED_KEY_0, /* eFUSE User Red Key 0 */
120 XSECURE_AES_EFUSE_USER_RED_KEY_1, /* eFUSE User Red Key 1 */
121 XSECURE_AES_KUP_KEY, /* KUP key */
122 XSECURE_AES_PUF_KEY, /* PUF key */
123 XSECURE_AES_USER_KEY_0, /* User Key 0 */
124 XSECURE_AES_USER_KEY_1, /* User Key 1 */
125 XSECURE_AES_USER_KEY_2, /* User Key 2 */
126 XSECURE_AES_USER_KEY_3, /* User Key 3 */
127 XSECURE_AES_USER_KEY_4, /* User Key 4 */
128 XSECURE_AES_USER_KEY_5, /* User Key 5 */
129 XSECURE_AES_USER_KEY_6, /* User Key 6 */
130 XSECURE_AES_USER_KEY_7, /* User Key 7 */
131 XSECURE_AES_EXPANDED_KEYS, /* Expanded keys */
132 XSECURE_AES_ALL_KEYS, /* AES All keys */
133 };
134
135 struct versal_payload {
136 struct versal_mbox_mem input_cmd;
137 struct versal_mbox_mem src;
138 struct versal_mbox_mem dst;
139 bool encrypt;
140 };
141
142 struct versal_aad {
143 struct versal_mbox_mem mem;
144 };
145
146 struct versal_node {
147 struct versal_payload payload;
148 struct versal_aad aad;
149 bool is_aad;
150 STAILQ_ENTRY(versal_node) link;
151 };
152
153 struct versal_init {
154 uint32_t key_len;
155 uint32_t operation;
156 struct versal_mbox_mem key;
157 struct versal_mbox_mem nonce;
158 struct versal_mbox_mem init_buf;
159 };
160
161 struct versal_ae_ctx {
162 struct crypto_authenc_ctx a_ctx;
163 };
164
165 struct versal_context_node {
166 struct versal_ae_ctx *ctx;
167 STAILQ_ENTRY(versal_context_node) link;
168 };
169
170 enum engine_state {
171 READY = 1, INIT = 2, FINALIZED = 3,
172 };
173
174 static struct mutex engine_lock = MUTEX_INITIALIZER;
175
176 static struct versal_engine {
177 enum aes_key_src key_src;
178 enum engine_state state;
179 struct versal_init init;
180 struct refcount refc;
181 struct mutex *lock; /* protect the HW instance */
182 STAILQ_HEAD(authenc_context_list, versal_context_node) context_list;
183 STAILQ_HEAD(authenc_replay_list, versal_node) replay_list;
184 } engine = {
185 .key_src = XSECURE_AES_USER_KEY_0,
186 .lock = &engine_lock,
187 };
188
to_versal_ctx(struct crypto_authenc_ctx * ctx)189 static struct versal_ae_ctx *to_versal_ctx(struct crypto_authenc_ctx *ctx)
190 {
191 assert(ctx);
192
193 return container_of(ctx, struct versal_ae_ctx, a_ctx);
194 }
195
replay_init(void)196 static TEE_Result replay_init(void)
197 {
198 struct versal_cmd_args arg = { };
199 uint32_t err = 0;
200
201 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) {
202 EMSG("AES_INIT error");
203 return TEE_ERROR_GENERIC;
204 }
205
206 arg.data[arg.dlen++] = engine.init.key_len;
207 arg.data[arg.dlen++] = engine.key_src;
208 arg.ibuf[0].mem = engine.init.key;
209
210 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) {
211 EMSG("AES_WRITE_KEY error");
212 return TEE_ERROR_GENERIC;
213 }
214
215 memset(&arg, 0, sizeof(arg));
216
217 arg.ibuf[0].mem = engine.init.init_buf;
218 arg.ibuf[1].mem = engine.init.nonce;
219 arg.ibuf[1].only_cache = true;
220
221 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) {
222 EMSG("AES_OP_INIT error");
223 return TEE_ERROR_GENERIC;
224 }
225
226 return TEE_SUCCESS;
227 }
228
replay_aad(struct versal_aad * p)229 static TEE_Result replay_aad(struct versal_aad *p)
230 {
231 struct versal_cmd_args arg = { };
232 uint32_t err = 0;
233
234 arg.data[arg.dlen++] = p->mem.len % 16 ? p->mem.alloc_len : p->mem.len;
235 arg.ibuf[0].mem = p->mem;
236
237 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) {
238 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err));
239 return TEE_ERROR_GENERIC;
240 }
241
242 return TEE_SUCCESS;
243 }
244
replay_payload(struct versal_payload * p)245 static TEE_Result replay_payload(struct versal_payload *p)
246 {
247 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE;
248 struct versal_cmd_args arg = { };
249 uint32_t err = 0;
250
251 arg.ibuf[0].mem = p->input_cmd;
252 arg.ibuf[1].mem = p->dst;
253 arg.ibuf[2].mem = p->src;
254
255 if (p->encrypt)
256 id = VERSAL_AES_ENCRYPT_UPDATE;
257
258 if (versal_crypto_request(id, &arg, &err)) {
259 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err));
260 return TEE_ERROR_GENERIC;
261 }
262
263 return TEE_SUCCESS;
264 }
265
do_replay(void)266 static TEE_Result do_replay(void)
267 {
268 struct versal_node *node = NULL;
269 TEE_Result ret = TEE_SUCCESS;
270
271 ret = replay_init();
272 if (ret)
273 return ret;
274
275 STAILQ_FOREACH(node, &engine.replay_list, link) {
276 if (node->is_aad) {
277 ret = replay_aad(&node->aad);
278 if (ret)
279 return ret;
280 } else {
281 ret = replay_payload(&node->payload);
282 if (ret)
283 return ret;
284 }
285 }
286
287 /* Engine has been init */
288 engine.state = INIT;
289
290 return TEE_SUCCESS;
291 }
292
engine_in_use(void)293 static bool engine_in_use(void)
294 {
295 if (STAILQ_EMPTY(&engine.context_list))
296 return false;
297
298 return true;
299 }
300
context_allowed(struct crypto_authenc_ctx * ctx)301 static bool context_allowed(struct crypto_authenc_ctx *ctx)
302 {
303 struct versal_context_node *node = NULL;
304
305 STAILQ_FOREACH(node, &engine.context_list, link) {
306 if (node->ctx == to_versal_ctx(ctx))
307 return true;
308 }
309
310 return false;
311 }
312
do_init(struct drvcrypt_authenc_init * dinit)313 static TEE_Result do_init(struct drvcrypt_authenc_init *dinit)
314 {
315 uint32_t key_len = XSECURE_AES_KEY_SIZE_128;
316 struct versal_context_node *node = NULL;
317 struct versal_aes_init *init = NULL;
318 struct versal_mbox_mem init_buf = { };
319 struct versal_mbox_mem key = { };
320 struct versal_mbox_mem nonce = { };
321 struct versal_cmd_args arg = { };
322 TEE_Result ret = TEE_SUCCESS;
323 uint32_t err = 0;
324
325 if (engine_in_use()) {
326 EMSG("Versal: AES-GCM Engine busy");
327 return TEE_ERROR_BUSY;
328 }
329
330 if (dinit->key.length != 32 && dinit->key.length != 16)
331 return TEE_ERROR_BAD_PARAMETERS;
332
333 if (dinit->key.length == 32)
334 key_len = XSECURE_AES_KEY_SIZE_256;
335
336 if (engine.state != READY)
337 return TEE_ERROR_BAD_STATE;
338
339 /* Initialize the AES engine */
340 if (versal_crypto_request(VERSAL_AES_INIT, &arg, &err)) {
341 EMSG("AES_INIT error: %s", versal_aes_error(err));
342 return TEE_ERROR_GENERIC;
343 }
344
345 /* Write the key */
346 versal_mbox_alloc(dinit->key.length, dinit->key.data, &key);
347
348 arg.data[arg.dlen++] = key_len;
349 arg.data[arg.dlen++] = engine.key_src;
350 arg.ibuf[0].mem = key;
351
352 if (versal_crypto_request(VERSAL_AES_WRITE_KEY, &arg, &err)) {
353 EMSG("AES_WRITE_KEY error: %s", versal_aes_error(err));
354 ret = TEE_ERROR_GENERIC;
355 goto error;
356 }
357
358 memset(&arg, 0, sizeof(arg));
359
360 /* Send the initialization structure */
361 versal_mbox_alloc(sizeof(*init), NULL, &init_buf);
362 versal_mbox_alloc(dinit->nonce.length, dinit->nonce.data, &nonce);
363
364 init = init_buf.buf;
365 init->iv_addr = virt_to_phys(nonce.buf);
366 init->operation = dinit->encrypt ? XSECURE_ENCRYPT : XSECURE_DECRYPT;
367 init->key_src = engine.key_src;
368 init->key_len = key_len;
369
370 arg.ibuf[0].mem = init_buf;
371 arg.ibuf[1].mem = nonce;
372 arg.ibuf[1].only_cache = true;
373
374 if (versal_crypto_request(VERSAL_AES_OP_INIT, &arg, &err)) {
375 EMSG("AES_OP_INIT error: %s", versal_aes_error(err));
376 ret = TEE_ERROR_GENERIC;
377 goto error;
378 }
379
380 node = calloc(1, sizeof(*node));
381 if (!node) {
382 ret = TEE_ERROR_OUT_OF_MEMORY;
383 goto error;
384 }
385
386 /* Save key context */
387 engine.init.operation = dinit->encrypt ?
388 XSECURE_ENCRYPT : XSECURE_DECRYPT;
389 engine.init.key_len = key_len;
390 engine.init.init_buf = init_buf;
391 engine.init.nonce = nonce;
392 engine.init.key = key;
393
394 /* Active context */
395 node->ctx = to_versal_ctx(dinit->ctx);
396 STAILQ_INSERT_TAIL(&engine.context_list, node, link);
397
398 /* Engine has been init*/
399 engine.state = INIT;
400
401 return TEE_SUCCESS;
402 error:
403 free(key.buf);
404 free(init_buf.buf);
405 free(nonce.buf);
406
407 return ret;
408 }
409
do_update_aad(struct drvcrypt_authenc_update_aad * dupdate)410 static TEE_Result do_update_aad(struct drvcrypt_authenc_update_aad *dupdate)
411 {
412 struct versal_cmd_args arg = { };
413 struct versal_mbox_mem p = { };
414 TEE_Result ret = TEE_SUCCESS;
415 struct versal_node *node = NULL;
416 uint32_t err = 0;
417
418 /* This context has not been inited */
419 if (!context_allowed(dupdate->ctx))
420 return TEE_ERROR_BUSY;
421
422 /* There is a copy of the context: don't allow updates, only finalize */
423 if (refcount_val(&engine.refc) > 1)
424 return TEE_ERROR_BUSY;
425
426 /* There was a copy of the context and it was finalized, then replay */
427 if (engine.state == FINALIZED)
428 do_replay();
429
430 versal_mbox_alloc(dupdate->aad.length, dupdate->aad.data, &p);
431
432 arg.data[arg.dlen++] = p.len % 16 ? p.alloc_len : p.len;
433 arg.ibuf[0].mem = p;
434
435 #if DEBUG_VERSAL_AES
436 IMSG("versal: aad length - requested: %zu, sent to plm: %"PRIu32,
437 dupdate->aad.length, arg.data[0]);
438 #endif
439 if (versal_crypto_request(VERSAL_AES_UPDATE_AAD, &arg, &err)) {
440 EMSG("AES_UPDATE_AAD error: %s", versal_aes_error(err));
441 ret = TEE_ERROR_GENERIC;
442 goto error;
443 }
444
445 node = calloc(1, sizeof(*node));
446 if (!node) {
447 ret = TEE_ERROR_OUT_OF_MEMORY;
448 goto error;
449 }
450
451 /* Save the context */
452 node->aad.mem = p;
453 node->is_aad = true;
454 STAILQ_INSERT_TAIL(&engine.replay_list, node, link);
455
456 return TEE_SUCCESS;
457 error:
458 free(p.buf);
459 return ret;
460 }
461
462 static TEE_Result
update_payload(struct drvcrypt_authenc_update_payload * dupdate,bool is_last)463 update_payload(struct drvcrypt_authenc_update_payload *dupdate, bool is_last)
464 {
465 enum versal_crypto_api id = VERSAL_AES_DECRYPT_UPDATE;
466 struct versal_aes_input_param *input = NULL;
467 struct versal_mbox_mem input_cmd = { };
468 struct versal_mbox_mem p = { };
469 struct versal_mbox_mem q = { };
470 TEE_Result ret = TEE_SUCCESS;
471 struct versal_cmd_args arg = { };
472 struct versal_node *node = NULL;
473 uint32_t err = 0;
474
475 if (!context_allowed(dupdate->ctx))
476 return TEE_ERROR_BUSY;
477
478 if (!dupdate->src.length || dupdate->src.length % 4) {
479 EMSG("Versal AES payload length not word aligned (len = %zu)",
480 dupdate->src.length);
481 return TEE_ERROR_BAD_PARAMETERS;
482 }
483
484 versal_mbox_alloc(dupdate->src.length, dupdate->src.data, &p);
485 versal_mbox_alloc(dupdate->dst.length, NULL, &q);
486 versal_mbox_alloc(sizeof(*input), NULL, &input_cmd);
487
488 input = input_cmd.buf;
489 input->input_addr = virt_to_phys(p.buf);
490 input->input_len = p.len % 16 ? p.alloc_len : p.len;
491 input->is_last = is_last;
492
493 arg.ibuf[0].mem = input_cmd;
494 arg.ibuf[1].mem = q;
495 arg.ibuf[2].mem = p;
496
497 if (dupdate->encrypt)
498 id = VERSAL_AES_ENCRYPT_UPDATE;
499
500 #if DEBUG_VERSAL_AES
501 IMSG("versal: payload length - requested %zu, sent to plm: %"PRIu32,
502 dupdate->src.length, input->input_len);
503 IMSG("versal: destination length - %zu ", dupdate->dst.length);
504 #endif
505 if (versal_crypto_request(id, &arg, &err)) {
506 EMSG("AES_UPDATE_PAYLOAD error: %s", versal_aes_error(err));
507 ret = TEE_ERROR_GENERIC;
508 goto out;
509 }
510
511 if (dupdate->dst.data)
512 memcpy(dupdate->dst.data, q.buf, dupdate->dst.length);
513
514 if (!is_last) {
515 node = calloc(1, sizeof(*node));
516 if (!node) {
517 ret = TEE_ERROR_OUT_OF_MEMORY;
518 goto out;
519 }
520
521 node->is_aad = false;
522 node->payload.dst = q;
523 node->payload.src = p;
524 node->payload.input_cmd = input_cmd;
525 node->payload.encrypt = dupdate->encrypt;
526 STAILQ_INSERT_TAIL(&engine.replay_list, node, link);
527
528 return TEE_SUCCESS;
529 }
530 out:
531 free(p.buf);
532 free(q.buf);
533 free(input_cmd.buf);
534
535 return ret;
536 }
537
do_update_payload(struct drvcrypt_authenc_update_payload * p)538 static TEE_Result do_update_payload(struct drvcrypt_authenc_update_payload *p)
539 {
540 TEE_Result ret = TEE_SUCCESS;
541
542 if (!context_allowed(p->ctx))
543 return TEE_ERROR_BUSY;
544
545 /*
546 * If there is a copy, we don't allow updates until one of the copies
547 * has been deleted
548 */
549 if (refcount_val(&engine.refc) > 1)
550 return TEE_ERROR_BUSY;
551
552 /*
553 * If there was a copy and it was finalized, we need to replay before
554 * we can update; do not clear the list so the state can be copied
555 */
556 if (engine.state == FINALIZED) {
557 ret = do_replay();
558 if (ret)
559 return ret;
560 }
561
562 return update_payload(p, false);
563 }
564
do_enc_final(struct drvcrypt_authenc_final * dfinal)565 static TEE_Result do_enc_final(struct drvcrypt_authenc_final *dfinal)
566 {
567 struct drvcrypt_authenc_update_payload last = { };
568 struct versal_cmd_args arg = { };
569 struct versal_mbox_mem p = { };
570 TEE_Result ret = TEE_SUCCESS;
571 uint32_t err = 0;
572
573 if (!context_allowed(dfinal->ctx))
574 return TEE_ERROR_BUSY;
575
576 if (engine.state == FINALIZED) {
577 DMSG("Operation was already finalized");
578 ret = do_replay();
579 if (ret)
580 return ret;
581 }
582
583 if (engine.state != INIT)
584 panic();
585
586 last.ctx = dfinal->ctx;
587 last.dst = dfinal->dst;
588 last.encrypt = true;
589 last.src = dfinal->src;
590
591 ret = update_payload(&last, true);
592 if (ret)
593 return ret;
594
595 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length);
596
597 versal_mbox_alloc(GCM_TAG_LEN, NULL, &p);
598
599 arg.ibuf[0].mem = p;
600 if (versal_crypto_request(VERSAL_AES_ENCRYPT_FINAL, &arg, &err)) {
601 EMSG("AES_ENCRYPT_FINAL error: %s", versal_aes_error(err));
602 ret = TEE_ERROR_GENERIC;
603 goto out;
604 }
605
606 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN);
607 dfinal->tag.length = GCM_TAG_LEN;
608 out:
609 free(p.buf);
610
611 if (refcount_val(&engine.refc) > 1)
612 engine.state = FINALIZED;
613 else
614 engine.state = READY;
615
616 return ret;
617 }
618
do_dec_final(struct drvcrypt_authenc_final * dfinal)619 static TEE_Result do_dec_final(struct drvcrypt_authenc_final *dfinal)
620 {
621 struct drvcrypt_authenc_update_payload last = { };
622 struct versal_cmd_args arg = { };
623 struct versal_mbox_mem p = { };
624 TEE_Result ret = TEE_SUCCESS;
625 uint32_t err = 0;
626
627 if (!context_allowed(dfinal->ctx))
628 return TEE_ERROR_BUSY;
629
630 if (engine.state == FINALIZED) {
631 DMSG("Operation was already finalized");
632 ret = do_replay();
633 if (ret)
634 return ret;
635 }
636
637 if (engine.state != INIT)
638 panic();
639
640 last.encrypt = false;
641 last.ctx = dfinal->ctx;
642 last.dst = dfinal->dst;
643 last.src = dfinal->src;
644
645 ret = update_payload(&last, true);
646 if (ret)
647 return ret;
648
649 versal_mbox_alloc(dfinal->tag.length, dfinal->tag.data, &p);
650 arg.ibuf[0].mem = p;
651
652 if (versal_crypto_request(VERSAL_AES_DECRYPT_FINAL, &arg, &err)) {
653 EMSG("AES_DECRYPT_FINAL error: %s", versal_aes_error(err));
654 ret = TEE_ERROR_GENERIC;
655 goto out;
656 }
657
658 memcpy(dfinal->dst.data, last.dst.data, dfinal->dst.length);
659 memcpy(dfinal->tag.data, p.buf, GCM_TAG_LEN);
660 dfinal->tag.length = GCM_TAG_LEN;
661 out:
662 free(p.buf);
663
664 if (refcount_val(&engine.refc) > 1)
665 engine.state = FINALIZED;
666 else
667 engine.state = READY;
668
669 return ret;
670 }
671
do_final(void * ctx __unused)672 static void do_final(void *ctx __unused)
673 {
674 }
675
do_free(void * ctx)676 static void do_free(void *ctx)
677 {
678 struct versal_ae_ctx *c = to_versal_ctx(ctx);
679 struct versal_node *next = NULL;
680 struct versal_node *node = NULL;
681 struct versal_context_node *ctx_next = NULL;
682 struct versal_context_node *ctx_node = NULL;
683 bool release = false;
684
685 if (refcount_dec(&engine.refc)) {
686 /* this is a final release */
687 release = true;
688 refcount_set(&engine.refc, 1);
689 engine.state = READY;
690 free(engine.init.init_buf.buf);
691 free(engine.init.nonce.buf);
692 free(engine.init.key.buf);
693 memset(&engine.init, 0, sizeof(engine.init));
694 STAILQ_FOREACH_SAFE(node, &engine.replay_list, link, next) {
695 STAILQ_REMOVE(&engine.replay_list, node,
696 versal_node, link);
697 if (node->is_aad) {
698 free(node->aad.mem.buf);
699 } else {
700 free(node->payload.dst.buf);
701 free(node->payload.src.buf);
702 free(node->payload.input_cmd.buf);
703 }
704 free(node);
705 }
706 }
707
708 STAILQ_FOREACH_SAFE(ctx_node, &engine.context_list, link, ctx_next) {
709 if (c == ctx_node->ctx) {
710 STAILQ_REMOVE(&engine.context_list, ctx_node,
711 versal_context_node, link);
712 free(ctx_node);
713 }
714 }
715
716 if (release && engine_in_use())
717 panic();
718
719 free(c);
720 }
721
do_copy_state(void * dst_ctx,void * src_ctx)722 static void do_copy_state(void *dst_ctx, void *src_ctx)
723 {
724 struct versal_context_node *node = NULL;
725
726 STAILQ_FOREACH(node, &engine.context_list, link) {
727 if (node->ctx != to_versal_ctx(src_ctx))
728 continue;
729 /*
730 * The running context has been copied: from now on we can only
731 * finalize any of the contexts until there is only one active
732 * again.
733 */
734 node = calloc(1, sizeof(*node));
735 if (!node)
736 panic();
737
738 node->ctx = to_versal_ctx(dst_ctx);
739 STAILQ_INSERT_TAIL(&engine.context_list, node, link);
740
741 /* number of active contexts */
742 refcount_inc(&engine.refc);
743
744 return;
745 }
746
747 panic();
748 }
749
do_allocate(void ** ctx,uint32_t algo)750 static TEE_Result do_allocate(void **ctx, uint32_t algo)
751 {
752 struct versal_ae_ctx *c = NULL;
753
754 if (algo != TEE_ALG_AES_GCM)
755 return TEE_ERROR_NOT_IMPLEMENTED;
756
757 c = calloc(1, sizeof(*c));
758 if (!c)
759 return TEE_ERROR_OUT_OF_MEMORY;
760
761 *ctx = &c->a_ctx;
762
763 return TEE_SUCCESS;
764 }
765
766 static TEE_Result
do_update_payload_locked(struct drvcrypt_authenc_update_payload * p)767 do_update_payload_locked(struct drvcrypt_authenc_update_payload *p)
768 {
769 TEE_Result ret = TEE_SUCCESS;
770
771 mutex_lock(engine.lock);
772 ret = do_update_payload(p);
773 mutex_unlock(engine.lock);
774 return ret;
775 }
776
777 static TEE_Result
do_update_aad_locked(struct drvcrypt_authenc_update_aad * p)778 do_update_aad_locked(struct drvcrypt_authenc_update_aad *p)
779 {
780 TEE_Result ret = TEE_SUCCESS;
781
782 mutex_lock(engine.lock);
783 ret = do_update_aad(p);
784 mutex_unlock(engine.lock);
785 return ret;
786 }
787
do_copy_state_locked(void * dst,void * src)788 static void do_copy_state_locked(void *dst, void *src)
789 {
790 mutex_lock(engine.lock);
791 do_copy_state(dst, src);
792 mutex_unlock(engine.lock);
793 }
794
do_enc_final_locked(struct drvcrypt_authenc_final * p)795 static TEE_Result do_enc_final_locked(struct drvcrypt_authenc_final *p)
796 {
797 TEE_Result ret = TEE_SUCCESS;
798
799 mutex_lock(engine.lock);
800 ret = do_enc_final(p);
801 mutex_unlock(engine.lock);
802 return ret;
803 }
804
do_dec_final_locked(struct drvcrypt_authenc_final * p)805 static TEE_Result do_dec_final_locked(struct drvcrypt_authenc_final *p)
806 {
807 TEE_Result ret = TEE_SUCCESS;
808
809 mutex_lock(engine.lock);
810 ret = do_dec_final(p);
811 mutex_unlock(engine.lock);
812 return ret;
813 }
814
do_free_locked(void * p)815 static void do_free_locked(void *p)
816 {
817 mutex_lock(engine.lock);
818 do_free(p);
819 mutex_unlock(engine.lock);
820 }
821
do_init_locked(struct drvcrypt_authenc_init * p)822 static TEE_Result do_init_locked(struct drvcrypt_authenc_init *p)
823 {
824 TEE_Result ret = TEE_SUCCESS;
825
826 mutex_lock(engine.lock);
827 ret = do_init(p);
828 mutex_unlock(engine.lock);
829 return ret;
830 }
831
832 static struct drvcrypt_authenc versal_authenc = {
833 .update_payload = do_update_payload_locked,
834 .update_aad = do_update_aad_locked,
835 .copy_state = do_copy_state_locked,
836 .enc_final = do_enc_final_locked,
837 .dec_final = do_dec_final_locked,
838 .free_ctx = do_free_locked,
839 .alloc_ctx = do_allocate,
840 .init = do_init_locked,
841 .final = do_final,
842 };
843
enable_secure_status(void)844 static TEE_Result enable_secure_status(void)
845 {
846 /* Once Linux has support, we need to reserve the device */
847 return TEE_SUCCESS;
848 }
849
versal_register_authenc(void)850 static TEE_Result versal_register_authenc(void)
851 {
852 TEE_Result ret = TEE_SUCCESS;
853
854 ret = drvcrypt_register_authenc(&versal_authenc);
855 if (ret)
856 return ret;
857
858 if (engine.key_src < XSECURE_AES_USER_KEY_0 ||
859 engine.key_src > XSECURE_AES_USER_KEY_7)
860 return TEE_ERROR_GENERIC;
861
862 engine.state = READY;
863 STAILQ_INIT(&engine.replay_list);
864 STAILQ_INIT(&engine.context_list);
865 refcount_set(&engine.refc, 1);
866
867 return enable_secure_status();
868 }
869
870 driver_init_late(versal_register_authenc);
871