1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Glue code for DES encryption optimized for sparc64 crypto opcodes.
3 *
4 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
5 */
6
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9 #include <linux/crypto.h>
10 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/mm.h>
13 #include <linux/types.h>
14 #include <crypto/algapi.h>
15 #include <crypto/internal/des.h>
16 #include <crypto/internal/skcipher.h>
17
18 #include <asm/fpumacro.h>
19 #include <asm/opcodes.h>
20 #include <asm/pstate.h>
21 #include <asm/elf.h>
22
23 struct des_sparc64_ctx {
24 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
25 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
26 };
27
28 struct des3_ede_sparc64_ctx {
29 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
30 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
31 };
32
encrypt_to_decrypt(u64 * d,const u64 * e)33 static void encrypt_to_decrypt(u64 *d, const u64 *e)
34 {
35 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
36 int i;
37
38 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
39 *d++ = *s--;
40 }
41
42 extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
43
des_set_key(struct crypto_tfm * tfm,const u8 * key,unsigned int keylen)44 static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
45 unsigned int keylen)
46 {
47 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
48 int err;
49
50 /* Even though we have special instructions for key expansion,
51 * we call des_verify_key() so that we don't have to write our own
52 * weak key detection code.
53 */
54 err = crypto_des_verify_key(tfm, key);
55 if (err)
56 return err;
57
58 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
59 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
60
61 return 0;
62 }
63
des_set_key_skcipher(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)64 static int des_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
65 unsigned int keylen)
66 {
67 return des_set_key(crypto_skcipher_tfm(tfm), key, keylen);
68 }
69
70 extern void des_sparc64_crypt(const u64 *key, const u64 *input,
71 u64 *output);
72
sparc_des_encrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)73 static void sparc_des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
74 {
75 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
76 const u64 *K = ctx->encrypt_expkey;
77
78 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
79 }
80
sparc_des_decrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)81 static void sparc_des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
82 {
83 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
84 const u64 *K = ctx->decrypt_expkey;
85
86 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
87 }
88
89 extern void des_sparc64_load_keys(const u64 *key);
90
91 extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
92 unsigned int len);
93
__ecb_crypt(struct skcipher_request * req,bool encrypt)94 static int __ecb_crypt(struct skcipher_request *req, bool encrypt)
95 {
96 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
97 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
98 struct skcipher_walk walk;
99 unsigned int nbytes;
100 int err;
101
102 err = skcipher_walk_virt(&walk, req, true);
103 if (err)
104 return err;
105
106 if (encrypt)
107 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
108 else
109 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
110 while ((nbytes = walk.nbytes) != 0) {
111 des_sparc64_ecb_crypt(walk.src.virt.addr, walk.dst.virt.addr,
112 round_down(nbytes, DES_BLOCK_SIZE));
113 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
114 }
115 fprs_write(0);
116 return err;
117 }
118
ecb_encrypt(struct skcipher_request * req)119 static int ecb_encrypt(struct skcipher_request *req)
120 {
121 return __ecb_crypt(req, true);
122 }
123
ecb_decrypt(struct skcipher_request * req)124 static int ecb_decrypt(struct skcipher_request *req)
125 {
126 return __ecb_crypt(req, false);
127 }
128
129 extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
130 unsigned int len, u64 *iv);
131
132 extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
133 unsigned int len, u64 *iv);
134
__cbc_crypt(struct skcipher_request * req,bool encrypt)135 static int __cbc_crypt(struct skcipher_request *req, bool encrypt)
136 {
137 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
138 const struct des_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
139 struct skcipher_walk walk;
140 unsigned int nbytes;
141 int err;
142
143 err = skcipher_walk_virt(&walk, req, true);
144 if (err)
145 return err;
146
147 if (encrypt)
148 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
149 else
150 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
151 while ((nbytes = walk.nbytes) != 0) {
152 if (encrypt)
153 des_sparc64_cbc_encrypt(walk.src.virt.addr,
154 walk.dst.virt.addr,
155 round_down(nbytes,
156 DES_BLOCK_SIZE),
157 walk.iv);
158 else
159 des_sparc64_cbc_decrypt(walk.src.virt.addr,
160 walk.dst.virt.addr,
161 round_down(nbytes,
162 DES_BLOCK_SIZE),
163 walk.iv);
164 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
165 }
166 fprs_write(0);
167 return err;
168 }
169
cbc_encrypt(struct skcipher_request * req)170 static int cbc_encrypt(struct skcipher_request *req)
171 {
172 return __cbc_crypt(req, true);
173 }
174
cbc_decrypt(struct skcipher_request * req)175 static int cbc_decrypt(struct skcipher_request *req)
176 {
177 return __cbc_crypt(req, false);
178 }
179
des3_ede_set_key(struct crypto_tfm * tfm,const u8 * key,unsigned int keylen)180 static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
181 unsigned int keylen)
182 {
183 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
184 u64 k1[DES_EXPKEY_WORDS / 2];
185 u64 k2[DES_EXPKEY_WORDS / 2];
186 u64 k3[DES_EXPKEY_WORDS / 2];
187 int err;
188
189 err = crypto_des3_ede_verify_key(tfm, key);
190 if (err)
191 return err;
192
193 des_sparc64_key_expand((const u32 *)key, k1);
194 key += DES_KEY_SIZE;
195 des_sparc64_key_expand((const u32 *)key, k2);
196 key += DES_KEY_SIZE;
197 des_sparc64_key_expand((const u32 *)key, k3);
198
199 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
200 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
201 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
202 &k3[0], sizeof(k3));
203
204 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
205 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
206 &k2[0], sizeof(k2));
207 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
208 &k1[0]);
209
210 return 0;
211 }
212
des3_ede_set_key_skcipher(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)213 static int des3_ede_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *key,
214 unsigned int keylen)
215 {
216 return des3_ede_set_key(crypto_skcipher_tfm(tfm), key, keylen);
217 }
218
219 extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
220 u64 *output);
221
sparc_des3_ede_encrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)222 static void sparc_des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
223 {
224 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
225 const u64 *K = ctx->encrypt_expkey;
226
227 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
228 }
229
sparc_des3_ede_decrypt(struct crypto_tfm * tfm,u8 * dst,const u8 * src)230 static void sparc_des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
231 {
232 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
233 const u64 *K = ctx->decrypt_expkey;
234
235 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
236 }
237
238 extern void des3_ede_sparc64_load_keys(const u64 *key);
239
240 extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
241 u64 *output, unsigned int len);
242
__ecb3_crypt(struct skcipher_request * req,bool encrypt)243 static int __ecb3_crypt(struct skcipher_request *req, bool encrypt)
244 {
245 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
246 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
247 struct skcipher_walk walk;
248 const u64 *K;
249 unsigned int nbytes;
250 int err;
251
252 err = skcipher_walk_virt(&walk, req, true);
253 if (err)
254 return err;
255
256 if (encrypt)
257 K = &ctx->encrypt_expkey[0];
258 else
259 K = &ctx->decrypt_expkey[0];
260 des3_ede_sparc64_load_keys(K);
261 while ((nbytes = walk.nbytes) != 0) {
262 des3_ede_sparc64_ecb_crypt(K, walk.src.virt.addr,
263 walk.dst.virt.addr,
264 round_down(nbytes, DES_BLOCK_SIZE));
265 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
266 }
267 fprs_write(0);
268 return err;
269 }
270
ecb3_encrypt(struct skcipher_request * req)271 static int ecb3_encrypt(struct skcipher_request *req)
272 {
273 return __ecb3_crypt(req, true);
274 }
275
ecb3_decrypt(struct skcipher_request * req)276 static int ecb3_decrypt(struct skcipher_request *req)
277 {
278 return __ecb3_crypt(req, false);
279 }
280
281 extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
282 u64 *output, unsigned int len,
283 u64 *iv);
284
285 extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
286 u64 *output, unsigned int len,
287 u64 *iv);
288
__cbc3_crypt(struct skcipher_request * req,bool encrypt)289 static int __cbc3_crypt(struct skcipher_request *req, bool encrypt)
290 {
291 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
292 const struct des3_ede_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
293 struct skcipher_walk walk;
294 const u64 *K;
295 unsigned int nbytes;
296 int err;
297
298 err = skcipher_walk_virt(&walk, req, true);
299 if (err)
300 return err;
301
302 if (encrypt)
303 K = &ctx->encrypt_expkey[0];
304 else
305 K = &ctx->decrypt_expkey[0];
306 des3_ede_sparc64_load_keys(K);
307 while ((nbytes = walk.nbytes) != 0) {
308 if (encrypt)
309 des3_ede_sparc64_cbc_encrypt(K, walk.src.virt.addr,
310 walk.dst.virt.addr,
311 round_down(nbytes,
312 DES_BLOCK_SIZE),
313 walk.iv);
314 else
315 des3_ede_sparc64_cbc_decrypt(K, walk.src.virt.addr,
316 walk.dst.virt.addr,
317 round_down(nbytes,
318 DES_BLOCK_SIZE),
319 walk.iv);
320 err = skcipher_walk_done(&walk, nbytes % DES_BLOCK_SIZE);
321 }
322 fprs_write(0);
323 return err;
324 }
325
cbc3_encrypt(struct skcipher_request * req)326 static int cbc3_encrypt(struct skcipher_request *req)
327 {
328 return __cbc3_crypt(req, true);
329 }
330
cbc3_decrypt(struct skcipher_request * req)331 static int cbc3_decrypt(struct skcipher_request *req)
332 {
333 return __cbc3_crypt(req, false);
334 }
335
336 static struct crypto_alg cipher_algs[] = {
337 {
338 .cra_name = "des",
339 .cra_driver_name = "des-sparc64",
340 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
341 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
342 .cra_blocksize = DES_BLOCK_SIZE,
343 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
344 .cra_alignmask = 7,
345 .cra_module = THIS_MODULE,
346 .cra_u = {
347 .cipher = {
348 .cia_min_keysize = DES_KEY_SIZE,
349 .cia_max_keysize = DES_KEY_SIZE,
350 .cia_setkey = des_set_key,
351 .cia_encrypt = sparc_des_encrypt,
352 .cia_decrypt = sparc_des_decrypt
353 }
354 }
355 }, {
356 .cra_name = "des3_ede",
357 .cra_driver_name = "des3_ede-sparc64",
358 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
359 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
360 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
361 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
362 .cra_alignmask = 7,
363 .cra_module = THIS_MODULE,
364 .cra_u = {
365 .cipher = {
366 .cia_min_keysize = DES3_EDE_KEY_SIZE,
367 .cia_max_keysize = DES3_EDE_KEY_SIZE,
368 .cia_setkey = des3_ede_set_key,
369 .cia_encrypt = sparc_des3_ede_encrypt,
370 .cia_decrypt = sparc_des3_ede_decrypt
371 }
372 }
373 }
374 };
375
376 static struct skcipher_alg skcipher_algs[] = {
377 {
378 .base.cra_name = "ecb(des)",
379 .base.cra_driver_name = "ecb-des-sparc64",
380 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
381 .base.cra_blocksize = DES_BLOCK_SIZE,
382 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx),
383 .base.cra_alignmask = 7,
384 .base.cra_module = THIS_MODULE,
385 .min_keysize = DES_KEY_SIZE,
386 .max_keysize = DES_KEY_SIZE,
387 .setkey = des_set_key_skcipher,
388 .encrypt = ecb_encrypt,
389 .decrypt = ecb_decrypt,
390 }, {
391 .base.cra_name = "cbc(des)",
392 .base.cra_driver_name = "cbc-des-sparc64",
393 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
394 .base.cra_blocksize = DES_BLOCK_SIZE,
395 .base.cra_ctxsize = sizeof(struct des_sparc64_ctx),
396 .base.cra_alignmask = 7,
397 .base.cra_module = THIS_MODULE,
398 .min_keysize = DES_KEY_SIZE,
399 .max_keysize = DES_KEY_SIZE,
400 .ivsize = DES_BLOCK_SIZE,
401 .setkey = des_set_key_skcipher,
402 .encrypt = cbc_encrypt,
403 .decrypt = cbc_decrypt,
404 }, {
405 .base.cra_name = "ecb(des3_ede)",
406 .base.cra_driver_name = "ecb-des3_ede-sparc64",
407 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
408 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
409 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
410 .base.cra_alignmask = 7,
411 .base.cra_module = THIS_MODULE,
412 .min_keysize = DES3_EDE_KEY_SIZE,
413 .max_keysize = DES3_EDE_KEY_SIZE,
414 .setkey = des3_ede_set_key_skcipher,
415 .encrypt = ecb3_encrypt,
416 .decrypt = ecb3_decrypt,
417 }, {
418 .base.cra_name = "cbc(des3_ede)",
419 .base.cra_driver_name = "cbc-des3_ede-sparc64",
420 .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
421 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
422 .base.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
423 .base.cra_alignmask = 7,
424 .base.cra_module = THIS_MODULE,
425 .min_keysize = DES3_EDE_KEY_SIZE,
426 .max_keysize = DES3_EDE_KEY_SIZE,
427 .ivsize = DES3_EDE_BLOCK_SIZE,
428 .setkey = des3_ede_set_key_skcipher,
429 .encrypt = cbc3_encrypt,
430 .decrypt = cbc3_decrypt,
431 }
432 };
433
sparc64_has_des_opcode(void)434 static bool __init sparc64_has_des_opcode(void)
435 {
436 unsigned long cfr;
437
438 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
439 return false;
440
441 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
442 if (!(cfr & CFR_DES))
443 return false;
444
445 return true;
446 }
447
des_sparc64_mod_init(void)448 static int __init des_sparc64_mod_init(void)
449 {
450 int err;
451
452 if (!sparc64_has_des_opcode()) {
453 pr_info("sparc64 des opcodes not available.\n");
454 return -ENODEV;
455 }
456 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
457 err = crypto_register_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
458 if (err)
459 return err;
460 err = crypto_register_skciphers(skcipher_algs,
461 ARRAY_SIZE(skcipher_algs));
462 if (err)
463 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
464 return err;
465 }
466
des_sparc64_mod_fini(void)467 static void __exit des_sparc64_mod_fini(void)
468 {
469 crypto_unregister_algs(cipher_algs, ARRAY_SIZE(cipher_algs));
470 crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
471 }
472
473 module_init(des_sparc64_mod_init);
474 module_exit(des_sparc64_mod_fini);
475
476 MODULE_LICENSE("GPL");
477 MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
478
479 MODULE_ALIAS_CRYPTO("des");
480 MODULE_ALIAS_CRYPTO("des3_ede");
481
482 #include "crop_devid.c"
483