1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3  * SM3 Secure Hash Algorithm, AVX assembler accelerated.
4  * specified in: https://datatracker.ietf.org/doc/html/draft-sca-cfrg-sm3-02
5  *
6  * Copyright (C) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
7  */
8 
9 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
10 
11 #include <crypto/internal/hash.h>
12 #include <crypto/internal/simd.h>
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/types.h>
16 #include <crypto/sm3.h>
17 #include <crypto/sm3_base.h>
18 #include <asm/simd.h>
19 
20 asmlinkage void sm3_transform_avx(struct sm3_state *state,
21 			const u8 *data, int nblocks);
22 
sm3_avx_update(struct shash_desc * desc,const u8 * data,unsigned int len)23 static int sm3_avx_update(struct shash_desc *desc, const u8 *data,
24 			 unsigned int len)
25 {
26 	struct sm3_state *sctx = shash_desc_ctx(desc);
27 
28 	if (!crypto_simd_usable() ||
29 			(sctx->count % SM3_BLOCK_SIZE) + len < SM3_BLOCK_SIZE) {
30 		sm3_update(sctx, data, len);
31 		return 0;
32 	}
33 
34 	/*
35 	 * Make sure struct sm3_state begins directly with the SM3
36 	 * 256-bit internal state, as this is what the asm functions expect.
37 	 */
38 	BUILD_BUG_ON(offsetof(struct sm3_state, state) != 0);
39 
40 	kernel_fpu_begin();
41 	sm3_base_do_update(desc, data, len, sm3_transform_avx);
42 	kernel_fpu_end();
43 
44 	return 0;
45 }
46 
sm3_avx_finup(struct shash_desc * desc,const u8 * data,unsigned int len,u8 * out)47 static int sm3_avx_finup(struct shash_desc *desc, const u8 *data,
48 		      unsigned int len, u8 *out)
49 {
50 	if (!crypto_simd_usable()) {
51 		struct sm3_state *sctx = shash_desc_ctx(desc);
52 
53 		if (len)
54 			sm3_update(sctx, data, len);
55 
56 		sm3_final(sctx, out);
57 		return 0;
58 	}
59 
60 	kernel_fpu_begin();
61 	if (len)
62 		sm3_base_do_update(desc, data, len, sm3_transform_avx);
63 	sm3_base_do_finalize(desc, sm3_transform_avx);
64 	kernel_fpu_end();
65 
66 	return sm3_base_finish(desc, out);
67 }
68 
sm3_avx_final(struct shash_desc * desc,u8 * out)69 static int sm3_avx_final(struct shash_desc *desc, u8 *out)
70 {
71 	if (!crypto_simd_usable()) {
72 		sm3_final(shash_desc_ctx(desc), out);
73 		return 0;
74 	}
75 
76 	kernel_fpu_begin();
77 	sm3_base_do_finalize(desc, sm3_transform_avx);
78 	kernel_fpu_end();
79 
80 	return sm3_base_finish(desc, out);
81 }
82 
83 static struct shash_alg sm3_avx_alg = {
84 	.digestsize	=	SM3_DIGEST_SIZE,
85 	.init		=	sm3_base_init,
86 	.update		=	sm3_avx_update,
87 	.final		=	sm3_avx_final,
88 	.finup		=	sm3_avx_finup,
89 	.descsize	=	sizeof(struct sm3_state),
90 	.base		=	{
91 		.cra_name	=	"sm3",
92 		.cra_driver_name =	"sm3-avx",
93 		.cra_priority	=	300,
94 		.cra_blocksize	=	SM3_BLOCK_SIZE,
95 		.cra_module	=	THIS_MODULE,
96 	}
97 };
98 
sm3_avx_mod_init(void)99 static int __init sm3_avx_mod_init(void)
100 {
101 	const char *feature_name;
102 
103 	if (!boot_cpu_has(X86_FEATURE_AVX)) {
104 		pr_info("AVX instruction are not detected.\n");
105 		return -ENODEV;
106 	}
107 
108 	if (!boot_cpu_has(X86_FEATURE_BMI2)) {
109 		pr_info("BMI2 instruction are not detected.\n");
110 		return -ENODEV;
111 	}
112 
113 	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
114 				&feature_name)) {
115 		pr_info("CPU feature '%s' is not supported.\n", feature_name);
116 		return -ENODEV;
117 	}
118 
119 	return crypto_register_shash(&sm3_avx_alg);
120 }
121 
sm3_avx_mod_exit(void)122 static void __exit sm3_avx_mod_exit(void)
123 {
124 	crypto_unregister_shash(&sm3_avx_alg);
125 }
126 
127 module_init(sm3_avx_mod_init);
128 module_exit(sm3_avx_mod_exit);
129 
130 MODULE_LICENSE("GPL v2");
131 MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
132 MODULE_DESCRIPTION("SM3 Secure Hash Algorithm, AVX assembler accelerated");
133 MODULE_ALIAS_CRYPTO("sm3");
134 MODULE_ALIAS_CRYPTO("sm3-avx");
135