1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Glue code for POLYVAL using ARMv8 Crypto Extensions
4 *
5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
6 * Copyright (c) 2009 Intel Corp.
7 * Author: Huang Ying <ying.huang@intel.com>
8 * Copyright 2021 Google LLC
9 */
10
11 /*
12 * Glue code based on ghash-clmulni-intel_glue.c.
13 *
14 * This implementation of POLYVAL uses montgomery multiplication accelerated by
15 * ARMv8 Crypto Extensions instructions to implement the finite field operations.
16 */
17
18 #include <asm/neon.h>
19 #include <crypto/internal/hash.h>
20 #include <crypto/polyval.h>
21 #include <crypto/utils.h>
22 #include <linux/cpufeature.h>
23 #include <linux/errno.h>
24 #include <linux/kernel.h>
25 #include <linux/module.h>
26 #include <linux/string.h>
27
28 #define NUM_KEY_POWERS 8
29
30 struct polyval_tfm_ctx {
31 /*
32 * These powers must be in the order h^8, ..., h^1.
33 */
34 u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE];
35 };
36
37 struct polyval_desc_ctx {
38 u8 buffer[POLYVAL_BLOCK_SIZE];
39 };
40
41 asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys,
42 const u8 *in, size_t nblocks, u8 *accumulator);
43 asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2);
44
internal_polyval_update(const struct polyval_tfm_ctx * keys,const u8 * in,size_t nblocks,u8 * accumulator)45 static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
46 const u8 *in, size_t nblocks, u8 *accumulator)
47 {
48 kernel_neon_begin();
49 pmull_polyval_update(keys, in, nblocks, accumulator);
50 kernel_neon_end();
51 }
52
internal_polyval_mul(u8 * op1,const u8 * op2)53 static void internal_polyval_mul(u8 *op1, const u8 *op2)
54 {
55 kernel_neon_begin();
56 pmull_polyval_mul(op1, op2);
57 kernel_neon_end();
58 }
59
polyval_arm64_setkey(struct crypto_shash * tfm,const u8 * key,unsigned int keylen)60 static int polyval_arm64_setkey(struct crypto_shash *tfm,
61 const u8 *key, unsigned int keylen)
62 {
63 struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm);
64 int i;
65
66 if (keylen != POLYVAL_BLOCK_SIZE)
67 return -EINVAL;
68
69 memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
70
71 for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
72 memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
73 internal_polyval_mul(tctx->key_powers[i],
74 tctx->key_powers[i+1]);
75 }
76
77 return 0;
78 }
79
polyval_arm64_init(struct shash_desc * desc)80 static int polyval_arm64_init(struct shash_desc *desc)
81 {
82 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
83
84 memset(dctx, 0, sizeof(*dctx));
85
86 return 0;
87 }
88
polyval_arm64_update(struct shash_desc * desc,const u8 * src,unsigned int srclen)89 static int polyval_arm64_update(struct shash_desc *desc,
90 const u8 *src, unsigned int srclen)
91 {
92 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
93 const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
94 unsigned int nblocks;
95
96 do {
97 /* allow rescheduling every 4K bytes */
98 nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
99 internal_polyval_update(tctx, src, nblocks, dctx->buffer);
100 srclen -= nblocks * POLYVAL_BLOCK_SIZE;
101 src += nblocks * POLYVAL_BLOCK_SIZE;
102 } while (srclen >= POLYVAL_BLOCK_SIZE);
103
104 return srclen;
105 }
106
polyval_arm64_finup(struct shash_desc * desc,const u8 * src,unsigned int len,u8 * dst)107 static int polyval_arm64_finup(struct shash_desc *desc, const u8 *src,
108 unsigned int len, u8 *dst)
109 {
110 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
111 const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
112
113 if (len) {
114 crypto_xor(dctx->buffer, src, len);
115 internal_polyval_mul(dctx->buffer,
116 tctx->key_powers[NUM_KEY_POWERS-1]);
117 }
118
119 memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
120
121 return 0;
122 }
123
124 static struct shash_alg polyval_alg = {
125 .digestsize = POLYVAL_DIGEST_SIZE,
126 .init = polyval_arm64_init,
127 .update = polyval_arm64_update,
128 .finup = polyval_arm64_finup,
129 .setkey = polyval_arm64_setkey,
130 .descsize = sizeof(struct polyval_desc_ctx),
131 .base = {
132 .cra_name = "polyval",
133 .cra_driver_name = "polyval-ce",
134 .cra_priority = 200,
135 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
136 .cra_blocksize = POLYVAL_BLOCK_SIZE,
137 .cra_ctxsize = sizeof(struct polyval_tfm_ctx),
138 .cra_module = THIS_MODULE,
139 },
140 };
141
polyval_ce_mod_init(void)142 static int __init polyval_ce_mod_init(void)
143 {
144 return crypto_register_shash(&polyval_alg);
145 }
146
polyval_ce_mod_exit(void)147 static void __exit polyval_ce_mod_exit(void)
148 {
149 crypto_unregister_shash(&polyval_alg);
150 }
151
152 module_cpu_feature_match(PMULL, polyval_ce_mod_init)
153 module_exit(polyval_ce_mod_exit);
154
155 MODULE_LICENSE("GPL");
156 MODULE_DESCRIPTION("POLYVAL hash function accelerated by ARMv8 Crypto Extensions");
157 MODULE_ALIAS_CRYPTO("polyval");
158 MODULE_ALIAS_CRYPTO("polyval-ce");
159