1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
4 *
5 * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8 #include <linux/cpufeature.h>
9
10 #include <crypto/internal/simd.h>
11
12 #include <asm/neon.h>
13 #include <asm/simd.h>
14
15 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd);
16 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
17
18 #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
19
20 asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len,
21 u8 out[16]);
22 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
23
crc_t10dif_arch(u16 crc,const u8 * data,size_t length)24 static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length)
25 {
26 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) {
27 if (static_branch_likely(&have_pmull)) {
28 if (crypto_simd_usable()) {
29 kernel_neon_begin();
30 crc = crc_t10dif_pmull_p64(crc, data, length);
31 kernel_neon_end();
32 return crc;
33 }
34 } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE &&
35 static_branch_likely(&have_asimd) &&
36 crypto_simd_usable()) {
37 u8 buf[16];
38
39 kernel_neon_begin();
40 crc_t10dif_pmull_p8(crc, data, length, buf);
41 kernel_neon_end();
42
43 return crc_t10dif_generic(0, buf, sizeof(buf));
44 }
45 }
46 return crc_t10dif_generic(crc, data, length);
47 }
48
49 #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
crc_t10dif_mod_init_arch(void)50 static inline void crc_t10dif_mod_init_arch(void)
51 {
52 if (cpu_have_named_feature(ASIMD)) {
53 static_branch_enable(&have_asimd);
54 if (cpu_have_named_feature(PMULL))
55 static_branch_enable(&have_pmull);
56 }
57 }
58