1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Calculate a CRC T10-DIF with vpmsum acceleration
4 *
5 * Copyright 2017, Daniel Axtens, IBM Corporation.
6 * [based on crc32c-vpmsum_glue.c]
7 */
8
9 #include <asm/switch_to.h>
10 #include <crypto/internal/simd.h>
11 #include <linux/cpufeature.h>
12 #include <linux/jump_label.h>
13 #include <linux/preempt.h>
14 #include <linux/uaccess.h>
15
16 #define VMX_ALIGN 16
17 #define VMX_ALIGN_MASK (VMX_ALIGN-1)
18
19 #define VECTOR_BREAKPOINT 64
20
21 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
22
23 u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
24
crc_t10dif_arch(u16 crci,const u8 * p,size_t len)25 static inline u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
26 {
27 unsigned int prealign;
28 unsigned int tail;
29 u32 crc = crci;
30
31 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32 !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
33 return crc_t10dif_generic(crc, p, len);
34
35 if ((unsigned long)p & VMX_ALIGN_MASK) {
36 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
37 crc = crc_t10dif_generic(crc, p, prealign);
38 len -= prealign;
39 p += prealign;
40 }
41
42 if (len & ~VMX_ALIGN_MASK) {
43 crc <<= 16;
44 preempt_disable();
45 pagefault_disable();
46 enable_kernel_altivec();
47 crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
48 disable_kernel_altivec();
49 pagefault_enable();
50 preempt_enable();
51 crc >>= 16;
52 }
53
54 tail = len & VMX_ALIGN_MASK;
55 if (tail) {
56 p += len & ~VMX_ALIGN_MASK;
57 crc = crc_t10dif_generic(crc, p, tail);
58 }
59
60 return crc & 0xffff;
61 }
62
63 #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch
crc_t10dif_mod_init_arch(void)64 static inline void crc_t10dif_mod_init_arch(void)
65 {
66 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
67 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
68 static_branch_enable(&have_vec_crypto);
69 }
70