1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions
4 *
5 * Module based on arm64/crypto/crc32-arm.c
6 *
7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8 * Copyright (C) 2018 MIPS Tech, LLC
9 */
10
11 #include <linux/cpufeature.h>
12 #include <asm/mipsregs.h>
13 #include <linux/unaligned.h>
14
15 #ifndef TOOLCHAIN_SUPPORTS_CRC
16 #define _ASM_SET_CRC(OP, SZ, TYPE) \
17 _ASM_MACRO_3R(OP, rt, rs, rt2, \
18 ".ifnc \\rt, \\rt2\n\t" \
19 ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \
20 ".endif\n\t" \
21 _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \
22 ((SZ) << 6) | ((TYPE) << 8)) \
23 _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \
24 ((SZ) << 14) | ((TYPE) << 3)))
25 #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t"
26 #else /* !TOOLCHAIN_SUPPORTS_CRC */
27 #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t"
28 #define _ASM_UNSET_CRC(op, SZ, TYPE)
29 #endif
30
31 #define __CRC32(crc, value, op, SZ, TYPE) \
32 do { \
33 __asm__ __volatile__( \
34 ".set push\n\t" \
35 _ASM_SET_CRC(op, SZ, TYPE) \
36 #op " %0, %1, %0\n\t" \
37 _ASM_UNSET_CRC(op, SZ, TYPE) \
38 ".set pop" \
39 : "+r" (crc) \
40 : "r" (value)); \
41 } while (0)
42
43 #define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0)
44 #define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0)
45 #define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0)
46 #define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0)
47 #define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1)
48 #define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1)
49 #define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1)
50 #define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1)
51
52 #define _CRC32(crc, value, size, op) \
53 _CRC32_##op##size(crc, value)
54
55 #define CRC32(crc, value, size) \
56 _CRC32(crc, value, size, crc32)
57
58 #define CRC32C(crc, value, size) \
59 _CRC32(crc, value, size, crc32c)
60
61 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32);
62
crc32_le_arch(u32 crc,const u8 * p,size_t len)63 static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
64 {
65 if (!static_branch_likely(&have_crc32))
66 return crc32_le_base(crc, p, len);
67
68 if (IS_ENABLED(CONFIG_64BIT)) {
69 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
70 u64 value = get_unaligned_le64(p);
71
72 CRC32(crc, value, d);
73 }
74
75 if (len & sizeof(u32)) {
76 u32 value = get_unaligned_le32(p);
77
78 CRC32(crc, value, w);
79 p += sizeof(u32);
80 }
81 } else {
82 for (; len >= sizeof(u32); len -= sizeof(u32)) {
83 u32 value = get_unaligned_le32(p);
84
85 CRC32(crc, value, w);
86 p += sizeof(u32);
87 }
88 }
89
90 if (len & sizeof(u16)) {
91 u16 value = get_unaligned_le16(p);
92
93 CRC32(crc, value, h);
94 p += sizeof(u16);
95 }
96
97 if (len & sizeof(u8)) {
98 u8 value = *p++;
99
100 CRC32(crc, value, b);
101 }
102
103 return crc;
104 }
105
crc32c_arch(u32 crc,const u8 * p,size_t len)106 static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
107 {
108 if (!static_branch_likely(&have_crc32))
109 return crc32c_base(crc, p, len);
110
111 if (IS_ENABLED(CONFIG_64BIT)) {
112 for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) {
113 u64 value = get_unaligned_le64(p);
114
115 CRC32C(crc, value, d);
116 }
117
118 if (len & sizeof(u32)) {
119 u32 value = get_unaligned_le32(p);
120
121 CRC32C(crc, value, w);
122 p += sizeof(u32);
123 }
124 } else {
125 for (; len >= sizeof(u32); len -= sizeof(u32)) {
126 u32 value = get_unaligned_le32(p);
127
128 CRC32C(crc, value, w);
129 p += sizeof(u32);
130 }
131 }
132
133 if (len & sizeof(u16)) {
134 u16 value = get_unaligned_le16(p);
135
136 CRC32C(crc, value, h);
137 p += sizeof(u16);
138 }
139
140 if (len & sizeof(u8)) {
141 u8 value = *p++;
142
143 CRC32C(crc, value, b);
144 }
145 return crc;
146 }
147
148 #define crc32_be_arch crc32_be_base /* not implemented on this arch */
149
150 #define crc32_mod_init_arch crc32_mod_init_arch
crc32_mod_init_arch(void)151 static inline void crc32_mod_init_arch(void)
152 {
153 if (cpu_have_feature(cpu_feature(MIPS_CRC32)))
154 static_branch_enable(&have_crc32);
155 }
156
crc32_optimizations_arch(void)157 static inline u32 crc32_optimizations_arch(void)
158 {
159 if (static_key_enabled(&have_crc32))
160 return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION;
161 return 0;
162 }
163