1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * arch/arm64/include/asm/xor.h
4  *
5  * Authors: Jackie Liu <liuyun01@kylinos.cn>
6  * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
7  */
8 
9 #include <linux/hardirq.h>
10 #include <asm-generic/xor.h>
11 #include <asm/hwcap.h>
12 #include <asm/neon.h>
13 
14 #ifdef CONFIG_KERNEL_MODE_NEON
15 
16 extern struct xor_block_template const xor_block_inner_neon;
17 
18 static void
xor_neon_2(unsigned long bytes,unsigned long * __restrict p1,const unsigned long * __restrict p2)19 xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
20 	   const unsigned long * __restrict p2)
21 {
22 	kernel_neon_begin();
23 	xor_block_inner_neon.do_2(bytes, p1, p2);
24 	kernel_neon_end();
25 }
26 
27 static void
xor_neon_3(unsigned long bytes,unsigned long * __restrict p1,const unsigned long * __restrict p2,const unsigned long * __restrict p3)28 xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
29 	   const unsigned long * __restrict p2,
30 	   const unsigned long * __restrict p3)
31 {
32 	kernel_neon_begin();
33 	xor_block_inner_neon.do_3(bytes, p1, p2, p3);
34 	kernel_neon_end();
35 }
36 
37 static void
xor_neon_4(unsigned long bytes,unsigned long * __restrict p1,const unsigned long * __restrict p2,const unsigned long * __restrict p3,const unsigned long * __restrict p4)38 xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
39 	   const unsigned long * __restrict p2,
40 	   const unsigned long * __restrict p3,
41 	   const unsigned long * __restrict p4)
42 {
43 	kernel_neon_begin();
44 	xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
45 	kernel_neon_end();
46 }
47 
48 static void
xor_neon_5(unsigned long bytes,unsigned long * __restrict p1,const unsigned long * __restrict p2,const unsigned long * __restrict p3,const unsigned long * __restrict p4,const unsigned long * __restrict p5)49 xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
50 	   const unsigned long * __restrict p2,
51 	   const unsigned long * __restrict p3,
52 	   const unsigned long * __restrict p4,
53 	   const unsigned long * __restrict p5)
54 {
55 	kernel_neon_begin();
56 	xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
57 	kernel_neon_end();
58 }
59 
60 static struct xor_block_template xor_block_arm64 = {
61 	.name   = "arm64_neon",
62 	.do_2   = xor_neon_2,
63 	.do_3   = xor_neon_3,
64 	.do_4   = xor_neon_4,
65 	.do_5	= xor_neon_5
66 };
67 #undef XOR_TRY_TEMPLATES
68 #define XOR_TRY_TEMPLATES           \
69 	do {        \
70 		xor_speed(&xor_block_8regs);    \
71 		xor_speed(&xor_block_32regs);    \
72 		if (cpu_has_neon()) { \
73 			xor_speed(&xor_block_arm64);\
74 		} \
75 	} while (0)
76 
77 #endif /* ! CONFIG_KERNEL_MODE_NEON */
78