1#! /usr/bin/env perl
2# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     https://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# ====================================================================
17# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
18# project.
19# ====================================================================
20#
21# SHA256/512 for ARMv8.
22#
23# Performance in cycles per processed byte and improvement coefficient
24# over code generated with "default" compiler:
25#
26#		SHA256-hw	SHA256(*)	SHA512
27# Apple A7	1.97		10.5 (+33%)	6.73 (-1%(**))
28# Cortex-A53	2.38		15.5 (+115%)	10.0 (+150%(***))
29# Cortex-A57	2.31		11.6 (+86%)	7.51 (+260%(***))
30# Denver	2.01		10.5 (+26%)	6.70 (+8%)
31# X-Gene			20.0 (+100%)	12.8 (+300%(***))
32# Mongoose	2.36		13.0 (+50%)	8.36 (+33%)
33# Kryo		1.92		17.4 (+30%)	11.2 (+8%)
34#
35# (*)	Software SHA256 results are of lesser relevance, presented
36#	mostly for informational purposes.
37# (**)	The result is a trade-off: it's possible to improve it by
38#	10% (or by 1 cycle per round), but at the cost of 20% loss
39#	on Cortex-A53 (or by 4 cycles per round).
40# (***)	Super-impressive coefficients over gcc-generated code are
41#	indication of some compiler "pathology", most notably code
42#	generated with -mgeneral-regs-only is significantly faster
43#	and the gap is only 40-90%.
44
45my ($flavour, $hash, $output) = @ARGV;
46
47if ($hash eq "sha512") {
48	$BITS=512;
49	$SZ=8;
50	@Sigma0=(28,34,39);
51	@Sigma1=(14,18,41);
52	@sigma0=(1,  8, 7);
53	@sigma1=(19,61, 6);
54	$rounds=80;
55	$reg_t="x";
56} elsif ($hash eq "sha256") {
57	$BITS=256;
58	$SZ=4;
59	@Sigma0=( 2,13,22);
60	@Sigma1=( 6,11,25);
61	@sigma0=( 7,18, 3);
62	@sigma1=(17,19,10);
63	$rounds=64;
64	$reg_t="w";
65} else {
66	die "unknown hash: $hash";
67}
68
69if ($flavour && $flavour ne "void") {
70    $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
71    ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
72    ( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or
73    die "can't locate arm-xlate.pl";
74
75    open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
76    *STDOUT=*OUT;
77} else {
78    open OUT,">$output";
79    *STDOUT=*OUT;
80}
81
82$func="sha${BITS}_block_data_order_nohw";
83
84($ctx,$inp,$num,$Ktbl)=map("x$_",(0..2,30));
85
86@X=map("$reg_t$_",(3..15,0..2));
87@V=($A,$B,$C,$D,$E,$F,$G,$H)=map("$reg_t$_",(20..27));
88($t0,$t1,$t2,$t3)=map("$reg_t$_",(16,17,19,28));
89
90sub BODY_00_xx {
91my ($i,$a,$b,$c,$d,$e,$f,$g,$h)=@_;
92my $j=($i+1)&15;
93my ($T0,$T1,$T2)=(@X[($i-8)&15],@X[($i-9)&15],@X[($i-10)&15]);
94   $T0=@X[$i+3] if ($i<11);
95
96$code.=<<___	if ($i<16);
97#ifndef	__AARCH64EB__
98	rev	@X[$i],@X[$i]			// $i
99#endif
100___
101$code.=<<___	if ($i<13 && ($i&1));
102	ldp	@X[$i+1],@X[$i+2],[$inp],#2*$SZ
103___
104$code.=<<___	if ($i==13);
105	ldp	@X[14],@X[15],[$inp]
106___
107$code.=<<___	if ($i>=14);
108	ldr	@X[($i-11)&15],[sp,#`$SZ*(($i-11)%4)`]
109___
110$code.=<<___	if ($i>0 && $i<16);
111	add	$a,$a,$t1			// h+=Sigma0(a)
112___
113$code.=<<___	if ($i>=11);
114	str	@X[($i-8)&15],[sp,#`$SZ*(($i-8)%4)`]
115___
116# While ARMv8 specifies merged rotate-n-logical operation such as
117# 'eor x,y,z,ror#n', it was found to negatively affect performance
118# on Apple A7. The reason seems to be that it requires even 'y' to
119# be available earlier. This means that such merged instruction is
120# not necessarily best choice on critical path... On the other hand
121# Cortex-A5x handles merged instructions much better than disjoint
122# rotate and logical... See (**) footnote above.
123$code.=<<___	if ($i<15);
124	ror	$t0,$e,#$Sigma1[0]
125	add	$h,$h,$t2			// h+=K[i]
126	eor	$T0,$e,$e,ror#`$Sigma1[2]-$Sigma1[1]`
127	and	$t1,$f,$e
128	bic	$t2,$g,$e
129	add	$h,$h,@X[$i&15]			// h+=X[i]
130	orr	$t1,$t1,$t2			// Ch(e,f,g)
131	eor	$t2,$a,$b			// a^b, b^c in next round
132	eor	$t0,$t0,$T0,ror#$Sigma1[1]	// Sigma1(e)
133	ror	$T0,$a,#$Sigma0[0]
134	add	$h,$h,$t1			// h+=Ch(e,f,g)
135	eor	$t1,$a,$a,ror#`$Sigma0[2]-$Sigma0[1]`
136	add	$h,$h,$t0			// h+=Sigma1(e)
137	and	$t3,$t3,$t2			// (b^c)&=(a^b)
138	add	$d,$d,$h			// d+=h
139	eor	$t3,$t3,$b			// Maj(a,b,c)
140	eor	$t1,$T0,$t1,ror#$Sigma0[1]	// Sigma0(a)
141	add	$h,$h,$t3			// h+=Maj(a,b,c)
142	ldr	$t3,[$Ktbl],#$SZ		// *K++, $t2 in next round
143	//add	$h,$h,$t1			// h+=Sigma0(a)
144___
145$code.=<<___	if ($i>=15);
146	ror	$t0,$e,#$Sigma1[0]
147	add	$h,$h,$t2			// h+=K[i]
148	ror	$T1,@X[($j+1)&15],#$sigma0[0]
149	and	$t1,$f,$e
150	ror	$T2,@X[($j+14)&15],#$sigma1[0]
151	bic	$t2,$g,$e
152	ror	$T0,$a,#$Sigma0[0]
153	add	$h,$h,@X[$i&15]			// h+=X[i]
154	eor	$t0,$t0,$e,ror#$Sigma1[1]
155	eor	$T1,$T1,@X[($j+1)&15],ror#$sigma0[1]
156	orr	$t1,$t1,$t2			// Ch(e,f,g)
157	eor	$t2,$a,$b			// a^b, b^c in next round
158	eor	$t0,$t0,$e,ror#$Sigma1[2]	// Sigma1(e)
159	eor	$T0,$T0,$a,ror#$Sigma0[1]
160	add	$h,$h,$t1			// h+=Ch(e,f,g)
161	and	$t3,$t3,$t2			// (b^c)&=(a^b)
162	eor	$T2,$T2,@X[($j+14)&15],ror#$sigma1[1]
163	eor	$T1,$T1,@X[($j+1)&15],lsr#$sigma0[2]	// sigma0(X[i+1])
164	add	$h,$h,$t0			// h+=Sigma1(e)
165	eor	$t3,$t3,$b			// Maj(a,b,c)
166	eor	$t1,$T0,$a,ror#$Sigma0[2]	// Sigma0(a)
167	eor	$T2,$T2,@X[($j+14)&15],lsr#$sigma1[2]	// sigma1(X[i+14])
168	add	@X[$j],@X[$j],@X[($j+9)&15]
169	add	$d,$d,$h			// d+=h
170	add	$h,$h,$t3			// h+=Maj(a,b,c)
171	ldr	$t3,[$Ktbl],#$SZ		// *K++, $t2 in next round
172	add	@X[$j],@X[$j],$T1
173	add	$h,$h,$t1			// h+=Sigma0(a)
174	add	@X[$j],@X[$j],$T2
175___
176	($t2,$t3)=($t3,$t2);
177}
178
179$code.=<<___;
180.text
181
182.globl	$func
183.type	$func,%function
184.align	6
185$func:
186	AARCH64_SIGN_LINK_REGISTER
187	stp	x29,x30,[sp,#-128]!
188	add	x29,sp,#0
189
190	stp	x19,x20,[sp,#16]
191	stp	x21,x22,[sp,#32]
192	stp	x23,x24,[sp,#48]
193	stp	x25,x26,[sp,#64]
194	stp	x27,x28,[sp,#80]
195	sub	sp,sp,#4*$SZ
196
197	ldp	$A,$B,[$ctx]				// load context
198	ldp	$C,$D,[$ctx,#2*$SZ]
199	ldp	$E,$F,[$ctx,#4*$SZ]
200	add	$num,$inp,$num,lsl#`log(16*$SZ)/log(2)`	// end of input
201	ldp	$G,$H,[$ctx,#6*$SZ]
202	adrp	$Ktbl,:pg_hi21:.LK$BITS
203	add	$Ktbl,$Ktbl,:lo12:.LK$BITS
204	stp	$ctx,$num,[x29,#96]
205
206.Loop:
207	ldp	@X[0],@X[1],[$inp],#2*$SZ
208	ldr	$t2,[$Ktbl],#$SZ			// *K++
209	eor	$t3,$B,$C				// magic seed
210	str	$inp,[x29,#112]
211___
212for ($i=0;$i<16;$i++)	{ &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
213$code.=".Loop_16_xx:\n";
214for (;$i<32;$i++)	{ &BODY_00_xx($i,@V); unshift(@V,pop(@V)); }
215$code.=<<___;
216	cbnz	$t2,.Loop_16_xx
217
218	ldp	$ctx,$num,[x29,#96]
219	ldr	$inp,[x29,#112]
220	sub	$Ktbl,$Ktbl,#`$SZ*($rounds+1)`		// rewind
221
222	ldp	@X[0],@X[1],[$ctx]
223	ldp	@X[2],@X[3],[$ctx,#2*$SZ]
224	add	$inp,$inp,#14*$SZ			// advance input pointer
225	ldp	@X[4],@X[5],[$ctx,#4*$SZ]
226	add	$A,$A,@X[0]
227	ldp	@X[6],@X[7],[$ctx,#6*$SZ]
228	add	$B,$B,@X[1]
229	add	$C,$C,@X[2]
230	add	$D,$D,@X[3]
231	stp	$A,$B,[$ctx]
232	add	$E,$E,@X[4]
233	add	$F,$F,@X[5]
234	stp	$C,$D,[$ctx,#2*$SZ]
235	add	$G,$G,@X[6]
236	add	$H,$H,@X[7]
237	cmp	$inp,$num
238	stp	$E,$F,[$ctx,#4*$SZ]
239	stp	$G,$H,[$ctx,#6*$SZ]
240	b.ne	.Loop
241
242	ldp	x19,x20,[x29,#16]
243	add	sp,sp,#4*$SZ
244	ldp	x21,x22,[x29,#32]
245	ldp	x23,x24,[x29,#48]
246	ldp	x25,x26,[x29,#64]
247	ldp	x27,x28,[x29,#80]
248	ldp	x29,x30,[sp],#128
249	AARCH64_VALIDATE_LINK_REGISTER
250	ret
251.size	$func,.-$func
252
253.section .rodata
254.align	6
255.type	.LK$BITS,%object
256.LK$BITS:
257___
258$code.=<<___ if ($SZ==8);
259	.quad	0x428a2f98d728ae22,0x7137449123ef65cd
260	.quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
261	.quad	0x3956c25bf348b538,0x59f111f1b605d019
262	.quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
263	.quad	0xd807aa98a3030242,0x12835b0145706fbe
264	.quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
265	.quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
266	.quad	0x9bdc06a725c71235,0xc19bf174cf692694
267	.quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
268	.quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
269	.quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
270	.quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
271	.quad	0x983e5152ee66dfab,0xa831c66d2db43210
272	.quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
273	.quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
274	.quad	0x06ca6351e003826f,0x142929670a0e6e70
275	.quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
276	.quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
277	.quad	0x650a73548baf63de,0x766a0abb3c77b2a8
278	.quad	0x81c2c92e47edaee6,0x92722c851482353b
279	.quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
280	.quad	0xc24b8b70d0f89791,0xc76c51a30654be30
281	.quad	0xd192e819d6ef5218,0xd69906245565a910
282	.quad	0xf40e35855771202a,0x106aa07032bbd1b8
283	.quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
284	.quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
285	.quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
286	.quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
287	.quad	0x748f82ee5defb2fc,0x78a5636f43172f60
288	.quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
289	.quad	0x90befffa23631e28,0xa4506cebde82bde9
290	.quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
291	.quad	0xca273eceea26619c,0xd186b8c721c0c207
292	.quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
293	.quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
294	.quad	0x113f9804bef90dae,0x1b710b35131c471b
295	.quad	0x28db77f523047d84,0x32caab7b40c72493
296	.quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
297	.quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
298	.quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
299	.quad	0	// terminator
300___
301$code.=<<___ if ($SZ==4);
302	.long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
303	.long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
304	.long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
305	.long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
306	.long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
307	.long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
308	.long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
309	.long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
310	.long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
311	.long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
312	.long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
313	.long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
314	.long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
315	.long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
316	.long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
317	.long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
318	.long	0	//terminator
319___
320$code.=<<___;
321.size	.LK$BITS,.-.LK$BITS
322.asciz	"SHA$BITS block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
323.align	2
324___
325
326if ($SZ==4) {
327my $Ktbl="x3";
328
329my ($ABCD,$EFGH,$abcd)=map("v$_.16b",(0..2));
330my @MSG=map("v$_.16b",(4..7));
331my ($W0,$W1)=("v16.4s","v17.4s");
332my ($ABCD_SAVE,$EFGH_SAVE)=("v18.16b","v19.16b");
333
334$code.=<<___;
335.text
336#ifndef	__KERNEL__
337.globl	sha256_block_data_order_hw
338.type	sha256_block_data_order_hw,%function
339.align	6
340sha256_block_data_order_hw:
341	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
342	AARCH64_VALID_CALL_TARGET
343	stp		x29,x30,[sp,#-16]!
344	add		x29,sp,#0
345
346	ld1.32		{$ABCD,$EFGH},[$ctx]
347	adrp		$Ktbl,:pg_hi21:.LK256
348	add		$Ktbl,$Ktbl,:lo12:.LK256
349
350.Loop_hw:
351	ld1		{@MSG[0]-@MSG[3]},[$inp],#64
352	sub		$num,$num,#1
353	ld1.32		{$W0},[$Ktbl],#16
354	rev32		@MSG[0],@MSG[0]
355	rev32		@MSG[1],@MSG[1]
356	rev32		@MSG[2],@MSG[2]
357	rev32		@MSG[3],@MSG[3]
358	orr		$ABCD_SAVE,$ABCD,$ABCD		// offload
359	orr		$EFGH_SAVE,$EFGH,$EFGH
360___
361for($i=0;$i<12;$i++) {
362$code.=<<___;
363	ld1.32		{$W1},[$Ktbl],#16
364	add.i32		$W0,$W0,@MSG[0]
365	sha256su0	@MSG[0],@MSG[1]
366	orr		$abcd,$ABCD,$ABCD
367	sha256h		$ABCD,$EFGH,$W0
368	sha256h2	$EFGH,$abcd,$W0
369	sha256su1	@MSG[0],@MSG[2],@MSG[3]
370___
371	($W0,$W1)=($W1,$W0);	push(@MSG,shift(@MSG));
372}
373$code.=<<___;
374	ld1.32		{$W1},[$Ktbl],#16
375	add.i32		$W0,$W0,@MSG[0]
376	orr		$abcd,$ABCD,$ABCD
377	sha256h		$ABCD,$EFGH,$W0
378	sha256h2	$EFGH,$abcd,$W0
379
380	ld1.32		{$W0},[$Ktbl],#16
381	add.i32		$W1,$W1,@MSG[1]
382	orr		$abcd,$ABCD,$ABCD
383	sha256h		$ABCD,$EFGH,$W1
384	sha256h2	$EFGH,$abcd,$W1
385
386	ld1.32		{$W1},[$Ktbl]
387	add.i32		$W0,$W0,@MSG[2]
388	sub		$Ktbl,$Ktbl,#$rounds*$SZ-16	// rewind
389	orr		$abcd,$ABCD,$ABCD
390	sha256h		$ABCD,$EFGH,$W0
391	sha256h2	$EFGH,$abcd,$W0
392
393	add.i32		$W1,$W1,@MSG[3]
394	orr		$abcd,$ABCD,$ABCD
395	sha256h		$ABCD,$EFGH,$W1
396	sha256h2	$EFGH,$abcd,$W1
397
398	add.i32		$ABCD,$ABCD,$ABCD_SAVE
399	add.i32		$EFGH,$EFGH,$EFGH_SAVE
400
401	cbnz		$num,.Loop_hw
402
403	st1.32		{$ABCD,$EFGH},[$ctx]
404
405	ldr		x29,[sp],#16
406	ret
407.size	sha256_block_data_order_hw,.-sha256_block_data_order_hw
408#endif
409___
410}
411
412if ($SZ==8) {
413my $Ktbl="x3";
414
415my @H = map("v$_.16b",(0..4));
416my ($fg,$de,$m9_10)=map("v$_.16b",(5..7));
417my @MSG=map("v$_.16b",(16..23));
418my ($W0,$W1)=("v24.2d","v25.2d");
419my ($AB,$CD,$EF,$GH)=map("v$_.16b",(26..29));
420
421$code.=<<___;
422.text
423#ifndef	__KERNEL__
424.globl	sha512_block_data_order_hw
425.type	sha512_block_data_order_hw,%function
426.align	6
427sha512_block_data_order_hw:
428	// Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
429	AARCH64_VALID_CALL_TARGET
430	stp		x29,x30,[sp,#-16]!
431	add		x29,sp,#0
432
433	ld1		{@MSG[0]-@MSG[3]},[$inp],#64	// load input
434	ld1		{@MSG[4]-@MSG[7]},[$inp],#64
435
436	ld1.64		{@H[0]-@H[3]},[$ctx]		// load context
437	adrp		$Ktbl,:pg_hi21:.LK512
438	add		$Ktbl,$Ktbl,:lo12:.LK512
439
440	rev64		@MSG[0],@MSG[0]
441	rev64		@MSG[1],@MSG[1]
442	rev64		@MSG[2],@MSG[2]
443	rev64		@MSG[3],@MSG[3]
444	rev64		@MSG[4],@MSG[4]
445	rev64		@MSG[5],@MSG[5]
446	rev64		@MSG[6],@MSG[6]
447	rev64		@MSG[7],@MSG[7]
448	b		.Loop_hw
449
450.align	4
451.Loop_hw:
452	ld1.64		{$W0},[$Ktbl],#16
453	subs		$num,$num,#1
454	sub		x4,$inp,#128
455	orr		$AB,@H[0],@H[0]			// offload
456	orr		$CD,@H[1],@H[1]
457	orr		$EF,@H[2],@H[2]
458	orr		$GH,@H[3],@H[3]
459	csel		$inp,$inp,x4,ne			// conditional rewind
460___
461for($i=0;$i<32;$i++) {
462$code.=<<___;
463	add.i64		$W0,$W0,@MSG[0]
464	ld1.64		{$W1},[$Ktbl],#16
465	ext		$W0,$W0,$W0,#8
466	ext		$fg,@H[2],@H[3],#8
467	ext		$de,@H[1],@H[2],#8
468	add.i64		@H[3],@H[3],$W0			// "T1 + H + K512[i]"
469	 sha512su0	@MSG[0],@MSG[1]
470	 ext		$m9_10,@MSG[4],@MSG[5],#8
471	sha512h		@H[3],$fg,$de
472	 sha512su1	@MSG[0],@MSG[7],$m9_10
473	add.i64		@H[4],@H[1],@H[3]		// "D + T1"
474	sha512h2	@H[3],$H[1],@H[0]
475___
476	($W0,$W1)=($W1,$W0);	push(@MSG,shift(@MSG));
477	@H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
478}
479for(;$i<40;$i++) {
480$code.=<<___	if ($i<39);
481	ld1.64		{$W1},[$Ktbl],#16
482___
483$code.=<<___	if ($i==39);
484	sub		$Ktbl,$Ktbl,#$rounds*$SZ	// rewind
485___
486$code.=<<___;
487	add.i64		$W0,$W0,@MSG[0]
488	 ld1		{@MSG[0]},[$inp],#16		// load next input
489	ext		$W0,$W0,$W0,#8
490	ext		$fg,@H[2],@H[3],#8
491	ext		$de,@H[1],@H[2],#8
492	add.i64		@H[3],@H[3],$W0			// "T1 + H + K512[i]"
493	sha512h		@H[3],$fg,$de
494	 rev64		@MSG[0],@MSG[0]
495	add.i64		@H[4],@H[1],@H[3]		// "D + T1"
496	sha512h2	@H[3],$H[1],@H[0]
497___
498	($W0,$W1)=($W1,$W0);	push(@MSG,shift(@MSG));
499	@H = (@H[3],@H[0],@H[4],@H[2],@H[1]);
500}
501$code.=<<___;
502	add.i64		@H[0],@H[0],$AB			// accumulate
503	add.i64		@H[1],@H[1],$CD
504	add.i64		@H[2],@H[2],$EF
505	add.i64		@H[3],@H[3],$GH
506
507	cbnz		$num,.Loop_hw
508
509	st1.64		{@H[0]-@H[3]},[$ctx]		// store context
510
511	ldr		x29,[sp],#16
512	ret
513.size	sha512_block_data_order_hw,.-sha512_block_data_order_hw
514#endif
515___
516}
517
518{   my  %opcode = (
519	"sha256h"	=> 0x5e004000,	"sha256h2"	=> 0x5e005000,
520	"sha256su0"	=> 0x5e282800,	"sha256su1"	=> 0x5e006000	);
521
522    sub unsha256 {
523	my ($mnemonic,$arg)=@_;
524
525	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
526	&&
527	sprintf ".inst\t0x%08x\t//%s %s",
528			$opcode{$mnemonic}|$1|($2<<5)|($3<<16),
529			$mnemonic,$arg;
530    }
531}
532
533{   my  %opcode = (
534	"sha512h"	=> 0xce608000,	"sha512h2"	=> 0xce608400,
535	"sha512su0"	=> 0xcec08000,	"sha512su1"	=> 0xce608800	);
536
537    sub unsha512 {
538	my ($mnemonic,$arg)=@_;
539
540	$arg =~ m/[qv]([0-9]+)[^,]*,\s*[qv]([0-9]+)[^,]*(?:,\s*[qv]([0-9]+))?/o
541	&&
542	sprintf ".inst\t0x%08x\t//%s %s",
543			$opcode{$mnemonic}|$1|($2<<5)|($3<<16),
544			$mnemonic,$arg;
545    }
546}
547
548open SELF,$0;
549while(<SELF>) {
550        next if (/^#!/);
551        last if (!s/^#/\/\// and !/^$/);
552        print;
553}
554close SELF;
555
556foreach(split("\n",$code)) {
557
558	s/\`([^\`]*)\`/eval($1)/ge;
559
560	s/\b(sha512\w+)\s+([qv].*)/unsha512($1,$2)/ge	or
561	s/\b(sha256\w+)\s+([qv].*)/unsha256($1,$2)/ge;
562
563	s/\bq([0-9]+)\b/v$1.16b/g;		# old->new registers
564
565	s/\.[ui]?8(\s)/$1/;
566	s/\.\w?64\b//		and s/\.16b/\.2d/g	or
567	s/\.\w?32\b//		and s/\.16b/\.4s/g;
568	m/\bext\b/		and s/\.2d/\.16b/g	or
569	m/(ld|st)1[^\[]+\[0\]/	and s/\.4s/\.s/g;
570
571	print $_,"\n";
572}
573
574close STDOUT or die "error closing STDOUT: $!";
575