1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *    User Space Access Routines
4 *
5 *    Copyright (C) 2000-2002 Hewlett-Packard (John Marvin)
6 *    Copyright (C) 2000 Richard Hirst <rhirst with parisc-linux.org>
7 *    Copyright (C) 2001 Matthieu Delahaye <delahaym at esiee.fr>
8 *    Copyright (C) 2003 Randolph Chung <tausq with parisc-linux.org>
9 *    Copyright (C) 2017 Helge Deller <deller@gmx.de>
10 *    Copyright (C) 2017 John David Anglin <dave.anglin@bell.net>
11 */
12
13/*
14 * These routines still have plenty of room for optimization
15 * (word & doubleword load/store, dual issue, store hints, etc.).
16 */
17
18/*
19 * The following routines assume that space register 3 (sr3) contains
20 * the space id associated with the current users address space.
21 */
22
23
24	.text
25
26#include <asm/assembly.h>
27#include <asm/errno.h>
28#include <linux/linkage.h>
29
30	/*
31	 * unsigned long lclear_user(void *to, unsigned long n)
32	 *
33	 * Returns 0 for success.
34	 * otherwise, returns number of bytes not transferred.
35	 */
36
37ENTRY_CFI(lclear_user)
38	comib,=,n   0,%r25,$lclu_done
39$lclu_loop:
40	addib,<>    -1,%r25,$lclu_loop
411:	stbs,ma     %r0,1(%sr3,%r26)
42
43$lclu_done:
44	bv          %r0(%r2)
45	copy        %r25,%r28
46
472:	b           $lclu_done
48	ldo         1(%r25),%r25
49
50	ASM_EXCEPTIONTABLE_ENTRY(1b,2b)
51ENDPROC_CFI(lclear_user)
52
53
54/*
55 * unsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len)
56 *
57 * Inputs:
58 * - sr1 already contains space of source region
59 * - sr2 already contains space of destination region
60 *
61 * Returns:
62 * - number of bytes that could not be copied.
63 *   On success, this will be zero.
64 *
65 * This code is based on a C-implementation of a copy routine written by
66 * Randolph Chung, which in turn was derived from the glibc.
67 *
68 * Several strategies are tried to try to get the best performance for various
69 * conditions. In the optimal case, we copy by loops that copy 32- or 16-bytes
70 * at a time using general registers.  Unaligned copies are handled either by
71 * aligning the destination and then using shift-and-write method, or in a few
72 * cases by falling back to a byte-at-a-time copy.
73 *
74 * Testing with various alignments and buffer sizes shows that this code is
75 * often >10x faster than a simple byte-at-a-time copy, even for strangely
76 * aligned operands. It is interesting to note that the glibc version of memcpy
77 * (written in C) is actually quite fast already. This routine is able to beat
78 * it by 30-40% for aligned copies because of the loop unrolling, but in some
79 * cases the glibc version is still slightly faster. This lends more
80 * credibility that gcc can generate very good code as long as we are careful.
81 *
82 * Possible optimizations:
83 * - add cache prefetching
84 * - try not to use the post-increment address modifiers; they may create
85 *   additional interlocks. Assumption is that those were only efficient on old
86 *   machines (pre PA8000 processors)
87 */
88
89	dst = arg0
90	src = arg1
91	len = arg2
92	end = arg3
93	t1  = r19
94	t2  = r20
95	t3  = r21
96	t4  = r22
97	srcspc = sr1
98	dstspc = sr2
99
100	t0 = r1
101	a1 = t1
102	a2 = t2
103	a3 = t3
104	a0 = t4
105
106	save_src = ret0
107	save_dst = ret1
108	save_len = r31
109
110ENTRY_CFI(pa_memcpy)
111	/* Last destination address */
112	add	dst,len,end
113
114	/* short copy with less than 16 bytes? */
115	cmpib,COND(>>=),n 15,len,.Lbyte_loop
116
117	/* same alignment? */
118	xor	src,dst,t0
119	extru	t0,31,2,t1
120	cmpib,<>,n  0,t1,.Lunaligned_copy
121
122#ifdef CONFIG_64BIT
123	/* only do 64-bit copies if we can get aligned. */
124	extru	t0,31,3,t1
125	cmpib,<>,n  0,t1,.Lalign_loop32
126
127	/* loop until we are 64-bit aligned */
128.Lalign_loop64:
129	extru	dst,31,3,t1
130	cmpib,=,n	0,t1,.Lcopy_loop_16_start
13120:	ldb,ma	1(srcspc,src),t1
13221:	stb,ma	t1,1(dstspc,dst)
133	b	.Lalign_loop64
134	ldo	-1(len),len
135
136	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
137	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
138
139.Lcopy_loop_16_start:
140	ldi	31,t0
141.Lcopy_loop_16:
142	cmpb,COND(>>=),n t0,len,.Lword_loop
143
14410:	ldd	0(srcspc,src),t1
14511:	ldd	8(srcspc,src),t2
146	ldo	16(src),src
14712:	std,ma	t1,8(dstspc,dst)
14813:	std,ma	t2,8(dstspc,dst)
14914:	ldd	0(srcspc,src),t1
15015:	ldd	8(srcspc,src),t2
151	ldo	16(src),src
15216:	std,ma	t1,8(dstspc,dst)
15317:	std,ma	t2,8(dstspc,dst)
154
155	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
156	ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy16_fault)
157	ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
158	ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
159	ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
160	ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy16_fault)
161	ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
162	ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
163
164	b	.Lcopy_loop_16
165	ldo	-32(len),len
166
167.Lword_loop:
168	cmpib,COND(>>=),n 3,len,.Lbyte_loop
16920:	ldw,ma	4(srcspc,src),t1
17021:	stw,ma	t1,4(dstspc,dst)
171	b	.Lword_loop
172	ldo	-4(len),len
173
174	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
175	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
176
177#endif /* CONFIG_64BIT */
178
179	/* loop until we are 32-bit aligned */
180.Lalign_loop32:
181	extru	dst,31,2,t1
182	cmpib,=,n	0,t1,.Lcopy_loop_8
18320:	ldb,ma	1(srcspc,src),t1
18421:	stb,ma	t1,1(dstspc,dst)
185	b	.Lalign_loop32
186	ldo	-1(len),len
187
188	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
189	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
190
191
192.Lcopy_loop_8:
193	cmpib,COND(>>=),n 15,len,.Lbyte_loop
194
19510:	ldw	0(srcspc,src),t1
19611:	ldw	4(srcspc,src),t2
19712:	stw,ma	t1,4(dstspc,dst)
19813:	stw,ma	t2,4(dstspc,dst)
19914:	ldw	8(srcspc,src),t1
20015:	ldw	12(srcspc,src),t2
201	ldo	16(src),src
20216:	stw,ma	t1,4(dstspc,dst)
20317:	stw,ma	t2,4(dstspc,dst)
204
205	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
206	ASM_EXCEPTIONTABLE_ENTRY(11b,.Lcopy8_fault)
207	ASM_EXCEPTIONTABLE_ENTRY(12b,.Lcopy_done)
208	ASM_EXCEPTIONTABLE_ENTRY(13b,.Lcopy_done)
209	ASM_EXCEPTIONTABLE_ENTRY(14b,.Lcopy_done)
210	ASM_EXCEPTIONTABLE_ENTRY(15b,.Lcopy8_fault)
211	ASM_EXCEPTIONTABLE_ENTRY(16b,.Lcopy_done)
212	ASM_EXCEPTIONTABLE_ENTRY(17b,.Lcopy_done)
213
214	b	.Lcopy_loop_8
215	ldo	-16(len),len
216
217.Lbyte_loop:
218	cmpclr,COND(<>) len,%r0,%r0
219	b,n	.Lcopy_done
22020:	ldb	0(srcspc,src),t1
221	ldo	1(src),src
22221:	stb,ma	t1,1(dstspc,dst)
223	b	.Lbyte_loop
224	ldo	-1(len),len
225
226	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
227	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
228
229.Lcopy_done:
230	bv	%r0(%r2)
231	sub	end,dst,ret0
232
233
234	/* src and dst are not aligned the same way. */
235	/* need to go the hard way */
236.Lunaligned_copy:
237	/* align until dst is 32bit-word-aligned */
238	extru	dst,31,2,t1
239	cmpib,=,n	0,t1,.Lcopy_dstaligned
24020:	ldb	0(srcspc,src),t1
241	ldo	1(src),src
24221:	stb,ma	t1,1(dstspc,dst)
243	b	.Lunaligned_copy
244	ldo	-1(len),len
245
246	ASM_EXCEPTIONTABLE_ENTRY(20b,.Lcopy_done)
247	ASM_EXCEPTIONTABLE_ENTRY(21b,.Lcopy_done)
248
249.Lcopy_dstaligned:
250
251	/* store src, dst and len in safe place */
252	copy	src,save_src
253	copy	dst,save_dst
254	copy	len,save_len
255
256	/* len now needs give number of words to copy */
257	SHRREG	len,2,len
258
259	/*
260	 * Copy from a not-aligned src to an aligned dst using shifts.
261	 * Handles 4 words per loop.
262	 */
263
264	depw,z src,28,2,t0
265	subi 32,t0,t0
266	mtsar t0
267	extru len,31,2,t0
268	cmpib,= 2,t0,.Lcase2
269	/* Make src aligned by rounding it down.  */
270	depi 0,31,2,src
271
272	cmpiclr,<> 3,t0,%r0
273	b,n .Lcase3
274	cmpiclr,<> 1,t0,%r0
275	b,n .Lcase1
276.Lcase0:
277	cmpb,COND(=) %r0,len,.Lcda_finish
278	nop
279
2801:	ldw,ma 4(srcspc,src), a3
281	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
2821:	ldw,ma 4(srcspc,src), a0
283	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
284	b,n .Ldo3
285.Lcase1:
2861:	ldw,ma 4(srcspc,src), a2
287	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
2881:	ldw,ma 4(srcspc,src), a3
289	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
290	ldo -1(len),len
291	cmpb,COND(=),n %r0,len,.Ldo0
292.Ldo4:
2931:	ldw,ma 4(srcspc,src), a0
294	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
295	shrpw a2, a3, %sar, t0
2961:	stw,ma t0, 4(dstspc,dst)
297	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
298.Ldo3:
2991:	ldw,ma 4(srcspc,src), a1
300	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
301	shrpw a3, a0, %sar, t0
3021:	stw,ma t0, 4(dstspc,dst)
303	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
304.Ldo2:
3051:	ldw,ma 4(srcspc,src), a2
306	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
307	shrpw a0, a1, %sar, t0
3081:	stw,ma t0, 4(dstspc,dst)
309	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
310.Ldo1:
3111:	ldw,ma 4(srcspc,src), a3
312	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
313	shrpw a1, a2, %sar, t0
3141:	stw,ma t0, 4(dstspc,dst)
315	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
316	ldo -4(len),len
317	cmpb,COND(<>) %r0,len,.Ldo4
318	nop
319.Ldo0:
320	shrpw a2, a3, %sar, t0
3211:	stw,ma t0, 4(dstspc,dst)
322	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcopy_done)
323
324.Lcda_rdfault:
325.Lcda_finish:
326	/* calculate new src, dst and len and jump to byte-copy loop */
327	sub	dst,save_dst,t0
328	add	save_src,t0,src
329	b	.Lbyte_loop
330	sub	save_len,t0,len
331
332.Lcase3:
3331:	ldw,ma 4(srcspc,src), a0
334	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
3351:	ldw,ma 4(srcspc,src), a1
336	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
337	b .Ldo2
338	ldo 1(len),len
339.Lcase2:
3401:	ldw,ma 4(srcspc,src), a1
341	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
3421:	ldw,ma 4(srcspc,src), a2
343	ASM_EXCEPTIONTABLE_ENTRY(1b,.Lcda_rdfault)
344	b .Ldo1
345	ldo 2(len),len
346
347
348	/* fault exception fixup handlers: */
349#ifdef CONFIG_64BIT
350.Lcopy16_fault:
351	b	.Lcopy_done
35210:	std,ma	t1,8(dstspc,dst)
353	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
354#endif
355
356.Lcopy8_fault:
357	b	.Lcopy_done
35810:	stw,ma	t1,4(dstspc,dst)
359	ASM_EXCEPTIONTABLE_ENTRY(10b,.Lcopy_done)
360ENDPROC_CFI(pa_memcpy)
361
362	.end
363