1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5#include <asm/asm.h>
6#include <asm/export.h>
7#include <asm/loongarch.h>
8#include <asm/page.h>
9#include <asm/pgtable.h>
10#include <asm/regdef.h>
11#include <asm/stackframe.h>
12
13#define INVTLB_ADDR_GFALSE_AND_ASID	5
14
15#define PTRS_PER_PGD_BITS	(PAGE_SHIFT - 3)
16#define PTRS_PER_PUD_BITS	(PAGE_SHIFT - 3)
17#define PTRS_PER_PMD_BITS	(PAGE_SHIFT - 3)
18#define PTRS_PER_PTE_BITS	(PAGE_SHIFT - 3)
19
20	.macro tlb_do_page_fault, write
21	SYM_FUNC_START(tlb_do_page_fault_\write)
22	SAVE_ALL
23	csrrd		a2, LOONGARCH_CSR_BADV
24	move		a0, sp
25	REG_S		a2, sp, PT_BVADDR
26	li.w		a1, \write
27	bl		do_page_fault
28	RESTORE_ALL_AND_RET
29	SYM_FUNC_END(tlb_do_page_fault_\write)
30	.endm
31
32	tlb_do_page_fault 0
33	tlb_do_page_fault 1
34
35SYM_FUNC_START(handle_tlb_protect)
36	BACKUP_T0T1
37	SAVE_ALL
38	move		a0, sp
39	move		a1, zero
40	csrrd		a2, LOONGARCH_CSR_BADV
41	REG_S		a2, sp, PT_BVADDR
42	la_abs		t0, do_page_fault
43	jirl		ra, t0, 0
44	RESTORE_ALL_AND_RET
45SYM_FUNC_END(handle_tlb_protect)
46
47SYM_FUNC_START(handle_tlb_load)
48	csrwr		t0, EXCEPTION_KS0
49	csrwr		t1, EXCEPTION_KS1
50	csrwr		ra, EXCEPTION_KS2
51
52	/*
53	 * The vmalloc handling is not in the hotpath.
54	 */
55	csrrd		t0, LOONGARCH_CSR_BADV
56	bltz		t0, vmalloc_load
57	csrrd		t1, LOONGARCH_CSR_PGDL
58
59vmalloc_done_load:
60	/* Get PGD offset in bytes */
61	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
62	alsl.d		t1, ra, t1, 3
63#if CONFIG_PGTABLE_LEVELS > 3
64	ld.d		t1, t1, 0
65	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
66	alsl.d		t1, ra, t1, 3
67#endif
68#if CONFIG_PGTABLE_LEVELS > 2
69	ld.d		t1, t1, 0
70	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
71	alsl.d		t1, ra, t1, 3
72#endif
73	ld.d		ra, t1, 0
74
75	/*
76	 * For huge tlb entries, pmde doesn't contain an address but
77	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
78	 * see if we need to jump to huge tlb processing.
79	 */
80	rotri.d		ra, ra, _PAGE_HUGE_SHIFT + 1
81	bltz		ra, tlb_huge_update_load
82
83	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
84	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
85	alsl.d		t1, t0, ra, _PTE_T_LOG2
86
87#ifdef CONFIG_SMP
88smp_pgtable_change_load:
89	ll.d		t0, t1, 0
90#else
91	ld.d		t0, t1, 0
92#endif
93	andi		ra, t0, _PAGE_PRESENT
94	beqz		ra, nopage_tlb_load
95
96	ori		t0, t0, _PAGE_VALID
97#ifdef CONFIG_SMP
98	sc.d		t0, t1, 0
99	beqz		t0, smp_pgtable_change_load
100#else
101	st.d		t0, t1, 0
102#endif
103	tlbsrch
104	bstrins.d	t1, zero, 3, 3
105	ld.d		t0, t1, 0
106	ld.d		t1, t1, 8
107	csrwr		t0, LOONGARCH_CSR_TLBELO0
108	csrwr		t1, LOONGARCH_CSR_TLBELO1
109	tlbwr
110
111	csrrd		t0, EXCEPTION_KS0
112	csrrd		t1, EXCEPTION_KS1
113	csrrd		ra, EXCEPTION_KS2
114	ertn
115
116#ifdef CONFIG_64BIT
117vmalloc_load:
118	la_abs		t1, swapper_pg_dir
119	b		vmalloc_done_load
120#endif
121
122	/* This is the entry point of a huge page. */
123tlb_huge_update_load:
124#ifdef CONFIG_SMP
125	ll.d		ra, t1, 0
126#endif
127	andi		t0, ra, _PAGE_PRESENT
128	beqz		t0, nopage_tlb_load
129
130#ifdef CONFIG_SMP
131	ori		t0, ra, _PAGE_VALID
132	sc.d		t0, t1, 0
133	beqz		t0, tlb_huge_update_load
134	ori		t0, ra, _PAGE_VALID
135#else
136	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
137	ori		t0, ra, _PAGE_VALID
138	st.d		t0, t1, 0
139#endif
140	csrrd		ra, LOONGARCH_CSR_ASID
141	csrrd		t1, LOONGARCH_CSR_BADV
142	andi		ra, ra, CSR_ASID_ASID
143	invtlb		INVTLB_ADDR_GFALSE_AND_ASID, ra, t1
144
145	/*
146	 * A huge PTE describes an area the size of the
147	 * configured huge page size. This is twice the
148	 * of the large TLB entry size we intend to use.
149	 * A TLB entry half the size of the configured
150	 * huge page size is configured into entrylo0
151	 * and entrylo1 to cover the contiguous huge PTE
152	 * address space.
153	 */
154	/* Huge page: Move Global bit */
155	xori		t0, t0, _PAGE_HUGE
156	lu12i.w		t1, _PAGE_HGLOBAL >> 12
157	and		t1, t0, t1
158	srli.d		t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
159	or		t0, t0, t1
160
161	move		ra, t0
162	csrwr		ra, LOONGARCH_CSR_TLBELO0
163
164	/* Convert to entrylo1 */
165	addi.d		t1, zero, 1
166	slli.d		t1, t1, (HPAGE_SHIFT - 1)
167	add.d		t0, t0, t1
168	csrwr		t0, LOONGARCH_CSR_TLBELO1
169
170	/* Set huge page tlb entry size */
171	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
172	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
173	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
174
175	tlbfill
176
177	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
178	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
179	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
180
181	csrrd		t0, EXCEPTION_KS0
182	csrrd		t1, EXCEPTION_KS1
183	csrrd		ra, EXCEPTION_KS2
184	ertn
185
186nopage_tlb_load:
187	dbar		0
188	csrrd		ra, EXCEPTION_KS2
189	la_abs		t0, tlb_do_page_fault_0
190	jr		t0
191SYM_FUNC_END(handle_tlb_load)
192
193SYM_FUNC_START(handle_tlb_store)
194	csrwr		t0, EXCEPTION_KS0
195	csrwr		t1, EXCEPTION_KS1
196	csrwr		ra, EXCEPTION_KS2
197
198	/*
199	 * The vmalloc handling is not in the hotpath.
200	 */
201	csrrd		t0, LOONGARCH_CSR_BADV
202	bltz		t0, vmalloc_store
203	csrrd		t1, LOONGARCH_CSR_PGDL
204
205vmalloc_done_store:
206	/* Get PGD offset in bytes */
207	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
208	alsl.d		t1, ra, t1, 3
209#if CONFIG_PGTABLE_LEVELS > 3
210	ld.d		t1, t1, 0
211	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
212	alsl.d		t1, ra, t1, 3
213#endif
214#if CONFIG_PGTABLE_LEVELS > 2
215	ld.d		t1, t1, 0
216	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
217	alsl.d		t1, ra, t1, 3
218#endif
219	ld.d		ra, t1, 0
220
221	/*
222	 * For huge tlb entries, pmde doesn't contain an address but
223	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
224	 * see if we need to jump to huge tlb processing.
225	 */
226	rotri.d		ra, ra, _PAGE_HUGE_SHIFT + 1
227	bltz		ra, tlb_huge_update_store
228
229	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
230	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
231	alsl.d		t1, t0, ra, _PTE_T_LOG2
232
233#ifdef CONFIG_SMP
234smp_pgtable_change_store:
235	ll.d		t0, t1, 0
236#else
237	ld.d		t0, t1, 0
238#endif
239	andi		ra, t0, _PAGE_PRESENT | _PAGE_WRITE
240	xori		ra, ra, _PAGE_PRESENT | _PAGE_WRITE
241	bnez		ra, nopage_tlb_store
242
243	ori		t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
244#ifdef CONFIG_SMP
245	sc.d		t0, t1, 0
246	beqz		t0, smp_pgtable_change_store
247#else
248	st.d		t0, t1, 0
249#endif
250	tlbsrch
251	bstrins.d	t1, zero, 3, 3
252	ld.d		t0, t1, 0
253	ld.d		t1, t1, 8
254	csrwr		t0, LOONGARCH_CSR_TLBELO0
255	csrwr		t1, LOONGARCH_CSR_TLBELO1
256	tlbwr
257
258	csrrd		t0, EXCEPTION_KS0
259	csrrd		t1, EXCEPTION_KS1
260	csrrd		ra, EXCEPTION_KS2
261	ertn
262
263#ifdef CONFIG_64BIT
264vmalloc_store:
265	la_abs		t1, swapper_pg_dir
266	b		vmalloc_done_store
267#endif
268
269	/* This is the entry point of a huge page. */
270tlb_huge_update_store:
271#ifdef CONFIG_SMP
272	ll.d		ra, t1, 0
273#endif
274	andi		t0, ra, _PAGE_PRESENT | _PAGE_WRITE
275	xori		t0, t0, _PAGE_PRESENT | _PAGE_WRITE
276	bnez		t0, nopage_tlb_store
277
278#ifdef CONFIG_SMP
279	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
280	sc.d		t0, t1, 0
281	beqz		t0, tlb_huge_update_store
282	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
283#else
284	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
285	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
286	st.d		t0, t1, 0
287#endif
288	csrrd		ra, LOONGARCH_CSR_ASID
289	csrrd		t1, LOONGARCH_CSR_BADV
290	andi		ra, ra, CSR_ASID_ASID
291	invtlb		INVTLB_ADDR_GFALSE_AND_ASID, ra, t1
292
293	/*
294	 * A huge PTE describes an area the size of the
295	 * configured huge page size. This is twice the
296	 * of the large TLB entry size we intend to use.
297	 * A TLB entry half the size of the configured
298	 * huge page size is configured into entrylo0
299	 * and entrylo1 to cover the contiguous huge PTE
300	 * address space.
301	 */
302	/* Huge page: Move Global bit */
303	xori		t0, t0, _PAGE_HUGE
304	lu12i.w		t1, _PAGE_HGLOBAL >> 12
305	and		t1, t0, t1
306	srli.d		t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
307	or		t0, t0, t1
308
309	move		ra, t0
310	csrwr		ra, LOONGARCH_CSR_TLBELO0
311
312	/* Convert to entrylo1 */
313	addi.d		t1, zero, 1
314	slli.d		t1, t1, (HPAGE_SHIFT - 1)
315	add.d		t0, t0, t1
316	csrwr		t0, LOONGARCH_CSR_TLBELO1
317
318	/* Set huge page tlb entry size */
319	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
320	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
321	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
322
323	tlbfill
324
325	/* Reset default page size */
326	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
327	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
328	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
329
330	csrrd		t0, EXCEPTION_KS0
331	csrrd		t1, EXCEPTION_KS1
332	csrrd		ra, EXCEPTION_KS2
333	ertn
334
335nopage_tlb_store:
336	dbar		0
337	csrrd		ra, EXCEPTION_KS2
338	la_abs		t0, tlb_do_page_fault_1
339	jr		t0
340SYM_FUNC_END(handle_tlb_store)
341
342SYM_FUNC_START(handle_tlb_modify)
343	csrwr		t0, EXCEPTION_KS0
344	csrwr		t1, EXCEPTION_KS1
345	csrwr		ra, EXCEPTION_KS2
346
347	/*
348	 * The vmalloc handling is not in the hotpath.
349	 */
350	csrrd		t0, LOONGARCH_CSR_BADV
351	bltz		t0, vmalloc_modify
352	csrrd		t1, LOONGARCH_CSR_PGDL
353
354vmalloc_done_modify:
355	/* Get PGD offset in bytes */
356	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
357	alsl.d		t1, ra, t1, 3
358#if CONFIG_PGTABLE_LEVELS > 3
359	ld.d		t1, t1, 0
360	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
361	alsl.d		t1, ra, t1, 3
362#endif
363#if CONFIG_PGTABLE_LEVELS > 2
364	ld.d		t1, t1, 0
365	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
366	alsl.d		t1, ra, t1, 3
367#endif
368	ld.d		ra, t1, 0
369
370	/*
371	 * For huge tlb entries, pmde doesn't contain an address but
372	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
373	 * see if we need to jump to huge tlb processing.
374	 */
375	rotri.d		ra, ra, _PAGE_HUGE_SHIFT + 1
376	bltz		ra, tlb_huge_update_modify
377
378	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
379	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
380	alsl.d		t1, t0, ra, _PTE_T_LOG2
381
382#ifdef CONFIG_SMP
383smp_pgtable_change_modify:
384	ll.d		t0, t1, 0
385#else
386	ld.d		t0, t1, 0
387#endif
388	andi		ra, t0, _PAGE_WRITE
389	beqz		ra, nopage_tlb_modify
390
391	ori		t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
392#ifdef CONFIG_SMP
393	sc.d		t0, t1, 0
394	beqz		t0, smp_pgtable_change_modify
395#else
396	st.d		t0, t1, 0
397#endif
398	tlbsrch
399	bstrins.d	t1, zero, 3, 3
400	ld.d		t0, t1, 0
401	ld.d		t1, t1, 8
402	csrwr		t0, LOONGARCH_CSR_TLBELO0
403	csrwr		t1, LOONGARCH_CSR_TLBELO1
404	tlbwr
405
406	csrrd		t0, EXCEPTION_KS0
407	csrrd		t1, EXCEPTION_KS1
408	csrrd		ra, EXCEPTION_KS2
409	ertn
410
411#ifdef CONFIG_64BIT
412vmalloc_modify:
413	la_abs		t1, swapper_pg_dir
414	b		vmalloc_done_modify
415#endif
416
417	/* This is the entry point of a huge page. */
418tlb_huge_update_modify:
419#ifdef CONFIG_SMP
420	ll.d		ra, t1, 0
421#endif
422	andi		t0, ra, _PAGE_WRITE
423	beqz		t0, nopage_tlb_modify
424
425#ifdef CONFIG_SMP
426	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
427	sc.d		t0, t1, 0
428	beqz		t0, tlb_huge_update_modify
429	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
430#else
431	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
432	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
433	st.d		t0, t1, 0
434#endif
435	csrrd		ra, LOONGARCH_CSR_ASID
436	csrrd		t1, LOONGARCH_CSR_BADV
437	andi		ra, ra, CSR_ASID_ASID
438	invtlb		INVTLB_ADDR_GFALSE_AND_ASID, ra, t1
439
440	/*
441	 * A huge PTE describes an area the size of the
442	 * configured huge page size. This is twice the
443	 * of the large TLB entry size we intend to use.
444	 * A TLB entry half the size of the configured
445	 * huge page size is configured into entrylo0
446	 * and entrylo1 to cover the contiguous huge PTE
447	 * address space.
448	 */
449	/* Huge page: Move Global bit */
450	xori		t0, t0, _PAGE_HUGE
451	lu12i.w		t1, _PAGE_HGLOBAL >> 12
452	and		t1, t0, t1
453	srli.d		t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
454	or		t0, t0, t1
455
456	move		ra, t0
457	csrwr		ra, LOONGARCH_CSR_TLBELO0
458
459	/* Convert to entrylo1 */
460	addi.d		t1, zero, 1
461	slli.d		t1, t1, (HPAGE_SHIFT - 1)
462	add.d		t0, t0, t1
463	csrwr		t0, LOONGARCH_CSR_TLBELO1
464
465	/* Set huge page tlb entry size */
466	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
467	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
468	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
469
470	tlbfill
471
472	/* Reset default page size */
473	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
474	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
475	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
476
477	csrrd		t0, EXCEPTION_KS0
478	csrrd		t1, EXCEPTION_KS1
479	csrrd		ra, EXCEPTION_KS2
480	ertn
481
482nopage_tlb_modify:
483	dbar		0
484	csrrd		ra, EXCEPTION_KS2
485	la_abs		t0, tlb_do_page_fault_1
486	jr		t0
487SYM_FUNC_END(handle_tlb_modify)
488
489SYM_FUNC_START(handle_tlb_refill)
490	csrwr		t0, LOONGARCH_CSR_TLBRSAVE
491	csrrd		t0, LOONGARCH_CSR_PGD
492	lddir		t0, t0, 3
493#if CONFIG_PGTABLE_LEVELS > 3
494	lddir		t0, t0, 2
495#endif
496#if CONFIG_PGTABLE_LEVELS > 2
497	lddir		t0, t0, 1
498#endif
499	ldpte		t0, 0
500	ldpte		t0, 1
501	tlbfill
502	csrrd		t0, LOONGARCH_CSR_TLBRSAVE
503	ertn
504SYM_FUNC_END(handle_tlb_refill)
505