1/*
2 * Copyright (c) 2019 Carlo Caione <ccaione@baylibre.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7/*
8 * ARM64 Cortex-A ISRs wrapper
9 */
10
11#include <zephyr/toolchain.h>
12#include <zephyr/linker/sections.h>
13#include <offsets_short.h>
14#include <zephyr/arch/cpu.h>
15#include <zephyr/sw_isr_table.h>
16#include <zephyr/drivers/interrupt_controller/gic.h>
17#include "macro_priv.inc"
18
19_ASM_FILE_PROLOGUE
20
21GDATA(_sw_isr_table)
22
23/*
24 * Wrapper around ISRs when inserted in software ISR table
25 *
26 * When inserted in the vector table, _isr_wrapper() demuxes the ISR table
27 * using the running interrupt number as the index, and invokes the registered
28 * ISR with its corresponding argument. When returning from the ISR, it
29 * determines if a context switch needs to happen.
30 */
31
32GTEXT(_isr_wrapper)
33SECTION_FUNC(TEXT, _isr_wrapper)
34
35	/* ++_current_cpu->nested to be checked by arch_is_in_isr() */
36	get_cpu	x0
37	ldr	w1, [x0, #___cpu_t_nested_OFFSET]
38	add	w2, w1, #1
39	str	w2, [x0, #___cpu_t_nested_OFFSET]
40
41	/* If not nested: switch to IRQ stack and save current sp on it. */
42	cbnz	w1, 1f
43	ldr	x1, [x0, #___cpu_t_irq_stack_OFFSET]
44	mov	x2, sp
45	mov	sp, x1
46	str	x2, [sp, #-16]!
47#if defined(CONFIG_ARM64_SAFE_EXCEPTION_STACK)
48	sub	x1, x1, #CONFIG_ISR_STACK_SIZE
49	str	x1, [x0, #_cpu_offset_to_current_stack_limit]
50#endif
511:
52#ifdef CONFIG_SCHED_THREAD_USAGE
53	bl	z_sched_usage_stop
54#endif
55
56#ifdef CONFIG_TRACING
57	bl	sys_trace_isr_enter
58#endif
59
60	/* Get active IRQ number from the interrupt controller */
61#if !defined(CONFIG_ARM_CUSTOM_INTERRUPT_CONTROLLER)
62	bl	arm_gic_get_active
63#else
64	bl	z_soc_irq_get_active
65#endif /* !CONFIG_ARM_CUSTOM_INTERRUPT_CONTROLLER */
66
67	/* Preserve original IAR value */
68	str	x0, [sp, #-16]!
69
70#if CONFIG_GIC_VER == 2 && defined(CONFIG_SMP)
71	/* Mask out GICC_IAR.CPUID [12:10] */
72	bic	x0, x0, #0x1c00
73#endif
74
75#if CONFIG_GIC_VER >= 3
76	/*
77	 * Ignore Special INTIDs 1020..1023 see 2.2.1 of Arm Generic Interrupt Controller
78	 * Architecture Specification GIC architecture version 3 and version 4
79	 */
80	cmp	x0, 1019
81	b.le	oob
82	cmp	x0, 1023
83	b.gt	oob
84	b	spurious_continue
85
86oob:
87#endif
88	/* IRQ out of bounds */
89	mov	x1, #(CONFIG_NUM_IRQS - 1)
90	cmp	x0, x1
91	b.hi	spurious_continue
92
93	/* Retrieve the interrupt service routine */
94	ldr	x1, =_sw_isr_table
95	add	x1, x1, x0, lsl #4	/* table is 16-byte wide */
96	ldp	x0, x3, [x1] /* arg in x0, ISR in x3 */
97
98	/*
99	 * Call the ISR. Unmask and mask again the IRQs to support nested
100	 * exception handlers
101	 */
102	msr	daifclr, #(DAIFCLR_IRQ_BIT)
103	blr	x3
104	msr	daifset, #(DAIFSET_IRQ_BIT)
105
106spurious_continue:
107
108	/* Retrieve original IAR value */
109	ldr	x0, [sp], #16
110
111	/* Signal end-of-interrupt */
112#if !defined(CONFIG_ARM_CUSTOM_INTERRUPT_CONTROLLER)
113	bl	arm_gic_eoi
114#else
115	bl	z_soc_irq_eoi
116#endif /* !CONFIG_ARM_CUSTOM_INTERRUPT_CONTROLLER */
117
118#ifdef CONFIG_TRACING
119	bl	sys_trace_isr_exit
120#endif
121
122GTEXT(z_arm64_irq_done)
123z_arm64_irq_done:
124	/* if (--_current_cpu->nested != 0) exit */
125	get_cpu	x0
126	ldr	w1, [x0, #___cpu_t_nested_OFFSET]
127	subs	w1, w1, #1
128	str	w1, [x0, #___cpu_t_nested_OFFSET]
129	bne	exit
130
131	/* No more nested: retrieve the task's stack. */
132	ldr	x1, [sp]
133	mov	sp, x1
134
135	/* retrieve pointer to the current thread */
136	ldr	x1, [x0, #___cpu_t_current_OFFSET]
137
138#if defined(CONFIG_ARM64_SAFE_EXCEPTION_STACK)
139	/* arch_curr_cpu()->arch.current_stack_limit = thread->arch.stack_limit */
140	ldr	x2, [x1, #_thread_offset_to_stack_limit]
141	str	x2, [x0, #_cpu_offset_to_current_stack_limit]
142#endif
143
144	/*
145	 * Get next thread to schedule with z_get_next_switch_handle().
146	 * We pass it a NULL as we didn't save the whole thread context yet.
147	 * If no scheduling is necessary then NULL will be returned.
148	 */
149	str	x1, [sp, #-16]!
150	mov	x0, xzr
151	bl	z_get_next_switch_handle
152	ldr	x1, [sp], #16
153	cbz	x0, exit
154
155	/*
156	 * Switch thread
157	 * x0: new thread
158	 * x1: old thread
159	 */
160	bl	z_arm64_context_switch
161
162exit:
163#ifdef CONFIG_STACK_SENTINEL
164	bl	z_check_stack_sentinel
165#endif
166	b	z_arm64_exit_exc
167
168