1 /*
2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230)
3 *
4 * SPDX-License-Identifier: GPL-2.0-only
5 */
6
7 #pragma once
8
9 #include <config.h>
10 #include <types.h>
11 #include <plat/machine.h>
12 #include <arch/smp/ipi.h>
13
14 #ifdef ENABLE_SMP_SUPPORT
15 #define MAX_IPI_ARGS 3 /* Maximum number of parameters to remote function */
16
17 static volatile struct {
18 word_t count;
19 word_t globalsense;
20
21 PAD_TO_NEXT_CACHE_LN(sizeof(word_t) + sizeof(word_t));
22 } ipiSyncBarrier = {0}; /* IPI barrier for remote call synchronization */
23
24 static volatile word_t totalCoreBarrier; /* number of cores involved in IPI 'in progress' */
25 static word_t ipi_args[MAX_IPI_ARGS]; /* data to be passed to the remote call function */
26
get_ipi_arg(word_t n)27 static inline word_t get_ipi_arg(word_t n)
28 {
29 assert(n < MAX_IPI_ARGS);
30 return ipi_args[n];
31 }
32
ipi_wait(word_t cores)33 static inline void ipi_wait(word_t cores)
34 {
35 word_t localsense = ipiSyncBarrier.globalsense;
36
37 if (__atomic_fetch_add(&ipiSyncBarrier.count, 1, __ATOMIC_ACQ_REL) == cores) {
38 ipiSyncBarrier.count = 0;
39 ipiSyncBarrier.globalsense =
40 ~ipiSyncBarrier.globalsense;
41 }
42
43 while (localsense == ipiSyncBarrier.globalsense) {
44 arch_pause();
45 }
46 }
47
48 /* Architecture independent function for sending handling pre-hardware-send IPIs */
49 void generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking);
50
51 /* An architecture/platform should implement this function either as a wrapper to
52 * its own arch_ipi_send_mask() or using the generic_ipi_send_mask() function
53 * provided to be architecture agnostic.
54 */
55 void ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking);
56
57 /* Hardware implementation for sending IPIs */
58 void ipi_send_target(irq_t irq, word_t cpuTargetList);
59
60 /* This function switches the core it is called on to the idle thread,
61 * in order to avoid IPI storms. If the core is waiting on the lock, the actual
62 * switch will not occur until the core attempts to obtain the lock, at which
63 * point the core will capture the pending IPI, which is discarded.
64
65 * The core who triggered the store is responsible for triggering a reschedule,
66 * or this call will idle forever */
67 void ipiStallCoreCallback(bool_t irqPath);
68
69 /* IPIs could be handled, both using hardware interrupts and software flag
70 * in CLH lock. 'irqPath' is used to differentiate the caller path, i.e.
71 * if it is called while waiting on the lock to handle the IRQ or not. The
72 * remote call handler, would decide if 'handleIPI' should return base
73 * on this value, as IRQs could be re/triggered asynchronous */
74 void handleIPI(irq_t irq, bool_t irqPath);
75
76 /*
77 * Run a synchronous function on all cores specified by mask. Return when target cores
78 * have all executed the function. Caller must hold the lock.
79 *
80 * @param func the function to run
81 * @param data1 passed to the function as first parameter
82 * @param data2 passed to the function as second parameter
83 * @param mask cores to run function on
84 */
85 void doRemoteMaskOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask);
86
87 /* Run a synchronous function on a core specified by cpu.
88 *
89 * @param func the function to run
90 * @param data1 passed to the function as first parameter
91 * @param data2 passed to the function as second parameter
92 * @param cpu core to run function on
93 */
doRemoteOp(IpiRemoteCall_t func,word_t data1,word_t data2,word_t data3,word_t cpu)94 static void inline doRemoteOp(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu)
95 {
96 doRemoteMaskOp(func, data1, data2, data3, BIT(cpu));
97 }
98
99 /* List of wrapper functions
100 *
101 * doRemote[Mask]Op0Arg: do remote operation without any argument
102 * doRemote[Mask]Op1Arg: do remote operation with one argument
103 * doRemote[Mask]Op2Arg: do remote operation with two arguments
104 * These should be used in favour of directly calling 'doRemote[Mask]Op'
105 * in case arguments change in future.
106 *
107 * @param func the function to run
108 * @param data passed to the function as parameters
109 * @param cpu[mask] cores to run function on
110 */
doRemoteMaskOp0Arg(IpiRemoteCall_t func,word_t mask)111 static void inline doRemoteMaskOp0Arg(IpiRemoteCall_t func, word_t mask)
112 {
113 doRemoteMaskOp(func, 0, 0, 0, mask);
114 }
115
doRemoteMaskOp1Arg(IpiRemoteCall_t func,word_t data1,word_t mask)116 static void inline doRemoteMaskOp1Arg(IpiRemoteCall_t func, word_t data1, word_t mask)
117 {
118 doRemoteMaskOp(func, data1, 0, 0, mask);
119 }
120
doRemoteMaskOp2Arg(IpiRemoteCall_t func,word_t data1,word_t data2,word_t mask)121 static void inline doRemoteMaskOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t mask)
122 {
123 doRemoteMaskOp(func, data1, data2, 0, mask);
124 }
125
doRemoteMaskOp3Arg(IpiRemoteCall_t func,word_t data1,word_t data2,word_t data3,word_t mask)126 static void inline doRemoteMaskOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t mask)
127 {
128 doRemoteMaskOp(func, data1, data2, data3, mask);
129 }
130
doRemoteOp0Arg(IpiRemoteCall_t func,word_t cpu)131 static void inline doRemoteOp0Arg(IpiRemoteCall_t func, word_t cpu)
132 {
133 doRemoteOp(func, 0, 0, 0, cpu);
134 }
135
doRemoteOp1Arg(IpiRemoteCall_t func,word_t data1,word_t cpu)136 static void inline doRemoteOp1Arg(IpiRemoteCall_t func, word_t data1, word_t cpu)
137 {
138 doRemoteOp(func, data1, 0, 0, cpu);
139 }
140
doRemoteOp2Arg(IpiRemoteCall_t func,word_t data1,word_t data2,word_t cpu)141 static void inline doRemoteOp2Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t cpu)
142 {
143 doRemoteOp(func, data1, data2, 0, cpu);
144 }
145
doRemoteOp3Arg(IpiRemoteCall_t func,word_t data1,word_t data2,word_t data3,word_t cpu)146 static void inline doRemoteOp3Arg(IpiRemoteCall_t func, word_t data1, word_t data2, word_t data3, word_t cpu)
147 {
148 doRemoteOp(func, data1, data2, data3, cpu);
149 }
150
151 /* This is asynchronous call and could be called outside the lock.
152 * Returns immediately.
153 *
154 * @param mask cores to request rescheduling
155 */
156 void doMaskReschedule(word_t mask);
157
158 /* Request rescheduling on a core specified by cpu.
159 * Returns immediately.
160 *
161 * @param cpu core to reschedule
162 */
doReschedule(word_t cpu)163 static void inline doReschedule(word_t cpu)
164 {
165 if (cpu != getCurrentCPUIndex()) {
166 assert(cpu < CONFIG_MAX_NUM_NODES);
167 doMaskReschedule(BIT(cpu));
168 }
169 }
170
171 #endif /* ENABLE_SMP_SUPPORT */
172
173