1 // © 2021 Qualcomm Innovation Center, Inc. All rights reserved.
2 //
3 // SPDX-License-Identifier: BSD-3-Clause
4
5 // CPU-local storage.
6 //
7 // This is a generic implementation for architectures which don't have a
8 // conveniently readable CPU index register, and must use a TLS variable
9 // that is updated on context-switch.
10
11 // Declarations and accessors for CPU-local storage.
12 //
13 // Note that variables declared this way must not be accessed if it is
14 // possible for the calling thread to be preempted and then migrate to a
15 // different CPU. To avoid this, this header provides macros that can be
16 // used to mark a critical section that accesses CPU-local variables.
17
18 #if SCHEDULER_CAN_MIGRATE
19 #define cpulocal_begin() preempt_disable()
20 #define cpulocal_end() preempt_enable()
21 #define assert_cpulocal_safe() assert_preempt_disabled()
22 #else
23 #define cpulocal_begin() ((void)0)
24 #define cpulocal_end() ((void)0)
25 #define assert_cpulocal_safe() ((void)0)
26 #endif
27
28 // Declarators for CPU-local data
29 #define CPULOCAL_DECLARE_EXTERN(type, name) \
30 extern type cpulocal_##name[PLATFORM_MAX_CORES]
31 #define CPULOCAL_DECLARE(type, name) type cpulocal_##name[PLATFORM_MAX_CORES]
32 #define CPULOCAL_DECLARE_STATIC(type, name) \
33 static type cpulocal_##name[PLATFORM_MAX_CORES]
34
35 // Accessors for CPU-local data
36 #define CPULOCAL(name) cpulocal_##name[cpulocal_get_index()]
37 #define CPULOCAL_BY_INDEX(name, index) \
38 cpulocal_##name[cpulocal_check_index(index)]
39
40 // Given a pointer to an object that is known to be an element of a given
41 // CPU-local data array, obtain its index. The behaviour is undefined (and
42 // violates MISRA rule 18.2) if the object is not a member of the specified
43 // array.
44 #define CPULOCAL_PTR_INDEX(name, ptr) \
45 (assert(ptr != NULL), \
46 cpulocal_check_index((cpu_index_t)(ptr - cpulocal_##name)))
47
48 // Return true if a CPU index is valid.
49 bool
50 cpulocal_index_valid(cpu_index_t index);
51
52 // Validate and return a CPU index.
53 //
54 // In debug kernels, this will assert that the index is in range. The input is
55 // returned unchanged.
56 cpu_index_t
57 cpulocal_check_index(cpu_index_t index);
58
59 // Get the CPU index of the caller at the instant of the call.
60 //
61 // This is the same as cpulocal_get_index(), except that it does not require
62 // preemption to be disabled. The result may therefore be stale by the time
63 // the caller gets it. The result should only be used for purposes where this
64 // does not matter, e.g. for debug traces.
65 //
66 // The result of this function is not checked and may be invalid if called
67 // very early in the boot sequence.
68 cpu_index_t
69 cpulocal_get_index_unsafe(void);
70
71 // Get the CPU index of the caller.
72 //
73 // All calls to this function should be inside a critical section, which may
74 // be either an explicit preemption disable, a spinlock, or a cpulocal
75 // critical section. All uses of its result should occur before the
76 // corresponding critical section ends.
77 static inline cpu_index_t
cpulocal_get_index(void)78 cpulocal_get_index(void) REQUIRE_PREEMPT_DISABLED
79 {
80 return cpulocal_check_index(cpulocal_get_index_unsafe());
81 }
82
83 // Get the CPU index of a specified thread.
84 //
85 // This will return a valid CPU ID if the thread is at any stage of execution on
86 // that CPU between the start of a thread_context_switch_pre event switching to
87 // the thread and the end of a thread_context_switch_post event switching away
88 // from the thread. If the thread is not running, it returns cpu_index_invalid.
89 //
90 // If the caller is not the specified thread, it should hold the scheduling lock
91 // for the thread, and all uses of its result should occur before that lock is
92 // released. If the caller is the specified thread, the same restrictions apply
93 // as for cpulocal_get_index().
94 cpu_index_t
95 cpulocal_get_index_for_thread(const thread_t *thread);
96