1 /*
2 * Copyright (c) 2016 Wind River Systems, Inc.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #include <zephyr/kernel.h>
8 #include <zephyr/kernel_structs.h>
9
10 #include <zephyr/toolchain.h>
11 #include <zephyr/linker/sections.h>
12 #include <zephyr/sys/dlist.h>
13 #include <zephyr/init.h>
14 #include <zephyr/sys/check.h>
15 #include <zephyr/sys/iterable_sections.h>
16 #include <string.h>
17 /* private kernel APIs */
18 #include <ksched.h>
19 #include <wait_q.h>
20
21 #ifdef CONFIG_OBJ_CORE_MEM_SLAB
22 static struct k_obj_type obj_type_mem_slab;
23
24 #ifdef CONFIG_OBJ_CORE_STATS_MEM_SLAB
25
k_mem_slab_stats_raw(struct k_obj_core * obj_core,void * stats)26 static int k_mem_slab_stats_raw(struct k_obj_core *obj_core, void *stats)
27 {
28 __ASSERT((obj_core != NULL) && (stats != NULL), "NULL parameter");
29
30 struct k_mem_slab *slab;
31 k_spinlock_key_t key;
32
33 slab = CONTAINER_OF(obj_core, struct k_mem_slab, obj_core);
34 key = k_spin_lock(&slab->lock);
35 memcpy(stats, &slab->info, sizeof(slab->info));
36 k_spin_unlock(&slab->lock, key);
37
38 return 0;
39 }
40
k_mem_slab_stats_query(struct k_obj_core * obj_core,void * stats)41 static int k_mem_slab_stats_query(struct k_obj_core *obj_core, void *stats)
42 {
43 __ASSERT((obj_core != NULL) && (stats != NULL), "NULL parameter");
44
45 struct k_mem_slab *slab;
46 k_spinlock_key_t key;
47 struct sys_memory_stats *ptr = stats;
48
49 slab = CONTAINER_OF(obj_core, struct k_mem_slab, obj_core);
50 key = k_spin_lock(&slab->lock);
51 ptr->free_bytes = (slab->info.num_blocks - slab->info.num_used) *
52 slab->info.block_size;
53 ptr->allocated_bytes = slab->info.num_used * slab->info.block_size;
54 #ifdef CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION
55 ptr->max_allocated_bytes = slab->info.max_used * slab->info.block_size;
56 #else
57 ptr->max_allocated_bytes = 0;
58 #endif /* CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION */
59 k_spin_unlock(&slab->lock, key);
60
61 return 0;
62 }
63
k_mem_slab_stats_reset(struct k_obj_core * obj_core)64 static int k_mem_slab_stats_reset(struct k_obj_core *obj_core)
65 {
66 __ASSERT(obj_core != NULL, "NULL parameter");
67
68 struct k_mem_slab *slab;
69 k_spinlock_key_t key;
70
71 slab = CONTAINER_OF(obj_core, struct k_mem_slab, obj_core);
72 key = k_spin_lock(&slab->lock);
73
74 #ifdef CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION
75 slab->info.max_used = slab->info.num_used;
76 #endif /* CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION */
77
78 k_spin_unlock(&slab->lock, key);
79
80 return 0;
81 }
82
83 static struct k_obj_core_stats_desc mem_slab_stats_desc = {
84 .raw_size = sizeof(struct k_mem_slab_info),
85 .query_size = sizeof(struct sys_memory_stats),
86 .raw = k_mem_slab_stats_raw,
87 .query = k_mem_slab_stats_query,
88 .reset = k_mem_slab_stats_reset,
89 .disable = NULL,
90 .enable = NULL,
91 };
92 #endif /* CONFIG_OBJ_CORE_STATS_MEM_SLAB */
93 #endif /* CONFIG_OBJ_CORE_MEM_SLAB */
94
95 /**
96 * @brief Initialize kernel memory slab subsystem.
97 *
98 * Perform any initialization of memory slabs that wasn't done at build time.
99 * Currently this just involves creating the list of free blocks for each slab.
100 *
101 * @retval 0 on success.
102 * @retval -EINVAL if @p slab contains invalid configuration and/or values.
103 */
create_free_list(struct k_mem_slab * slab)104 static int create_free_list(struct k_mem_slab *slab)
105 {
106 char *p;
107
108 /* blocks must be word aligned */
109 CHECKIF(((slab->info.block_size | (uintptr_t)slab->buffer) &
110 (sizeof(void *) - 1)) != 0U) {
111 return -EINVAL;
112 }
113
114 slab->free_list = NULL;
115 p = slab->buffer + slab->info.block_size * (slab->info.num_blocks - 1);
116
117 for (int i = slab->info.num_blocks - 1; i >= 0; i--) {
118 *(char **)p = slab->free_list;
119 slab->free_list = p;
120 p -= slab->info.block_size;
121 }
122
123 return 0;
124 }
125
126 /**
127 * @brief Complete initialization of statically defined memory slabs.
128 *
129 * Perform any initialization that wasn't done at build time.
130 *
131 * @return 0 on success, fails otherwise.
132 */
init_mem_slab_obj_core_list(void)133 static int init_mem_slab_obj_core_list(void)
134 {
135 int rc = 0;
136
137 /* Initialize mem_slab object type */
138
139 #ifdef CONFIG_OBJ_CORE_MEM_SLAB
140 z_obj_type_init(&obj_type_mem_slab, K_OBJ_TYPE_MEM_SLAB_ID,
141 offsetof(struct k_mem_slab, obj_core));
142 #ifdef CONFIG_OBJ_CORE_STATS_MEM_SLAB
143 k_obj_type_stats_init(&obj_type_mem_slab, &mem_slab_stats_desc);
144 #endif /* CONFIG_OBJ_CORE_STATS_MEM_SLAB */
145 #endif /* CONFIG_OBJ_CORE_MEM_SLAB */
146
147 /* Initialize statically defined mem_slabs */
148
149 STRUCT_SECTION_FOREACH(k_mem_slab, slab) {
150 rc = create_free_list(slab);
151 if (rc < 0) {
152 goto out;
153 }
154 k_object_init(slab);
155
156 #ifdef CONFIG_OBJ_CORE_MEM_SLAB
157 k_obj_core_init_and_link(K_OBJ_CORE(slab), &obj_type_mem_slab);
158 #ifdef CONFIG_OBJ_CORE_STATS_MEM_SLAB
159 k_obj_core_stats_register(K_OBJ_CORE(slab), &slab->info,
160 sizeof(struct k_mem_slab_info));
161 #endif /* CONFIG_OBJ_CORE_STATS_MEM_SLAB */
162 #endif /* CONFIG_OBJ_CORE_MEM_SLAB */
163 }
164
165 out:
166 return rc;
167 }
168
169 SYS_INIT(init_mem_slab_obj_core_list, PRE_KERNEL_1,
170 CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
171
k_mem_slab_init(struct k_mem_slab * slab,void * buffer,size_t block_size,uint32_t num_blocks)172 int k_mem_slab_init(struct k_mem_slab *slab, void *buffer,
173 size_t block_size, uint32_t num_blocks)
174 {
175 int rc;
176
177 slab->info.num_blocks = num_blocks;
178 slab->info.block_size = block_size;
179 slab->buffer = buffer;
180 slab->info.num_used = 0U;
181 slab->lock = (struct k_spinlock) {};
182
183 #ifdef CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION
184 slab->info.max_used = 0U;
185 #endif /* CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION */
186
187 rc = create_free_list(slab);
188 if (rc < 0) {
189 goto out;
190 }
191
192 #ifdef CONFIG_OBJ_CORE_MEM_SLAB
193 k_obj_core_init_and_link(K_OBJ_CORE(slab), &obj_type_mem_slab);
194 #endif /* CONFIG_OBJ_CORE_MEM_SLAB */
195 #ifdef CONFIG_OBJ_CORE_STATS_MEM_SLAB
196 k_obj_core_stats_register(K_OBJ_CORE(slab), &slab->info,
197 sizeof(struct k_mem_slab_info));
198 #endif /* CONFIG_OBJ_CORE_STATS_MEM_SLAB */
199
200 z_waitq_init(&slab->wait_q);
201 k_object_init(slab);
202 out:
203 SYS_PORT_TRACING_OBJ_INIT(k_mem_slab, slab, rc);
204
205 return rc;
206 }
207
slab_ptr_is_good(struct k_mem_slab * slab,const void * ptr)208 static bool slab_ptr_is_good(struct k_mem_slab *slab, const void *ptr)
209 {
210 if (!IS_ENABLED(CONFIG_MEM_SLAB_POINTER_VALIDATE)) {
211 return true;
212 }
213
214 const char *p = ptr;
215 ptrdiff_t offset = p - slab->buffer;
216
217 return (offset >= 0) &&
218 (offset < (slab->info.block_size * slab->info.num_blocks)) &&
219 ((offset % slab->info.block_size) == 0);
220 }
221
k_mem_slab_alloc(struct k_mem_slab * slab,void ** mem,k_timeout_t timeout)222 int k_mem_slab_alloc(struct k_mem_slab *slab, void **mem, k_timeout_t timeout)
223 {
224 k_spinlock_key_t key = k_spin_lock(&slab->lock);
225 int result;
226
227 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mem_slab, alloc, slab, timeout);
228
229 if (slab->free_list != NULL) {
230 /* take a free block */
231 *mem = slab->free_list;
232 slab->free_list = *(char **)(slab->free_list);
233 slab->info.num_used++;
234 __ASSERT((slab->free_list == NULL &&
235 slab->info.num_used == slab->info.num_blocks) ||
236 slab_ptr_is_good(slab, slab->free_list),
237 "slab corruption detected");
238
239 #ifdef CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION
240 slab->info.max_used = MAX(slab->info.num_used,
241 slab->info.max_used);
242 #endif /* CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION */
243
244 result = 0;
245 } else if (K_TIMEOUT_EQ(timeout, K_NO_WAIT) ||
246 !IS_ENABLED(CONFIG_MULTITHREADING)) {
247 /* don't wait for a free block to become available */
248 *mem = NULL;
249 result = -ENOMEM;
250 } else {
251 SYS_PORT_TRACING_OBJ_FUNC_BLOCKING(k_mem_slab, alloc, slab, timeout);
252
253 /* wait for a free block or timeout */
254 result = z_pend_curr(&slab->lock, key, &slab->wait_q, timeout);
255 if (result == 0) {
256 *mem = _current->base.swap_data;
257 }
258
259 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mem_slab, alloc, slab, timeout, result);
260
261 return result;
262 }
263
264 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mem_slab, alloc, slab, timeout, result);
265
266 k_spin_unlock(&slab->lock, key);
267
268 return result;
269 }
270
k_mem_slab_free(struct k_mem_slab * slab,void * mem)271 void k_mem_slab_free(struct k_mem_slab *slab, void *mem)
272 {
273 if (!slab_ptr_is_good(slab, mem)) {
274 __ASSERT(false, "Invalid memory pointer provided");
275 k_panic();
276 return;
277 }
278
279 k_spinlock_key_t key = k_spin_lock(&slab->lock);
280
281 SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_mem_slab, free, slab);
282 if (unlikely(slab->free_list == NULL) && IS_ENABLED(CONFIG_MULTITHREADING)) {
283 struct k_thread *pending_thread = z_unpend_first_thread(&slab->wait_q);
284
285 if (unlikely(pending_thread != NULL)) {
286 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mem_slab, free, slab);
287
288 z_thread_return_value_set_with_data(pending_thread, 0, mem);
289 z_ready_thread(pending_thread);
290 z_reschedule(&slab->lock, key);
291 return;
292 }
293 }
294 *(char **) mem = slab->free_list;
295 slab->free_list = (char *) mem;
296 slab->info.num_used--;
297
298 SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_mem_slab, free, slab);
299
300 k_spin_unlock(&slab->lock, key);
301 }
302
k_mem_slab_runtime_stats_get(struct k_mem_slab * slab,struct sys_memory_stats * stats)303 int k_mem_slab_runtime_stats_get(struct k_mem_slab *slab, struct sys_memory_stats *stats)
304 {
305 if ((slab == NULL) || (stats == NULL)) {
306 return -EINVAL;
307 }
308
309 k_spinlock_key_t key = k_spin_lock(&slab->lock);
310
311 stats->allocated_bytes = slab->info.num_used * slab->info.block_size;
312 stats->free_bytes = (slab->info.num_blocks - slab->info.num_used) *
313 slab->info.block_size;
314 #ifdef CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION
315 stats->max_allocated_bytes = slab->info.max_used *
316 slab->info.block_size;
317 #else
318 stats->max_allocated_bytes = 0;
319 #endif /* CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION */
320
321 k_spin_unlock(&slab->lock, key);
322
323 return 0;
324 }
325
326 #ifdef CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION
k_mem_slab_runtime_stats_reset_max(struct k_mem_slab * slab)327 int k_mem_slab_runtime_stats_reset_max(struct k_mem_slab *slab)
328 {
329 if (slab == NULL) {
330 return -EINVAL;
331 }
332
333 k_spinlock_key_t key = k_spin_lock(&slab->lock);
334
335 slab->info.max_used = slab->info.num_used;
336
337 k_spin_unlock(&slab->lock, key);
338
339 return 0;
340 }
341 #endif /* CONFIG_MEM_SLAB_TRACE_MAX_UTILIZATION */
342