1 /*
2 * Copyright (c) 2010-2014 Wind River Systems, Inc.
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 /**
8 * @file
9 * @brief Kernel initialization module
10 *
11 * This module contains routines that are used to initialize the kernel.
12 */
13
14 #include <ctype.h>
15 #include <stdbool.h>
16 #include <string.h>
17 #include <offsets_short.h>
18 #include <zephyr/kernel.h>
19 #include <zephyr/sys/printk.h>
20 #include <zephyr/debug/stack.h>
21 #include <zephyr/random/random.h>
22 #include <zephyr/linker/sections.h>
23 #include <zephyr/toolchain.h>
24 #include <zephyr/kernel_structs.h>
25 #include <zephyr/device.h>
26 #include <zephyr/init.h>
27 #include <zephyr/linker/linker-defs.h>
28 #include <zephyr/platform/hooks.h>
29 #include <ksched.h>
30 #include <kthread.h>
31 #include <zephyr/sys/dlist.h>
32 #include <kernel_internal.h>
33 #include <zephyr/drivers/entropy.h>
34 #include <zephyr/logging/log_ctrl.h>
35 #include <zephyr/tracing/tracing.h>
36 #include <zephyr/debug/gcov.h>
37 #include <kswap.h>
38 #include <zephyr/timing/timing.h>
39 #include <zephyr/logging/log.h>
40 #include <zephyr/pm/device_runtime.h>
41 #include <zephyr/internal/syscall_handler.h>
42 LOG_MODULE_REGISTER(os, CONFIG_KERNEL_LOG_LEVEL);
43
44 /* the only struct z_kernel instance */
45 __pinned_bss
46 struct z_kernel _kernel;
47
48 #ifdef CONFIG_PM
49 __pinned_bss atomic_t _cpus_active;
50 #endif
51
52 /* init/main and idle threads */
53 K_THREAD_PINNED_STACK_DEFINE(z_main_stack, CONFIG_MAIN_STACK_SIZE);
54 struct k_thread z_main_thread;
55
56 #ifdef CONFIG_MULTITHREADING
57 __pinned_bss
58 struct k_thread z_idle_threads[CONFIG_MP_MAX_NUM_CPUS];
59
60 static K_KERNEL_PINNED_STACK_ARRAY_DEFINE(z_idle_stacks,
61 CONFIG_MP_MAX_NUM_CPUS,
62 CONFIG_IDLE_STACK_SIZE);
63
z_init_static_threads(void)64 static void z_init_static_threads(void)
65 {
66 STRUCT_SECTION_FOREACH(_static_thread_data, thread_data) {
67 z_setup_new_thread(
68 thread_data->init_thread,
69 thread_data->init_stack,
70 thread_data->init_stack_size,
71 thread_data->init_entry,
72 thread_data->init_p1,
73 thread_data->init_p2,
74 thread_data->init_p3,
75 thread_data->init_prio,
76 thread_data->init_options,
77 thread_data->init_name);
78
79 thread_data->init_thread->init_data = thread_data;
80 }
81
82 #ifdef CONFIG_USERSPACE
83 STRUCT_SECTION_FOREACH(k_object_assignment, pos) {
84 for (int i = 0; pos->objects[i] != NULL; i++) {
85 k_object_access_grant(pos->objects[i],
86 pos->thread);
87 }
88 }
89 #endif /* CONFIG_USERSPACE */
90
91 /*
92 * Non-legacy static threads may be started immediately or
93 * after a previously specified delay. Even though the
94 * scheduler is locked, ticks can still be delivered and
95 * processed. Take a sched lock to prevent them from running
96 * until they are all started.
97 *
98 * Note that static threads defined using the legacy API have a
99 * delay of K_FOREVER.
100 */
101 k_sched_lock();
102 STRUCT_SECTION_FOREACH(_static_thread_data, thread_data) {
103 k_timeout_t init_delay = Z_THREAD_INIT_DELAY(thread_data);
104
105 if (!K_TIMEOUT_EQ(init_delay, K_FOREVER)) {
106 thread_schedule_new(thread_data->init_thread,
107 init_delay);
108 }
109 }
110 k_sched_unlock();
111 }
112 #else
113 #define z_init_static_threads() do { } while (false)
114 #endif /* CONFIG_MULTITHREADING */
115
116 extern const struct init_entry __init_start[];
117 extern const struct init_entry __init_EARLY_start[];
118 extern const struct init_entry __init_PRE_KERNEL_1_start[];
119 extern const struct init_entry __init_PRE_KERNEL_2_start[];
120 extern const struct init_entry __init_POST_KERNEL_start[];
121 extern const struct init_entry __init_APPLICATION_start[];
122 extern const struct init_entry __init_end[];
123
124 enum init_level {
125 INIT_LEVEL_EARLY = 0,
126 INIT_LEVEL_PRE_KERNEL_1,
127 INIT_LEVEL_PRE_KERNEL_2,
128 INIT_LEVEL_POST_KERNEL,
129 INIT_LEVEL_APPLICATION,
130 #ifdef CONFIG_SMP
131 INIT_LEVEL_SMP,
132 #endif /* CONFIG_SMP */
133 };
134
135 #ifdef CONFIG_SMP
136 extern const struct init_entry __init_SMP_start[];
137 #endif /* CONFIG_SMP */
138
139 /*
140 * storage space for the interrupt stack
141 *
142 * Note: This area is used as the system stack during kernel initialization,
143 * since the kernel hasn't yet set up its own stack areas. The dual purposing
144 * of this area is safe since interrupts are disabled until the kernel context
145 * switches to the init thread.
146 */
147 K_KERNEL_PINNED_STACK_ARRAY_DEFINE(z_interrupt_stacks,
148 CONFIG_MP_MAX_NUM_CPUS,
149 CONFIG_ISR_STACK_SIZE);
150
151 extern void idle(void *unused1, void *unused2, void *unused3);
152
153 #ifdef CONFIG_OBJ_CORE_SYSTEM
154 static struct k_obj_type obj_type_cpu;
155 static struct k_obj_type obj_type_kernel;
156
157 #ifdef CONFIG_OBJ_CORE_STATS_SYSTEM
158 static struct k_obj_core_stats_desc cpu_stats_desc = {
159 .raw_size = sizeof(struct k_cycle_stats),
160 .query_size = sizeof(struct k_thread_runtime_stats),
161 .raw = z_cpu_stats_raw,
162 .query = z_cpu_stats_query,
163 .reset = NULL,
164 .disable = NULL,
165 .enable = NULL,
166 };
167
168 static struct k_obj_core_stats_desc kernel_stats_desc = {
169 .raw_size = sizeof(struct k_cycle_stats) * CONFIG_MP_MAX_NUM_CPUS,
170 .query_size = sizeof(struct k_thread_runtime_stats),
171 .raw = z_kernel_stats_raw,
172 .query = z_kernel_stats_query,
173 .reset = NULL,
174 .disable = NULL,
175 .enable = NULL,
176 };
177 #endif /* CONFIG_OBJ_CORE_STATS_SYSTEM */
178 #endif /* CONFIG_OBJ_CORE_SYSTEM */
179
180 /* LCOV_EXCL_START
181 *
182 * This code is called so early in the boot process that code coverage
183 * doesn't work properly. In addition, not all arches call this code,
184 * some like x86 do this with optimized assembly
185 */
186
187 /**
188 * @brief equivalent of memset() for early boot usage
189 *
190 * Architectures that can't safely use the regular (optimized) memset very
191 * early during boot because e.g. hardware isn't yet sufficiently initialized
192 * may override this with their own safe implementation.
193 */
194 __boot_func
z_early_memset(void * dst,int c,size_t n)195 void __weak z_early_memset(void *dst, int c, size_t n)
196 {
197 (void) memset(dst, c, n);
198 }
199
200 /**
201 * @brief equivalent of memcpy() for early boot usage
202 *
203 * Architectures that can't safely use the regular (optimized) memcpy very
204 * early during boot because e.g. hardware isn't yet sufficiently initialized
205 * may override this with their own safe implementation.
206 */
207 __boot_func
z_early_memcpy(void * dst,const void * src,size_t n)208 void __weak z_early_memcpy(void *dst, const void *src, size_t n)
209 {
210 (void) memcpy(dst, src, n);
211 }
212
213 /**
214 * @brief Clear BSS
215 *
216 * This routine clears the BSS region, so all bytes are 0.
217 */
218 __boot_func
z_bss_zero(void)219 void z_bss_zero(void)
220 {
221 if (IS_ENABLED(CONFIG_SKIP_BSS_CLEAR)) {
222 return;
223 }
224
225 z_early_memset(__bss_start, 0, __bss_end - __bss_start);
226 #if DT_NODE_HAS_STATUS_OKAY(DT_CHOSEN(zephyr_ccm))
227 z_early_memset(&__ccm_bss_start, 0,
228 (uintptr_t) &__ccm_bss_end
229 - (uintptr_t) &__ccm_bss_start);
230 #endif
231 #if DT_NODE_HAS_STATUS_OKAY(DT_CHOSEN(zephyr_dtcm))
232 z_early_memset(&__dtcm_bss_start, 0,
233 (uintptr_t) &__dtcm_bss_end
234 - (uintptr_t) &__dtcm_bss_start);
235 #endif
236 #if DT_NODE_HAS_STATUS_OKAY(DT_CHOSEN(zephyr_ocm))
237 z_early_memset(&__ocm_bss_start, 0,
238 (uintptr_t) &__ocm_bss_end
239 - (uintptr_t) &__ocm_bss_start);
240 #endif
241 #ifdef CONFIG_CODE_DATA_RELOCATION
242 extern void bss_zeroing_relocation(void);
243
244 bss_zeroing_relocation();
245 #endif /* CONFIG_CODE_DATA_RELOCATION */
246 #ifdef CONFIG_COVERAGE_GCOV
247 z_early_memset(&__gcov_bss_start, 0,
248 ((uintptr_t) &__gcov_bss_end - (uintptr_t) &__gcov_bss_start));
249 #endif /* CONFIG_COVERAGE_GCOV */
250 #ifdef CONFIG_NOCACHE_MEMORY
251 z_early_memset(&_nocache_ram_start, 0,
252 (uintptr_t) &_nocache_ram_end
253 - (uintptr_t) &_nocache_ram_start);
254 #endif
255 }
256
257 #ifdef CONFIG_LINKER_USE_BOOT_SECTION
258 /**
259 * @brief Clear BSS within the boot region
260 *
261 * This routine clears the BSS within the boot region.
262 * This is separate from z_bss_zero() as boot region may
263 * contain symbols required for the boot process before
264 * paging is initialized.
265 */
266 __boot_func
z_bss_zero_boot(void)267 void z_bss_zero_boot(void)
268 {
269 z_early_memset(&lnkr_boot_bss_start, 0,
270 (uintptr_t)&lnkr_boot_bss_end
271 - (uintptr_t)&lnkr_boot_bss_start);
272 }
273 #endif /* CONFIG_LINKER_USE_BOOT_SECTION */
274
275 #ifdef CONFIG_LINKER_USE_PINNED_SECTION
276 /**
277 * @brief Clear BSS within the pinned region
278 *
279 * This routine clears the BSS within the pinned region.
280 * This is separate from z_bss_zero() as pinned region may
281 * contain symbols required for the boot process before
282 * paging is initialized.
283 */
284 #ifdef CONFIG_LINKER_USE_BOOT_SECTION
285 __boot_func
286 #else
287 __pinned_func
288 #endif /* CONFIG_LINKER_USE_BOOT_SECTION */
z_bss_zero_pinned(void)289 void z_bss_zero_pinned(void)
290 {
291 z_early_memset(&lnkr_pinned_bss_start, 0,
292 (uintptr_t)&lnkr_pinned_bss_end
293 - (uintptr_t)&lnkr_pinned_bss_start);
294 }
295 #endif /* CONFIG_LINKER_USE_PINNED_SECTION */
296
297 #ifdef CONFIG_REQUIRES_STACK_CANARIES
298 #ifdef CONFIG_STACK_CANARIES_TLS
299 extern Z_THREAD_LOCAL volatile uintptr_t __stack_chk_guard;
300 #else
301 extern volatile uintptr_t __stack_chk_guard;
302 #endif /* CONFIG_STACK_CANARIES_TLS */
303 #endif /* CONFIG_REQUIRES_STACK_CANARIES */
304
305 /* LCOV_EXCL_STOP */
306
307 __pinned_bss
308 bool z_sys_post_kernel;
309
do_device_init(const struct device * dev)310 static int do_device_init(const struct device *dev)
311 {
312 int rc = 0;
313
314 if (dev->ops.init != NULL) {
315 rc = dev->ops.init(dev);
316 /* Mark device initialized. If initialization
317 * failed, record the error condition.
318 */
319 if (rc != 0) {
320 if (rc < 0) {
321 rc = -rc;
322 }
323 if (rc > UINT8_MAX) {
324 rc = UINT8_MAX;
325 }
326 dev->state->init_res = rc;
327 }
328 }
329
330 dev->state->initialized = true;
331
332 if (rc == 0) {
333 /* Run automatic device runtime enablement */
334 (void)pm_device_runtime_auto_enable(dev);
335 }
336
337 return rc;
338 }
339
340 /**
341 * @brief Execute all the init entry initialization functions at a given level
342 *
343 * @details Invokes the initialization routine for each init entry object
344 * created by the INIT_ENTRY_DEFINE() macro using the specified level.
345 * The linker script places the init entry objects in memory in the order
346 * they need to be invoked, with symbols indicating where one level leaves
347 * off and the next one begins.
348 *
349 * @param level init level to run.
350 */
z_sys_init_run_level(enum init_level level)351 static void z_sys_init_run_level(enum init_level level)
352 {
353 static const struct init_entry *levels[] = {
354 __init_EARLY_start,
355 __init_PRE_KERNEL_1_start,
356 __init_PRE_KERNEL_2_start,
357 __init_POST_KERNEL_start,
358 __init_APPLICATION_start,
359 #ifdef CONFIG_SMP
360 __init_SMP_start,
361 #endif /* CONFIG_SMP */
362 /* End marker */
363 __init_end,
364 };
365 const struct init_entry *entry;
366
367 for (entry = levels[level]; entry < levels[level+1]; entry++) {
368 const struct device *dev = entry->dev;
369 int result = 0;
370
371 sys_trace_sys_init_enter(entry, level);
372 if (dev != NULL) {
373 if ((dev->flags & DEVICE_FLAG_INIT_DEFERRED) == 0U) {
374 result = do_device_init(dev);
375 }
376 } else {
377 result = entry->init_fn();
378 }
379 sys_trace_sys_init_exit(entry, level, result);
380 }
381 }
382
383
z_impl_device_init(const struct device * dev)384 int z_impl_device_init(const struct device *dev)
385 {
386 if (dev->state->initialized) {
387 return -EALREADY;
388 }
389
390 return do_device_init(dev);
391 }
392
393 #ifdef CONFIG_USERSPACE
z_vrfy_device_init(const struct device * dev)394 static inline int z_vrfy_device_init(const struct device *dev)
395 {
396 K_OOPS(K_SYSCALL_OBJ_INIT(dev, K_OBJ_ANY));
397
398 return z_impl_device_init(dev);
399 }
400 #include <zephyr/syscalls/device_init_mrsh.c>
401 #endif
402
403 extern void boot_banner(void);
404
405 #ifdef CONFIG_BOOTARGS
406 extern const char *get_bootargs(void);
prepare_main_args(int * argc)407 static char **prepare_main_args(int *argc)
408 {
409 #ifdef CONFIG_DYNAMIC_BOOTARGS
410 const char *bootargs = get_bootargs();
411 #else
412 const char bootargs[] = CONFIG_BOOTARGS_STRING;
413 #endif
414
415 /* beginning of the buffer contains argument's strings, end of it contains argvs */
416 static char args_buf[CONFIG_BOOTARGS_ARGS_BUFFER_SIZE];
417 char *strings_end = (char *)args_buf;
418 char **argv_begin = (char **)WB_DN(
419 args_buf + CONFIG_BOOTARGS_ARGS_BUFFER_SIZE - sizeof(char *));
420 int i = 0;
421
422 *argc = 0;
423 *argv_begin = NULL;
424
425 #ifdef CONFIG_DYNAMIC_BOOTARGS
426 if (!bootargs) {
427 return argv_begin;
428 }
429 #endif
430
431 while (1) {
432 while (isspace(bootargs[i])) {
433 i++;
434 }
435
436 if (bootargs[i] == '\0') {
437 return argv_begin;
438 }
439
440 if (strings_end + sizeof(char *) >= (char *)argv_begin) {
441 LOG_WRN("not enough space in args buffer to accommodate all bootargs"
442 " - bootargs truncated");
443 return argv_begin;
444 }
445
446 argv_begin--;
447 memmove(argv_begin, argv_begin + 1, *argc * sizeof(char *));
448 argv_begin[*argc] = strings_end;
449
450 bool quoted = false;
451
452 if (bootargs[i] == '\"' || bootargs[i] == '\'') {
453 char delimiter = bootargs[i];
454
455 for (int j = i + 1; bootargs[j] != '\0'; j++) {
456 if (bootargs[j] == delimiter) {
457 quoted = true;
458 break;
459 }
460 }
461 }
462
463 if (quoted) {
464 char delimiter = bootargs[i];
465
466 i++; /* strip quotes */
467 while (bootargs[i] != delimiter
468 && strings_end < (char *)argv_begin) {
469 *strings_end++ = bootargs[i++];
470 }
471 i++; /* strip quotes */
472 } else {
473 while (!isspace(bootargs[i])
474 && bootargs[i] != '\0'
475 && strings_end < (char *)argv_begin) {
476 *strings_end++ = bootargs[i++];
477 }
478 }
479
480 if (strings_end < (char *)argv_begin) {
481 *strings_end++ = '\0';
482 } else {
483 LOG_WRN("not enough space in args buffer to accommodate all bootargs"
484 " - bootargs truncated");
485 argv_begin[*argc] = NULL;
486 return argv_begin;
487 }
488 (*argc)++;
489 }
490 }
491
492 #endif
493
494 #ifdef CONFIG_STATIC_INIT_GNU
495
496 extern void (*__zephyr_init_array_start[])();
497 extern void (*__zephyr_init_array_end[])();
498
z_static_init_gnu(void)499 static void z_static_init_gnu(void)
500 {
501 void (**fn)();
502
503 for (fn = __zephyr_init_array_start; fn != __zephyr_init_array_end; fn++) {
504 /* MWDT toolchain sticks a NULL at the end of the array */
505 if (*fn == NULL) {
506 break;
507 }
508 (**fn)();
509 }
510 }
511
512 #endif
513
514 /**
515 * @brief Mainline for kernel's background thread
516 *
517 * This routine completes kernel initialization by invoking the remaining
518 * init functions, then invokes application's main() routine.
519 */
520 __boot_func
bg_thread_main(void * unused1,void * unused2,void * unused3)521 static void bg_thread_main(void *unused1, void *unused2, void *unused3)
522 {
523 ARG_UNUSED(unused1);
524 ARG_UNUSED(unused2);
525 ARG_UNUSED(unused3);
526
527 #ifdef CONFIG_MMU
528 /* Invoked here such that backing store or eviction algorithms may
529 * initialize kernel objects, and that all POST_KERNEL and later tasks
530 * may perform memory management tasks (except for
531 * k_mem_map_phys_bare() which is allowed at any time)
532 */
533 z_mem_manage_init();
534 #endif /* CONFIG_MMU */
535 z_sys_post_kernel = true;
536
537 #if CONFIG_IRQ_OFFLOAD
538 arch_irq_offload_init();
539 #endif
540 z_sys_init_run_level(INIT_LEVEL_POST_KERNEL);
541 #if CONFIG_SOC_LATE_INIT_HOOK
542 soc_late_init_hook();
543 #endif
544 #if CONFIG_BOARD_LATE_INIT_HOOK
545 board_late_init_hook();
546 #endif
547
548 #if defined(CONFIG_STACK_POINTER_RANDOM) && (CONFIG_STACK_POINTER_RANDOM != 0)
549 z_stack_adjust_initialized = 1;
550 #endif /* CONFIG_STACK_POINTER_RANDOM */
551 boot_banner();
552
553 #ifdef CONFIG_STATIC_INIT_GNU
554 z_static_init_gnu();
555 #endif /* CONFIG_STATIC_INIT_GNU */
556
557 /* Final init level before app starts */
558 z_sys_init_run_level(INIT_LEVEL_APPLICATION);
559
560 z_init_static_threads();
561
562 #ifdef CONFIG_KERNEL_COHERENCE
563 __ASSERT_NO_MSG(arch_mem_coherent(&_kernel));
564 #endif /* CONFIG_KERNEL_COHERENCE */
565
566 #ifdef CONFIG_SMP
567 if (!IS_ENABLED(CONFIG_SMP_BOOT_DELAY)) {
568 z_smp_init();
569 }
570 z_sys_init_run_level(INIT_LEVEL_SMP);
571 #endif /* CONFIG_SMP */
572
573 #ifdef CONFIG_MMU
574 z_mem_manage_boot_finish();
575 #endif /* CONFIG_MMU */
576
577 #ifdef CONFIG_BOOTARGS
578 extern int main(int, char **);
579
580 int argc = 0;
581 char **argv = prepare_main_args(&argc);
582 (void)main(argc, argv);
583 #else
584 extern int main(void);
585
586 (void)main();
587 #endif /* CONFIG_BOOTARGS */
588
589 /* Mark non-essential since main() has no more work to do */
590 z_thread_essential_clear(&z_main_thread);
591
592 #ifdef CONFIG_COVERAGE_DUMP
593 /* Dump coverage data once the main() has exited. */
594 gcov_coverage_dump();
595 #endif /* CONFIG_COVERAGE_DUMP */
596 } /* LCOV_EXCL_LINE ... because we just dumped final coverage data */
597
598 #if defined(CONFIG_MULTITHREADING)
599 __boot_func
init_idle_thread(int i)600 static void init_idle_thread(int i)
601 {
602 struct k_thread *thread = &z_idle_threads[i];
603 k_thread_stack_t *stack = z_idle_stacks[i];
604 size_t stack_size = K_KERNEL_STACK_SIZEOF(z_idle_stacks[i]);
605
606 #ifdef CONFIG_THREAD_NAME
607
608 #if CONFIG_MP_MAX_NUM_CPUS > 1
609 char tname[8];
610 snprintk(tname, 8, "idle %02d", i);
611 #else
612 char *tname = "idle";
613 #endif /* CONFIG_MP_MAX_NUM_CPUS */
614
615 #else
616 char *tname = NULL;
617 #endif /* CONFIG_THREAD_NAME */
618
619 z_setup_new_thread(thread, stack,
620 stack_size, idle, &_kernel.cpus[i],
621 NULL, NULL, K_IDLE_PRIO, K_ESSENTIAL,
622 tname);
623 z_mark_thread_as_not_sleeping(thread);
624
625 #ifdef CONFIG_SMP
626 thread->base.is_idle = 1U;
627 #endif /* CONFIG_SMP */
628 }
629
z_init_cpu(int id)630 void z_init_cpu(int id)
631 {
632 init_idle_thread(id);
633 _kernel.cpus[id].idle_thread = &z_idle_threads[id];
634 _kernel.cpus[id].id = id;
635 _kernel.cpus[id].irq_stack =
636 (K_KERNEL_STACK_BUFFER(z_interrupt_stacks[id]) +
637 K_KERNEL_STACK_SIZEOF(z_interrupt_stacks[id]));
638 #ifdef CONFIG_SCHED_THREAD_USAGE_ALL
639 _kernel.cpus[id].usage = &_kernel.usage[id];
640 _kernel.cpus[id].usage->track_usage =
641 CONFIG_SCHED_THREAD_USAGE_AUTO_ENABLE;
642 #endif
643
644 #ifdef CONFIG_PM
645 /*
646 * Increment number of CPUs active. The pm subsystem
647 * will keep track of this from here.
648 */
649 atomic_inc(&_cpus_active);
650 #endif
651
652 #ifdef CONFIG_OBJ_CORE_SYSTEM
653 k_obj_core_init_and_link(K_OBJ_CORE(&_kernel.cpus[id]), &obj_type_cpu);
654 #ifdef CONFIG_OBJ_CORE_STATS_SYSTEM
655 k_obj_core_stats_register(K_OBJ_CORE(&_kernel.cpus[id]),
656 _kernel.cpus[id].usage,
657 sizeof(struct k_cycle_stats));
658 #endif
659 #endif
660 }
661
662 /**
663 *
664 * @brief Initializes kernel data structures
665 *
666 * This routine initializes various kernel data structures, including
667 * the init and idle threads and any architecture-specific initialization.
668 *
669 * Note that all fields of "_kernel" are set to zero on entry, which may
670 * be all the initialization many of them require.
671 *
672 * @return initial stack pointer for the main thread
673 */
674 __boot_func
prepare_multithreading(void)675 static char *prepare_multithreading(void)
676 {
677 char *stack_ptr;
678
679 /* _kernel.ready_q is all zeroes */
680 z_sched_init();
681
682 #ifndef CONFIG_SMP
683 /*
684 * prime the cache with the main thread since:
685 *
686 * - the cache can never be NULL
687 * - the main thread will be the one to run first
688 * - no other thread is initialized yet and thus their priority fields
689 * contain garbage, which would prevent the cache loading algorithm
690 * to work as intended
691 */
692 _kernel.ready_q.cache = &z_main_thread;
693 #endif /* CONFIG_SMP */
694 stack_ptr = z_setup_new_thread(&z_main_thread, z_main_stack,
695 K_THREAD_STACK_SIZEOF(z_main_stack),
696 bg_thread_main,
697 NULL, NULL, NULL,
698 CONFIG_MAIN_THREAD_PRIORITY,
699 K_ESSENTIAL, "main");
700 z_mark_thread_as_not_sleeping(&z_main_thread);
701 z_ready_thread(&z_main_thread);
702
703 z_init_cpu(0);
704
705 return stack_ptr;
706 }
707
708 __boot_func
switch_to_main_thread(char * stack_ptr)709 static FUNC_NORETURN void switch_to_main_thread(char *stack_ptr)
710 {
711 #ifdef CONFIG_ARCH_HAS_CUSTOM_SWAP_TO_MAIN
712 arch_switch_to_main_thread(&z_main_thread, stack_ptr, bg_thread_main);
713 #else
714 ARG_UNUSED(stack_ptr);
715 /*
716 * Context switch to main task (entry function is _main()): the
717 * current fake thread is not on a wait queue or ready queue, so it
718 * will never be rescheduled in.
719 */
720 z_swap_unlocked();
721 #endif /* CONFIG_ARCH_HAS_CUSTOM_SWAP_TO_MAIN */
722 CODE_UNREACHABLE; /* LCOV_EXCL_LINE */
723 }
724 #endif /* CONFIG_MULTITHREADING */
725
726 __boot_func
z_early_rand_get(uint8_t * buf,size_t length)727 void __weak z_early_rand_get(uint8_t *buf, size_t length)
728 {
729 static uint64_t state = (uint64_t)CONFIG_TIMER_RANDOM_INITIAL_STATE;
730 int rc;
731
732 #ifdef CONFIG_ENTROPY_HAS_DRIVER
733 const struct device *const entropy = DEVICE_DT_GET_OR_NULL(DT_CHOSEN(zephyr_entropy));
734
735 if ((entropy != NULL) && device_is_ready(entropy)) {
736 /* Try to see if driver provides an ISR-specific API */
737 rc = entropy_get_entropy_isr(entropy, buf, length, ENTROPY_BUSYWAIT);
738 if (rc > 0) {
739 length -= rc;
740 buf += rc;
741 }
742 }
743 #endif /* CONFIG_ENTROPY_HAS_DRIVER */
744
745 while (length > 0) {
746 uint32_t val;
747
748 state = state + k_cycle_get_32();
749 state = state * 2862933555777941757ULL + 3037000493ULL;
750 val = (uint32_t)(state >> 32);
751 rc = MIN(length, sizeof(val));
752 z_early_memcpy((void *)buf, &val, rc);
753
754 length -= rc;
755 buf += rc;
756 }
757 }
758
759 /**
760 *
761 * @brief Initialize kernel
762 *
763 * This routine is invoked when the system is ready to run C code. The
764 * processor must be running in 32-bit mode, and the BSS must have been
765 * cleared/zeroed.
766 *
767 * @return Does not return
768 */
769 __boot_func
770 FUNC_NO_STACK_PROTECTOR
z_cstart(void)771 FUNC_NORETURN void z_cstart(void)
772 {
773 /* gcov hook needed to get the coverage report.*/
774 gcov_static_init();
775
776 /* initialize early init calls */
777 z_sys_init_run_level(INIT_LEVEL_EARLY);
778
779 /* perform any architecture-specific initialization */
780 arch_kernel_init();
781
782 LOG_CORE_INIT();
783
784 #if defined(CONFIG_MULTITHREADING)
785 z_dummy_thread_init(&_thread_dummy);
786 #endif /* CONFIG_MULTITHREADING */
787 /* do any necessary initialization of static devices */
788 z_device_state_init();
789
790 #if CONFIG_SOC_EARLY_INIT_HOOK
791 soc_early_init_hook();
792 #endif
793 #if CONFIG_BOARD_EARLY_INIT_HOOK
794 board_early_init_hook();
795 #endif
796 /* perform basic hardware initialization */
797 z_sys_init_run_level(INIT_LEVEL_PRE_KERNEL_1);
798 #if defined(CONFIG_SMP)
799 arch_smp_init();
800 #endif
801 z_sys_init_run_level(INIT_LEVEL_PRE_KERNEL_2);
802
803 #ifdef CONFIG_REQUIRES_STACK_CANARIES
804 uintptr_t stack_guard;
805
806 z_early_rand_get((uint8_t *)&stack_guard, sizeof(stack_guard));
807 __stack_chk_guard = stack_guard;
808 __stack_chk_guard <<= 8;
809 #endif /* CONFIG_REQUIRES_STACK_CANARIES */
810
811 #ifdef CONFIG_TIMING_FUNCTIONS_NEED_AT_BOOT
812 timing_init();
813 timing_start();
814 #endif /* CONFIG_TIMING_FUNCTIONS_NEED_AT_BOOT */
815
816 #ifdef CONFIG_MULTITHREADING
817 switch_to_main_thread(prepare_multithreading());
818 #else
819 #ifdef ARCH_SWITCH_TO_MAIN_NO_MULTITHREADING
820 /* Custom ARCH-specific routine to switch to main()
821 * in the case of no multi-threading.
822 */
823 ARCH_SWITCH_TO_MAIN_NO_MULTITHREADING(bg_thread_main,
824 NULL, NULL, NULL);
825 #else
826 bg_thread_main(NULL, NULL, NULL);
827
828 /* LCOV_EXCL_START
829 * We've already dumped coverage data at this point.
830 */
831 irq_lock();
832 while (true) {
833 }
834 /* LCOV_EXCL_STOP */
835 #endif /* ARCH_SWITCH_TO_MAIN_NO_MULTITHREADING */
836 #endif /* CONFIG_MULTITHREADING */
837
838 /*
839 * Compiler can't tell that the above routines won't return and issues
840 * a warning unless we explicitly tell it that control never gets this
841 * far.
842 */
843
844 CODE_UNREACHABLE; /* LCOV_EXCL_LINE */
845 }
846
847 #ifdef CONFIG_OBJ_CORE_SYSTEM
init_cpu_obj_core_list(void)848 static int init_cpu_obj_core_list(void)
849 {
850 /* Initialize CPU object type */
851
852 z_obj_type_init(&obj_type_cpu, K_OBJ_TYPE_CPU_ID,
853 offsetof(struct _cpu, obj_core));
854
855 #ifdef CONFIG_OBJ_CORE_STATS_SYSTEM
856 k_obj_type_stats_init(&obj_type_cpu, &cpu_stats_desc);
857 #endif /* CONFIG_OBJ_CORE_STATS_SYSTEM */
858
859 return 0;
860 }
861
init_kernel_obj_core_list(void)862 static int init_kernel_obj_core_list(void)
863 {
864 /* Initialize kernel object type */
865
866 z_obj_type_init(&obj_type_kernel, K_OBJ_TYPE_KERNEL_ID,
867 offsetof(struct z_kernel, obj_core));
868
869 #ifdef CONFIG_OBJ_CORE_STATS_SYSTEM
870 k_obj_type_stats_init(&obj_type_kernel, &kernel_stats_desc);
871 #endif /* CONFIG_OBJ_CORE_STATS_SYSTEM */
872
873 k_obj_core_init_and_link(K_OBJ_CORE(&_kernel), &obj_type_kernel);
874 #ifdef CONFIG_OBJ_CORE_STATS_SYSTEM
875 k_obj_core_stats_register(K_OBJ_CORE(&_kernel), _kernel.usage,
876 sizeof(_kernel.usage));
877 #endif /* CONFIG_OBJ_CORE_STATS_SYSTEM */
878
879 return 0;
880 }
881
882 SYS_INIT(init_cpu_obj_core_list, PRE_KERNEL_1,
883 CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
884
885 SYS_INIT(init_kernel_obj_core_list, PRE_KERNEL_1,
886 CONFIG_KERNEL_INIT_PRIORITY_OBJECTS);
887 #endif /* CONFIG_OBJ_CORE_SYSTEM */
888