Lines Matching refs:mod
110 static void mod_update_bounds(struct module *mod) in mod_update_bounds() argument
112 __mod_update_bounds(mod->core_layout.base, mod->core_layout.size, &mod_tree); in mod_update_bounds()
113 if (mod->init_layout.size) in mod_update_bounds()
114 __mod_update_bounds(mod->init_layout.base, mod->init_layout.size, &mod_tree); in mod_update_bounds()
116 __mod_update_bounds(mod->data_layout.base, mod->data_layout.size, &mod_data_tree); in mod_update_bounds()
146 static inline int strong_try_module_get(struct module *mod) in strong_try_module_get() argument
148 BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED); in strong_try_module_get()
149 if (mod && mod->state == MODULE_STATE_COMING) in strong_try_module_get()
151 if (try_module_get(mod)) in strong_try_module_get()
157 static inline void add_taint_module(struct module *mod, unsigned flag, in add_taint_module() argument
161 set_bit(flag, &mod->taints); in add_taint_module()
168 void __noreturn __module_put_and_kthread_exit(struct module *mod, long code) in __module_put_and_kthread_exit() argument
170 module_put(mod); in __module_put_and_kthread_exit()
305 struct module *mod; in find_symbol() local
314 list_for_each_entry_rcu(mod, &modules, list, in find_symbol()
317 { mod->syms, mod->syms + mod->num_syms, mod->crcs, in find_symbol()
319 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms, in find_symbol()
320 mod->gpl_crcs, in find_symbol()
324 if (mod->state == MODULE_STATE_UNFORMED) in find_symbol()
328 if (find_exported_symbol_in_section(&arr[i], mod, fsa)) in find_symbol()
343 struct module *mod; in find_module_all() local
347 list_for_each_entry_rcu(mod, &modules, list, in find_module_all()
349 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) in find_module_all()
351 if (strlen(mod->name) == len && !memcmp(mod->name, name, len)) in find_module_all()
352 return mod; in find_module_all()
364 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
366 return mod->percpu; in mod_percpu()
369 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
379 mod->name, align, PAGE_SIZE); in percpu_modalloc()
383 mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align); in percpu_modalloc()
384 if (!mod->percpu) { in percpu_modalloc()
386 mod->name, (unsigned long)pcpusec->sh_size); in percpu_modalloc()
389 mod->percpu_size = pcpusec->sh_size; in percpu_modalloc()
393 static void percpu_modfree(struct module *mod) in percpu_modfree() argument
395 free_percpu(mod->percpu); in percpu_modfree()
403 static void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
409 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); in percpu_modcopy()
414 struct module *mod; in __is_module_percpu_address() local
419 list_for_each_entry_rcu(mod, &modules, list) { in __is_module_percpu_address()
420 if (mod->state == MODULE_STATE_UNFORMED) in __is_module_percpu_address()
422 if (!mod->percpu_size) in __is_module_percpu_address()
425 void *start = per_cpu_ptr(mod->percpu, cpu); in __is_module_percpu_address()
428 if (va >= start && va < start + mod->percpu_size) { in __is_module_percpu_address()
432 per_cpu_ptr(mod->percpu, in __is_module_percpu_address()
460 static inline void __percpu *mod_percpu(struct module *mod) in mod_percpu() argument
464 static int percpu_modalloc(struct module *mod, struct load_info *info) in percpu_modalloc() argument
471 static inline void percpu_modfree(struct module *mod) in percpu_modfree() argument
478 static inline void percpu_modcopy(struct module *mod, in percpu_modcopy() argument
497 static void setup_modinfo_##field(struct module *mod, const char *s) \
499 mod->field = kstrdup(s, GFP_KERNEL); \
504 return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
506 static int modinfo_##field##_exists(struct module *mod) \
508 return mod->field != NULL; \
510 static void free_modinfo_##field(struct module *mod) \
512 kfree(mod->field); \
513 mod->field = NULL; \
539 static int module_unload_init(struct module *mod) in module_unload_init() argument
545 atomic_set(&mod->refcnt, MODULE_REF_BASE); in module_unload_init()
547 INIT_LIST_HEAD(&mod->source_list); in module_unload_init()
548 INIT_LIST_HEAD(&mod->target_list); in module_unload_init()
551 atomic_inc(&mod->refcnt); in module_unload_init()
616 static void module_unload_free(struct module *mod) in module_unload_free() argument
621 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) { in module_unload_free()
623 pr_debug("%s unusing %s\n", mod->name, i->name); in module_unload_free()
648 static int try_release_module_ref(struct module *mod) in try_release_module_ref() argument
653 ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt); in try_release_module_ref()
657 ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0); in try_release_module_ref()
662 static int try_stop_module(struct module *mod, int flags, int *forced) in try_stop_module() argument
665 if (try_release_module_ref(mod) != 0) { in try_stop_module()
672 mod->state = MODULE_STATE_GOING; in try_stop_module()
685 int module_refcount(struct module *mod) in module_refcount() argument
687 return atomic_read(&mod->refcnt) - MODULE_REF_BASE; in module_refcount()
692 static void free_module(struct module *mod);
697 struct module *mod; in SYSCALL_DEFINE2() local
714 mod = find_module(name); in SYSCALL_DEFINE2()
715 if (!mod) { in SYSCALL_DEFINE2()
720 if (!list_empty(&mod->source_list)) { in SYSCALL_DEFINE2()
727 if (mod->state != MODULE_STATE_LIVE) { in SYSCALL_DEFINE2()
729 pr_debug("%s already dying\n", mod->name); in SYSCALL_DEFINE2()
735 if (mod->init && !mod->exit) { in SYSCALL_DEFINE2()
744 ret = try_stop_module(mod, flags, &forced); in SYSCALL_DEFINE2()
750 if (mod->exit != NULL) in SYSCALL_DEFINE2()
751 mod->exit(); in SYSCALL_DEFINE2()
753 MODULE_STATE_GOING, mod); in SYSCALL_DEFINE2()
754 klp_module_going(mod); in SYSCALL_DEFINE2()
755 ftrace_release_mod(mod); in SYSCALL_DEFINE2()
760 strscpy(last_unloaded_module.name, mod->name, sizeof(last_unloaded_module.name)); in SYSCALL_DEFINE2()
761 …strscpy(last_unloaded_module.taints, module_flags(mod, buf, false), sizeof(last_unloaded_module.ta… in SYSCALL_DEFINE2()
763 free_module(mod); in SYSCALL_DEFINE2()
810 return sprintf(buffer, "%i\n", module_refcount(mk->mod)); in show_refcnt()
861 static inline void module_unload_free(struct module *mod) in module_unload_free() argument
870 static inline int module_unload_init(struct module *mod) in module_unload_init() argument
894 switch (mk->mod->state) { in show_initstate()
929 return sprintf(buffer, "%u\n", mk->mod->core_layout.size); in show_coresize()
939 return sprintf(buffer, "%u\n", mk->mod->data_layout.size); in show_datasize()
949 return sprintf(buffer, "%u\n", mk->mod->init_layout.size); in show_initsize()
960 l = module_flags_taint(mk->mod->taints, buffer); in show_taint()
989 int try_to_force_load(struct module *mod, const char *reason) in try_to_force_load() argument
993 pr_warn("%s: %s: kernel tainted.\n", mod->name, reason); in try_to_force_load()
994 add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE); in try_to_force_load()
1007 struct module *mod) in verify_namespace_is_imported() argument
1027 mod->name, kernel_symbol_name(sym), namespace); in verify_namespace_is_imported()
1035 static bool inherit_taint(struct module *mod, struct module *owner, const char *name) in inherit_taint() argument
1040 if (mod->using_gplonly_symbols) { in inherit_taint()
1042 mod->name, name, owner->name); in inherit_taint()
1046 if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) { in inherit_taint()
1048 mod->name, name, owner->name); in inherit_taint()
1049 set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints); in inherit_taint()
1055 static const struct kernel_symbol *resolve_symbol(struct module *mod, in resolve_symbol() argument
1062 .gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), in resolve_symbol()
1078 mod->using_gplonly_symbols = true; in resolve_symbol()
1080 if (!inherit_taint(mod, fsa.owner, name)) { in resolve_symbol()
1085 if (!check_version(info, name, mod, fsa.crc)) { in resolve_symbol()
1090 err = verify_namespace_is_imported(info, fsa.sym, mod); in resolve_symbol()
1096 err = ref_module(mod, fsa.owner); in resolve_symbol()
1111 resolve_symbol_wait(struct module *mod, in resolve_symbol_wait() argument
1119 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner)) in resolve_symbol_wait()
1123 mod->name, owner); in resolve_symbol_wait()
1138 void __weak module_arch_cleanup(struct module *mod) in module_arch_cleanup() argument
1142 void __weak module_arch_freeing_init(struct module *mod) in module_arch_freeing_init() argument
1147 static void free_module(struct module *mod) in free_module() argument
1149 trace_module_free(mod); in free_module()
1151 mod_sysfs_teardown(mod); in free_module()
1158 mod->state = MODULE_STATE_UNFORMED; in free_module()
1162 ddebug_remove_module(mod->name); in free_module()
1165 module_arch_cleanup(mod); in free_module()
1168 module_unload_free(mod); in free_module()
1171 destroy_params(mod->kp, mod->num_kp); in free_module()
1173 if (is_livepatch_module(mod)) in free_module()
1174 free_module_elf(mod); in free_module()
1179 list_del_rcu(&mod->list); in free_module()
1180 mod_tree_remove(mod); in free_module()
1182 module_bug_cleanup(mod); in free_module()
1185 if (try_add_tainted_module(mod)) in free_module()
1187 mod->name); in free_module()
1191 module_arch_freeing_init(mod); in free_module()
1192 module_memfree(mod->init_layout.base); in free_module()
1193 kfree(mod->args); in free_module()
1194 percpu_modfree(mod); in free_module()
1197 lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); in free_module()
1200 module_memfree(mod->core_layout.base); in free_module()
1202 vfree(mod->data_layout.base); in free_module()
1230 static int verify_exported_symbols(struct module *mod) in verify_exported_symbols() argument
1238 { mod->syms, mod->num_syms }, in verify_exported_symbols()
1239 { mod->gpl_syms, mod->num_gpl_syms }, in verify_exported_symbols()
1251 mod->name, kernel_symbol_name(s), in verify_exported_symbols()
1276 static int simplify_symbols(struct module *mod, const struct load_info *info) in simplify_symbols() argument
1300 mod->name); in simplify_symbols()
1315 ksym = resolve_symbol_wait(mod, info, name); in simplify_symbols()
1330 mod->name, name, ret); in simplify_symbols()
1336 secbase = (unsigned long)mod_percpu(mod); in simplify_symbols()
1347 static int apply_relocations(struct module *mod, const struct load_info *info) in apply_relocations() argument
1365 err = klp_apply_section_relocs(mod, info->sechdrs, in apply_relocations()
1372 info->index.sym, i, mod); in apply_relocations()
1375 info->index.sym, i, mod); in apply_relocations()
1383 unsigned int __weak arch_mod_section_prepend(struct module *mod, in arch_mod_section_prepend() argument
1391 long module_get_offset(struct module *mod, unsigned int *size, in module_get_offset() argument
1396 *size += arch_mod_section_prepend(mod, section); in module_get_offset()
1417 static void layout_sections(struct module *mod, struct load_info *info) in layout_sections() argument
1448 sizep = m ? &mod->data_layout.size : &mod->core_layout.size; in layout_sections()
1449 s->sh_entsize = module_get_offset(mod, sizep, s, i); in layout_sections()
1454 mod->core_layout.size = strict_align(mod->core_layout.size); in layout_sections()
1455 mod->core_layout.text_size = mod->core_layout.size; in layout_sections()
1458 mod->data_layout.size = strict_align(mod->data_layout.size); in layout_sections()
1459 mod->data_layout.ro_size = mod->data_layout.size; in layout_sections()
1462 mod->data_layout.size = strict_align(mod->data_layout.size); in layout_sections()
1463 mod->data_layout.ro_after_init_size = mod->data_layout.size; in layout_sections()
1466 mod->data_layout.size = strict_align(mod->data_layout.size); in layout_sections()
1482 s->sh_entsize = (module_get_offset(mod, &mod->init_layout.size, s, i) in layout_sections()
1488 mod->init_layout.size = strict_align(mod->init_layout.size); in layout_sections()
1489 mod->init_layout.text_size = mod->init_layout.size; in layout_sections()
1492 mod->init_layout.size = strict_align(mod->init_layout.size); in layout_sections()
1493 mod->init_layout.ro_size = mod->init_layout.size; in layout_sections()
1500 mod->init_layout.ro_after_init_size = mod->init_layout.ro_size; in layout_sections()
1503 mod->init_layout.size = strict_align(mod->init_layout.size); in layout_sections()
1509 static void set_license(struct module *mod, const char *license) in set_license() argument
1517 mod->name, license); in set_license()
1518 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in set_license()
1573 static void setup_modinfo(struct module *mod, struct load_info *info) in setup_modinfo() argument
1580 attr->setup(mod, get_modinfo(info, attr->attr.name)); in setup_modinfo()
1584 static void free_modinfo(struct module *mod) in free_modinfo() argument
1591 attr->free(mod); in free_modinfo()
1595 static void dynamic_debug_setup(struct module *mod, struct _ddebug_info *dyndbg) in dynamic_debug_setup() argument
1599 ddebug_add_module(dyndbg, mod->name); in dynamic_debug_setup()
1602 static void dynamic_debug_remove(struct module *mod, struct _ddebug_info *dyndbg) in dynamic_debug_remove() argument
1605 ddebug_remove_module(mod->name); in dynamic_debug_remove()
1801 static int check_modinfo_livepatch(struct module *mod, struct load_info *info) in check_modinfo_livepatch() argument
1807 if (set_livepatch_module(mod)) { in check_modinfo_livepatch()
1808 add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK); in check_modinfo_livepatch()
1810 mod->name); in check_modinfo_livepatch()
1815 mod->name); in check_modinfo_livepatch()
1819 static void check_modinfo_retpoline(struct module *mod, struct load_info *info) in check_modinfo_retpoline() argument
1825 mod->name); in check_modinfo_retpoline()
1928 info->index.mod = find_sec(info, ".gnu.linkonce.this_module"); in setup_load_info()
1929 if (!info->index.mod) { in setup_load_info()
1935 info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset; in setup_load_info()
1942 info->name = info->mod->name; in setup_load_info()
1954 static int check_modinfo(struct module *mod, struct load_info *info, int flags) in check_modinfo() argument
1964 err = try_to_force_load(mod, "bad vermagic"); in check_modinfo()
1976 mod->name); in check_modinfo()
1977 add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK); in check_modinfo()
1980 check_modinfo_retpoline(mod, info); in check_modinfo()
1983 add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK); in check_modinfo()
1985 "is unknown, you have been warned.\n", mod->name); in check_modinfo()
1988 err = check_modinfo_livepatch(mod, info); in check_modinfo()
1993 set_license(mod, get_modinfo(info, "license")); in check_modinfo()
1998 mod->name); in check_modinfo()
1999 add_taint_module(mod, TAINT_TEST, LOCKDEP_STILL_OK); in check_modinfo()
2005 static int find_module_sections(struct module *mod, struct load_info *info) in find_module_sections() argument
2007 mod->kp = section_objs(info, "__param", in find_module_sections()
2008 sizeof(*mod->kp), &mod->num_kp); in find_module_sections()
2009 mod->syms = section_objs(info, "__ksymtab", in find_module_sections()
2010 sizeof(*mod->syms), &mod->num_syms); in find_module_sections()
2011 mod->crcs = section_addr(info, "__kcrctab"); in find_module_sections()
2012 mod->gpl_syms = section_objs(info, "__ksymtab_gpl", in find_module_sections()
2013 sizeof(*mod->gpl_syms), in find_module_sections()
2014 &mod->num_gpl_syms); in find_module_sections()
2015 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl"); in find_module_sections()
2018 mod->ctors = section_objs(info, ".ctors", in find_module_sections()
2019 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
2020 if (!mod->ctors) in find_module_sections()
2021 mod->ctors = section_objs(info, ".init_array", in find_module_sections()
2022 sizeof(*mod->ctors), &mod->num_ctors); in find_module_sections()
2029 mod->name); in find_module_sections()
2034 mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1, in find_module_sections()
2035 &mod->noinstr_text_size); in find_module_sections()
2038 mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs", in find_module_sections()
2039 sizeof(*mod->tracepoints_ptrs), in find_module_sections()
2040 &mod->num_tracepoints); in find_module_sections()
2043 mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs", in find_module_sections()
2044 sizeof(*mod->srcu_struct_ptrs), in find_module_sections()
2045 &mod->num_srcu_structs); in find_module_sections()
2048 mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map", in find_module_sections()
2049 sizeof(*mod->bpf_raw_events), in find_module_sections()
2050 &mod->num_bpf_raw_events); in find_module_sections()
2053 mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size); in find_module_sections()
2056 mod->jump_entries = section_objs(info, "__jump_table", in find_module_sections()
2057 sizeof(*mod->jump_entries), in find_module_sections()
2058 &mod->num_jump_entries); in find_module_sections()
2061 mod->trace_events = section_objs(info, "_ftrace_events", in find_module_sections()
2062 sizeof(*mod->trace_events), in find_module_sections()
2063 &mod->num_trace_events); in find_module_sections()
2064 mod->trace_evals = section_objs(info, "_ftrace_eval_map", in find_module_sections()
2065 sizeof(*mod->trace_evals), in find_module_sections()
2066 &mod->num_trace_evals); in find_module_sections()
2069 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt", in find_module_sections()
2070 sizeof(*mod->trace_bprintk_fmt_start), in find_module_sections()
2071 &mod->num_trace_bprintk_fmt); in find_module_sections()
2075 mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION, in find_module_sections()
2076 sizeof(*mod->ftrace_callsites), in find_module_sections()
2077 &mod->num_ftrace_callsites); in find_module_sections()
2080 mod->ei_funcs = section_objs(info, "_error_injection_whitelist", in find_module_sections()
2081 sizeof(*mod->ei_funcs), in find_module_sections()
2082 &mod->num_ei_funcs); in find_module_sections()
2085 mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1, in find_module_sections()
2086 &mod->kprobes_text_size); in find_module_sections()
2087 mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist", in find_module_sections()
2089 &mod->num_kprobe_blacklist); in find_module_sections()
2092 mod->printk_index_start = section_objs(info, ".printk_index", in find_module_sections()
2093 sizeof(*mod->printk_index_start), in find_module_sections()
2094 &mod->printk_index_size); in find_module_sections()
2097 mod->static_call_sites = section_objs(info, ".static_call_sites", in find_module_sections()
2098 sizeof(*mod->static_call_sites), in find_module_sections()
2099 &mod->num_static_call_sites); in find_module_sections()
2102 mod->kunit_suites = section_objs(info, ".kunit_test_suites", in find_module_sections()
2103 sizeof(*mod->kunit_suites), in find_module_sections()
2104 &mod->num_kunit_suites); in find_module_sections()
2107 mod->extable = section_objs(info, "__ex_table", in find_module_sections()
2108 sizeof(*mod->extable), &mod->num_exentries); in find_module_sections()
2111 pr_warn("%s: Ignoring obsolete parameters\n", mod->name); in find_module_sections()
2121 static int move_module(struct module *mod, struct load_info *info) in move_module() argument
2127 ptr = module_alloc(mod->core_layout.size); in move_module()
2137 memset(ptr, 0, mod->core_layout.size); in move_module()
2138 mod->core_layout.base = ptr; in move_module()
2140 if (mod->init_layout.size) { in move_module()
2141 ptr = module_alloc(mod->init_layout.size); in move_module()
2150 module_memfree(mod->core_layout.base); in move_module()
2153 memset(ptr, 0, mod->init_layout.size); in move_module()
2154 mod->init_layout.base = ptr; in move_module()
2156 mod->init_layout.base = NULL; in move_module()
2160 ptr = vzalloc(mod->data_layout.size); in move_module()
2168 module_memfree(mod->core_layout.base); in move_module()
2169 module_memfree(mod->init_layout.base); in move_module()
2173 mod->data_layout.base = ptr; in move_module()
2185 dest = mod->init_layout.base in move_module()
2188 dest = mod->data_layout.base + shdr->sh_entsize; in move_module()
2190 dest = mod->core_layout.base + shdr->sh_entsize; in move_module()
2203 static int check_module_license_and_versions(struct module *mod) in check_module_license_and_versions() argument
2212 if (strcmp(mod->name, "ndiswrapper") == 0) in check_module_license_and_versions()
2216 if (strcmp(mod->name, "driverloader") == 0) in check_module_license_and_versions()
2217 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in check_module_license_and_versions()
2221 if (strcmp(mod->name, "lve") == 0) in check_module_license_and_versions()
2222 add_taint_module(mod, TAINT_PROPRIETARY_MODULE, in check_module_license_and_versions()
2226 pr_warn("%s: module license taints kernel.\n", mod->name); in check_module_license_and_versions()
2229 if ((mod->num_syms && !mod->crcs) || in check_module_license_and_versions()
2230 (mod->num_gpl_syms && !mod->gpl_crcs)) { in check_module_license_and_versions()
2231 return try_to_force_load(mod, in check_module_license_and_versions()
2238 static void flush_module_icache(const struct module *mod) in flush_module_icache() argument
2245 if (mod->init_layout.base) in flush_module_icache()
2246 flush_icache_range((unsigned long)mod->init_layout.base, in flush_module_icache()
2247 (unsigned long)mod->init_layout.base in flush_module_icache()
2248 + mod->init_layout.size); in flush_module_icache()
2249 flush_icache_range((unsigned long)mod->core_layout.base, in flush_module_icache()
2250 (unsigned long)mod->core_layout.base + mod->core_layout.size); in flush_module_icache()
2261 struct module *mod) in module_frob_arch_sections() argument
2289 struct module *mod; in layout_and_allocate() local
2293 err = check_modinfo(info->mod, info, flags); in layout_and_allocate()
2299 info->secstrings, info->mod); in layout_and_allocate()
2304 info->secstrings, info->mod); in layout_and_allocate()
2334 layout_sections(info->mod, info); in layout_and_allocate()
2335 layout_symtab(info->mod, info); in layout_and_allocate()
2338 err = move_module(info->mod, info); in layout_and_allocate()
2343 mod = (void *)info->sechdrs[info->index.mod].sh_addr; in layout_and_allocate()
2344 kmemleak_load_module(mod, info); in layout_and_allocate()
2345 return mod; in layout_and_allocate()
2349 static void module_deallocate(struct module *mod, struct load_info *info) in module_deallocate() argument
2351 percpu_modfree(mod); in module_deallocate()
2352 module_arch_freeing_init(mod); in module_deallocate()
2353 module_memfree(mod->init_layout.base); in module_deallocate()
2354 module_memfree(mod->core_layout.base); in module_deallocate()
2356 vfree(mod->data_layout.base); in module_deallocate()
2367 static int post_relocation(struct module *mod, const struct load_info *info) in post_relocation() argument
2370 sort_extable(mod->extable, mod->extable + mod->num_exentries); in post_relocation()
2373 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr, in post_relocation()
2377 add_kallsyms(mod, info); in post_relocation()
2380 return module_finalize(info->hdr, info->sechdrs, mod); in post_relocation()
2386 struct module *mod; in finished_loading() local
2396 mod = find_module_all(name, strlen(name), true); in finished_loading()
2397 ret = !mod || mod->state == MODULE_STATE_LIVE in finished_loading()
2398 || mod->state == MODULE_STATE_GOING; in finished_loading()
2405 static void do_mod_ctors(struct module *mod) in do_mod_ctors() argument
2410 for (i = 0; i < mod->num_ctors; i++) in do_mod_ctors()
2411 mod->ctors[i](); in do_mod_ctors()
2449 static noinline int do_init_module(struct module *mod) in do_init_module() argument
2459 freeinit->module_init = mod->init_layout.base; in do_init_module()
2461 do_mod_ctors(mod); in do_init_module()
2463 if (mod->init != NULL) in do_init_module()
2464 ret = do_one_initcall(mod->init); in do_init_module()
2472 __func__, mod->name, ret, __func__); in do_init_module()
2477 mod->state = MODULE_STATE_LIVE; in do_init_module()
2479 MODULE_STATE_LIVE, mod); in do_init_module()
2482 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); in do_init_module()
2492 if (!mod->async_probe_requested) in do_init_module()
2495 ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base + in do_init_module()
2496 mod->init_layout.size); in do_init_module()
2499 module_put(mod); in do_init_module()
2500 trim_init_extable(mod); in do_init_module()
2503 rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); in do_init_module()
2505 module_enable_ro(mod, true); in do_init_module()
2506 mod_tree_remove_init(mod); in do_init_module()
2507 module_arch_freeing_init(mod); in do_init_module()
2508 mod->init_layout.base = NULL; in do_init_module()
2509 mod->init_layout.size = 0; in do_init_module()
2510 mod->init_layout.ro_size = 0; in do_init_module()
2511 mod->init_layout.ro_after_init_size = 0; in do_init_module()
2512 mod->init_layout.text_size = 0; in do_init_module()
2515 mod->btf_data = NULL; in do_init_module()
2542 mod->state = MODULE_STATE_GOING; in do_init_module()
2544 module_put(mod); in do_init_module()
2546 MODULE_STATE_GOING, mod); in do_init_module()
2547 klp_module_going(mod); in do_init_module()
2548 ftrace_release_mod(mod); in do_init_module()
2549 free_module(mod); in do_init_module()
2567 static int add_unformed_module(struct module *mod) in add_unformed_module() argument
2572 mod->state = MODULE_STATE_UNFORMED; in add_unformed_module()
2575 old = find_module_all(mod->name, strlen(mod->name), true); in add_unformed_module()
2582 finished_loading(mod->name)); in add_unformed_module()
2588 old = find_module_all(mod->name, strlen(mod->name), in add_unformed_module()
2605 mod_update_bounds(mod); in add_unformed_module()
2606 list_add_rcu(&mod->list, &modules); in add_unformed_module()
2607 mod_tree_insert(mod); in add_unformed_module()
2616 static int complete_formation(struct module *mod, struct load_info *info) in complete_formation() argument
2623 err = verify_exported_symbols(mod); in complete_formation()
2628 module_bug_finalize(info->hdr, info->sechdrs, mod); in complete_formation()
2629 module_cfi_finalize(info->hdr, info->sechdrs, mod); in complete_formation()
2631 if (module_check_misalignment(mod)) in complete_formation()
2634 module_enable_ro(mod, false); in complete_formation()
2635 module_enable_nx(mod); in complete_formation()
2636 module_enable_x(mod); in complete_formation()
2642 mod->state = MODULE_STATE_COMING; in complete_formation()
2654 static int prepare_coming_module(struct module *mod) in prepare_coming_module() argument
2658 ftrace_module_enable(mod); in prepare_coming_module()
2659 err = klp_module_coming(mod); in prepare_coming_module()
2664 MODULE_STATE_COMING, MODULE_STATE_GOING, mod); in prepare_coming_module()
2667 klp_module_going(mod); in prepare_coming_module()
2675 struct module *mod = arg; in unknown_module_param_cb() local
2679 if (kstrtobool(val, &mod->async_probe_requested)) in unknown_module_param_cb()
2680 mod->async_probe_requested = true; in unknown_module_param_cb()
2698 struct module *mod; in load_module() local
2749 if (!check_modstruct_version(info, info->mod)) { in load_module()
2755 mod = layout_and_allocate(info, flags); in load_module()
2756 if (IS_ERR(mod)) { in load_module()
2757 err = PTR_ERR(mod); in load_module()
2761 audit_log_kern_module(mod->name); in load_module()
2764 err = add_unformed_module(mod); in load_module()
2769 mod->sig_ok = info->sig_ok; in load_module()
2770 if (!mod->sig_ok) { in load_module()
2773 "kernel\n", mod->name); in load_module()
2774 add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK); in load_module()
2779 err = percpu_modalloc(mod, info); in load_module()
2784 err = module_unload_init(mod); in load_module()
2788 init_param_lock(mod); in load_module()
2794 err = find_module_sections(mod, info); in load_module()
2798 err = check_module_license_and_versions(mod); in load_module()
2803 setup_modinfo(mod, info); in load_module()
2806 err = simplify_symbols(mod, info); in load_module()
2810 err = apply_relocations(mod, info); in load_module()
2814 err = post_relocation(mod, info); in load_module()
2818 flush_module_icache(mod); in load_module()
2821 mod->args = strndup_user(uargs, ~0UL >> 1); in load_module()
2822 if (IS_ERR(mod->args)) { in load_module()
2823 err = PTR_ERR(mod->args); in load_module()
2827 init_build_id(mod, info); in load_module()
2828 dynamic_debug_setup(mod, &info->dyndbg); in load_module()
2831 ftrace_module_init(mod); in load_module()
2834 err = complete_formation(mod, info); in load_module()
2838 err = prepare_coming_module(mod); in load_module()
2842 mod->async_probe_requested = async_probe; in load_module()
2845 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, in load_module()
2846 -32768, 32767, mod, in load_module()
2853 mod->name, after_dashes); in load_module()
2857 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp); in load_module()
2861 if (is_livepatch_module(mod)) { in load_module()
2862 err = copy_module_elf(mod, info); in load_module()
2871 trace_module_load(mod); in load_module()
2873 return do_init_module(mod); in load_module()
2876 mod_sysfs_teardown(mod); in load_module()
2878 mod->state = MODULE_STATE_GOING; in load_module()
2879 destroy_params(mod->kp, mod->num_kp); in load_module()
2881 MODULE_STATE_GOING, mod); in load_module()
2882 klp_module_going(mod); in load_module()
2884 mod->state = MODULE_STATE_GOING; in load_module()
2887 module_bug_cleanup(mod); in load_module()
2891 ftrace_release_mod(mod); in load_module()
2892 dynamic_debug_remove(mod, &info->dyndbg); in load_module()
2894 kfree(mod->args); in load_module()
2896 module_arch_cleanup(mod); in load_module()
2898 free_modinfo(mod); in load_module()
2900 module_unload_free(mod); in load_module()
2904 list_del_rcu(&mod->list); in load_module()
2905 mod_tree_remove(mod); in load_module()
2912 lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); in load_module()
2914 module_deallocate(mod, info); in load_module()
2982 char *module_flags(struct module *mod, char *buf, bool show_state) in module_flags() argument
2986 BUG_ON(mod->state == MODULE_STATE_UNFORMED); in module_flags()
2987 if (!mod->taints && !show_state) in module_flags()
2989 if (mod->taints || in module_flags()
2990 mod->state == MODULE_STATE_GOING || in module_flags()
2991 mod->state == MODULE_STATE_COMING) { in module_flags()
2993 bx += module_flags_taint(mod->taints, buf + bx); in module_flags()
2995 if (mod->state == MODULE_STATE_GOING && show_state) in module_flags()
2998 if (mod->state == MODULE_STATE_COMING && show_state) in module_flags()
3012 struct module *mod; in search_module_extables() local
3015 mod = __module_address(addr); in search_module_extables()
3016 if (!mod) in search_module_extables()
3019 if (!mod->num_exentries) in search_module_extables()
3022 e = search_extable(mod->extable, in search_module_extables()
3023 mod->num_exentries, in search_module_extables()
3062 struct module *mod; in __module_address() local
3076 mod = mod_find(addr, tree); in __module_address()
3077 if (mod) { in __module_address()
3078 BUG_ON(!within_module(addr, mod)); in __module_address()
3079 if (mod->state == MODULE_STATE_UNFORMED) in __module_address()
3080 mod = NULL; in __module_address()
3082 return mod; in __module_address()
3113 struct module *mod = __module_address(addr); in __module_text_address() local
3114 if (mod) { in __module_text_address()
3116 if (!within(addr, mod->init_layout.base, mod->init_layout.text_size) in __module_text_address()
3117 && !within(addr, mod->core_layout.base, mod->core_layout.text_size)) in __module_text_address()
3118 mod = NULL; in __module_text_address()
3120 return mod; in __module_text_address()
3126 struct module *mod; in print_modules() local
3132 list_for_each_entry_rcu(mod, &modules, list) { in print_modules()
3133 if (mod->state == MODULE_STATE_UNFORMED) in print_modules()
3135 pr_cont(" %s%s", mod->name, module_flags(mod, buf, true)); in print_modules()