1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/* Module internals
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
7 */
8
9#include <linux/elf.h>
10#include <linux/compiler.h>
11#include <linux/module.h>
12#include <linux/mutex.h>
13#include <linux/rculist.h>
14#include <linux/rcupdate.h>
15#include <linux/mm.h>
16
17#ifndef ARCH_SHF_SMALL
18#define ARCH_SHF_SMALL 0
19#endif
20
21/*
22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this
23 * section. This leaves 28 bits for offset on 32-bit systems, which is
24 * about 256 MiB (WARN_ON_ONCE if we exceed that).
25 */
26
27#define SH_ENTSIZE_TYPE_BITS 4
28#define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
29#define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
30#define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
31
32/* Maximum number of characters written by module_flags() */
33#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
34
35struct kernel_symbol {
36#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
37 int value_offset;
38 int name_offset;
39 int namespace_offset;
40#else
41 unsigned long value;
42 const char *name;
43 const char *namespace;
44#endif
45};
46
47extern struct mutex module_mutex;
48extern struct list_head modules;
49
50extern const struct module_attribute *const modinfo_attrs[];
51extern const size_t modinfo_attrs_count;
52
53/* Provided by the linker */
54extern const struct kernel_symbol __start___ksymtab[];
55extern const struct kernel_symbol __stop___ksymtab[];
56extern const struct kernel_symbol __start___ksymtab_gpl[];
57extern const struct kernel_symbol __stop___ksymtab_gpl[];
58extern const u32 __start___kcrctab[];
59extern const u32 __start___kcrctab_gpl[];
60
61#define KMOD_PATH_LEN 256
62extern char modprobe_path[];
63
64struct load_info {
65 const char *name;
66 /* pointer to module in temporary copy, freed at end of load_module() */
67 struct module *mod;
68 Elf_Ehdr *hdr;
69 unsigned long len;
70 Elf_Shdr *sechdrs;
71 char *secstrings, *strtab;
72 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs;
73 bool sig_ok;
74#ifdef CONFIG_KALLSYMS
75 unsigned long mod_kallsyms_init_off;
76#endif
77#ifdef CONFIG_MODULE_DECOMPRESS
78#ifdef CONFIG_MODULE_STATS
79 unsigned long compressed_len;
80#endif
81 struct page **pages;
82 unsigned int max_pages;
83 unsigned int used_pages;
84#endif
85 struct {
86 unsigned int sym;
87 unsigned int str;
88 unsigned int mod;
89 unsigned int vers;
90 unsigned int info;
91 unsigned int pcpu;
92 unsigned int vers_ext_crc;
93 unsigned int vers_ext_name;
94 } index;
95};
96
97enum mod_license {
98 NOT_GPL_ONLY,
99 GPL_ONLY,
100};
101
102struct find_symbol_arg {
103 /* Input */
104 const char *name;
105 bool gplok;
106 bool warn;
107
108 /* Output */
109 struct module *owner;
110 const u32 *crc;
111 const struct kernel_symbol *sym;
112 enum mod_license license;
113};
114
115/* modules using other modules */
116struct module_use {
117 struct list_head source_list;
118 struct list_head target_list;
119 struct module *source, *target;
120};
121
122int mod_verify_sig(const void *mod, struct load_info *info);
123int try_to_force_load(struct module *mod, const char *reason);
124bool find_symbol(struct find_symbol_arg *fsa);
125struct module *find_module_all(const char *name, size_t len, bool even_unformed);
126int cmp_name(const void *name, const void *sym);
127long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
128 Elf_Shdr *sechdr, unsigned int section);
129char *module_flags(struct module *mod, char *buf, bool show_state);
130size_t module_flags_taint(unsigned long taints, char *buf);
131
132char *module_next_tag_pair(char *string, unsigned long *secsize);
133
134#define for_each_modinfo_entry(entry, info, name) \
135 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry))
136
137static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
138{
139#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
140 return (unsigned long)offset_to_ptr(off: &sym->value_offset);
141#else
142 return sym->value;
143#endif
144}
145
146#ifdef CONFIG_LIVEPATCH
147int copy_module_elf(struct module *mod, struct load_info *info);
148void free_module_elf(struct module *mod);
149#else /* !CONFIG_LIVEPATCH */
150static inline int copy_module_elf(struct module *mod, struct load_info *info)
151{
152 return 0;
153}
154
155static inline void free_module_elf(struct module *mod) { }
156#endif /* CONFIG_LIVEPATCH */
157
158static inline bool set_livepatch_module(struct module *mod)
159{
160#ifdef CONFIG_LIVEPATCH
161 mod->klp = true;
162 return true;
163#else
164 return false;
165#endif
166}
167
168/**
169 * enum fail_dup_mod_reason - state at which a duplicate module was detected
170 *
171 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but
172 * we've determined that another module with the same name is already loaded
173 * or being processed on our &modules list. This happens on early_mod_check()
174 * right before layout_and_allocate(). The kernel would have already
175 * vmalloc()'d space for the entire module through finit_module(). If
176 * decompression was used two vmap() spaces were used. These failures can
177 * happen when userspace has not seen the module present on the kernel and
178 * tries to load the module multiple times at same time.
179 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation
180 * checks and the kernel determines that the module was unique and because
181 * of this allocated yet another private kernel copy of the module space in
182 * layout_and_allocate() but after this determined in add_unformed_module()
183 * that another module with the same name is already loaded or being processed.
184 * These failures should be mitigated as much as possible and are indicative
185 * of really fast races in loading modules. Without module decompression
186 * they waste twice as much vmap space. With module decompression three
187 * times the module's size vmap space is wasted.
188 */
189enum fail_dup_mod_reason {
190 FAIL_DUP_MOD_BECOMING = 0,
191 FAIL_DUP_MOD_LOAD,
192};
193
194#ifdef CONFIG_MODULE_DEBUGFS
195extern struct dentry *mod_debugfs_root;
196#endif
197
198#ifdef CONFIG_MODULE_STATS
199
200#define mod_stat_add_long(count, var) atomic_long_add(count, var)
201#define mod_stat_inc(name) atomic_inc(name)
202
203extern atomic_long_t total_mod_size;
204extern atomic_long_t total_text_size;
205extern atomic_long_t invalid_kread_bytes;
206extern atomic_long_t invalid_decompress_bytes;
207
208extern atomic_t modcount;
209extern atomic_t failed_kreads;
210extern atomic_t failed_decompress;
211struct mod_fail_load {
212 struct list_head list;
213 char name[MODULE_NAME_LEN];
214 atomic_long_t count;
215 unsigned long dup_fail_mask;
216};
217
218int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason);
219void mod_stat_bump_invalid(struct load_info *info, int flags);
220void mod_stat_bump_becoming(struct load_info *info, int flags);
221
222#else
223
224#define mod_stat_add_long(name, var)
225#define mod_stat_inc(name)
226
227static inline int try_add_failed_module(const char *name,
228 enum fail_dup_mod_reason reason)
229{
230 return 0;
231}
232
233static inline void mod_stat_bump_invalid(struct load_info *info, int flags)
234{
235}
236
237static inline void mod_stat_bump_becoming(struct load_info *info, int flags)
238{
239}
240
241#endif /* CONFIG_MODULE_STATS */
242
243#ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS
244bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret);
245void kmod_dup_request_announce(char *module_name, int ret);
246#else
247static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
248{
249 return false;
250}
251
252static inline void kmod_dup_request_announce(char *module_name, int ret)
253{
254}
255#endif
256
257#ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING
258struct mod_unload_taint {
259 struct list_head list;
260 char name[MODULE_NAME_LEN];
261 unsigned long taints;
262 u64 count;
263};
264
265int try_add_tainted_module(struct module *mod);
266void print_unloaded_tainted_modules(void);
267#else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
268static inline int try_add_tainted_module(struct module *mod)
269{
270 return 0;
271}
272
273static inline void print_unloaded_tainted_modules(void)
274{
275}
276#endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
277
278#ifdef CONFIG_MODULE_DECOMPRESS
279int module_decompress(struct load_info *info, const void *buf, size_t size);
280void module_decompress_cleanup(struct load_info *info);
281#else
282static inline int module_decompress(struct load_info *info,
283 const void *buf, size_t size)
284{
285 return -EOPNOTSUPP;
286}
287
288static inline void module_decompress_cleanup(struct load_info *info)
289{
290}
291#endif
292
293struct mod_tree_root {
294#ifdef CONFIG_MODULES_TREE_LOOKUP
295 struct latch_tree_root root;
296#endif
297 unsigned long addr_min;
298 unsigned long addr_max;
299#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
300 unsigned long data_addr_min;
301 unsigned long data_addr_max;
302#endif
303};
304
305extern struct mod_tree_root mod_tree;
306
307#ifdef CONFIG_MODULES_TREE_LOOKUP
308void mod_tree_insert(struct module *mod);
309void mod_tree_remove_init(struct module *mod);
310void mod_tree_remove(struct module *mod);
311struct module *mod_find(unsigned long addr, struct mod_tree_root *tree);
312#else /* !CONFIG_MODULES_TREE_LOOKUP */
313
314static inline void mod_tree_insert(struct module *mod) { }
315static inline void mod_tree_remove_init(struct module *mod) { }
316static inline void mod_tree_remove(struct module *mod) { }
317static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)
318{
319 struct module *mod;
320
321 list_for_each_entry_rcu(mod, &modules, list,
322 lockdep_is_held(&module_mutex)) {
323 if (within_module(addr, mod))
324 return mod;
325 }
326
327 return NULL;
328}
329#endif /* CONFIG_MODULES_TREE_LOOKUP */
330
331int module_enable_rodata_ro(const struct module *mod);
332int module_enable_rodata_ro_after_init(const struct module *mod);
333int module_enable_data_nx(const struct module *mod);
334int module_enable_text_rox(const struct module *mod);
335int module_enforce_rwx_sections(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
336 const char *secstrings,
337 const struct module *mod);
338void module_mark_ro_after_init(const Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
339 const char *secstrings);
340
341#ifdef CONFIG_MODULE_SIG
342int module_sig_check(struct load_info *info, int flags);
343#else /* !CONFIG_MODULE_SIG */
344static inline int module_sig_check(struct load_info *info, int flags)
345{
346 return 0;
347}
348#endif /* !CONFIG_MODULE_SIG */
349
350#ifdef CONFIG_DEBUG_KMEMLEAK
351void kmemleak_load_module(const struct module *mod, const struct load_info *info);
352#else /* !CONFIG_DEBUG_KMEMLEAK */
353static inline void kmemleak_load_module(const struct module *mod,
354 const struct load_info *info) { }
355#endif /* CONFIG_DEBUG_KMEMLEAK */
356
357#ifdef CONFIG_KALLSYMS
358void init_build_id(struct module *mod, const struct load_info *info);
359void layout_symtab(struct module *mod, struct load_info *info);
360void add_kallsyms(struct module *mod, const struct load_info *info);
361
362static inline bool sect_empty(const Elf_Shdr *sect)
363{
364 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
365}
366#else /* !CONFIG_KALLSYMS */
367static inline void init_build_id(struct module *mod, const struct load_info *info) { }
368static inline void layout_symtab(struct module *mod, struct load_info *info) { }
369static inline void add_kallsyms(struct module *mod, const struct load_info *info) { }
370#endif /* CONFIG_KALLSYMS */
371
372#ifdef CONFIG_SYSFS
373int mod_sysfs_setup(struct module *mod, const struct load_info *info,
374 struct kernel_param *kparam, unsigned int num_params);
375void mod_sysfs_teardown(struct module *mod);
376void init_param_lock(struct module *mod);
377#else /* !CONFIG_SYSFS */
378static inline int mod_sysfs_setup(struct module *mod,
379 const struct load_info *info,
380 struct kernel_param *kparam,
381 unsigned int num_params)
382{
383 return 0;
384}
385
386static inline void mod_sysfs_teardown(struct module *mod) { }
387static inline void init_param_lock(struct module *mod) { }
388#endif /* CONFIG_SYSFS */
389
390#ifdef CONFIG_MODVERSIONS
391int check_version(const struct load_info *info,
392 const char *symname, struct module *mod, const u32 *crc);
393void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp,
394 struct kernel_symbol *ks, struct tracepoint * const *tp);
395int check_modstruct_version(const struct load_info *info, struct module *mod);
396int same_magic(const char *amagic, const char *bmagic, bool has_crcs);
397struct modversion_info_ext {
398 size_t remaining;
399 const u32 *crc;
400 const char *name;
401};
402void modversion_ext_start(const struct load_info *info, struct modversion_info_ext *ver);
403void modversion_ext_advance(struct modversion_info_ext *ver);
404#define for_each_modversion_info_ext(ver, info) \
405 for (modversion_ext_start(info, &ver); ver.remaining > 0; modversion_ext_advance(&ver))
406#else /* !CONFIG_MODVERSIONS */
407static inline int check_version(const struct load_info *info,
408 const char *symname,
409 struct module *mod,
410 const u32 *crc)
411{
412 return 1;
413}
414
415static inline int check_modstruct_version(const struct load_info *info,
416 struct module *mod)
417{
418 return 1;
419}
420
421static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs)
422{
423 return strcmp(amagic, bmagic) == 0;
424}
425#endif /* CONFIG_MODVERSIONS */
426

source code of linux/kernel/module/internal.h