1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/alloc_tag.h>
3#include <linux/execmem.h>
4#include <linux/fs.h>
5#include <linux/gfp.h>
6#include <linux/kallsyms.h>
7#include <linux/module.h>
8#include <linux/page_ext.h>
9#include <linux/proc_fs.h>
10#include <linux/seq_buf.h>
11#include <linux/seq_file.h>
12#include <linux/string_choices.h>
13#include <linux/vmalloc.h>
14#include <linux/kmemleak.h>
15
16#define ALLOCINFO_FILE_NAME "allocinfo"
17#define MODULE_ALLOC_TAG_VMAP_SIZE (100000UL * sizeof(struct alloc_tag))
18#define SECTION_START(NAME) (CODETAG_SECTION_START_PREFIX NAME)
19#define SECTION_STOP(NAME) (CODETAG_SECTION_STOP_PREFIX NAME)
20
21#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
22static bool mem_profiling_support = true;
23#else
24static bool mem_profiling_support;
25#endif
26
27static struct codetag_type *alloc_tag_cttype;
28
29#ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU
30DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
31EXPORT_SYMBOL(_shared_alloc_tag);
32#endif
33
34DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
35 mem_alloc_profiling_key);
36EXPORT_SYMBOL(mem_alloc_profiling_key);
37
38DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
39
40struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
41unsigned long alloc_tag_ref_mask;
42int alloc_tag_ref_offs;
43
44struct allocinfo_private {
45 struct codetag_iterator iter;
46 bool print_header;
47};
48
49static void *allocinfo_start(struct seq_file *m, loff_t *pos)
50{
51 struct allocinfo_private *priv;
52 loff_t node = *pos;
53
54 priv = (struct allocinfo_private *)m->private;
55 codetag_lock_module_list(cttype: alloc_tag_cttype, lock: true);
56 if (node == 0) {
57 priv->print_header = true;
58 priv->iter = codetag_get_ct_iter(cttype: alloc_tag_cttype);
59 codetag_next_ct(iter: &priv->iter);
60 }
61 return priv->iter.ct ? priv : NULL;
62}
63
64static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
65{
66 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
67 struct codetag *ct = codetag_next_ct(iter: &priv->iter);
68
69 (*pos)++;
70 if (!ct)
71 return NULL;
72
73 return priv;
74}
75
76static void allocinfo_stop(struct seq_file *m, void *arg)
77{
78 codetag_lock_module_list(cttype: alloc_tag_cttype, lock: false);
79}
80
81static void print_allocinfo_header(struct seq_buf *buf)
82{
83 /* Output format version, so we can change it. */
84 seq_buf_printf(s: buf, fmt: "allocinfo - version: 2.0\n");
85 seq_buf_printf(s: buf, fmt: "# <size> <calls> <tag info>\n");
86}
87
88static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
89{
90 struct alloc_tag *tag = ct_to_alloc_tag(ct);
91 struct alloc_tag_counters counter = alloc_tag_read(tag);
92 s64 bytes = counter.bytes;
93
94 seq_buf_printf(s: out, fmt: "%12lli %8llu ", bytes, counter.calls);
95 codetag_to_text(out, ct);
96 if (unlikely(alloc_tag_is_inaccurate(tag)))
97 seq_buf_printf(s: out, fmt: " accurate:no");
98 seq_buf_putc(s: out, c: ' ');
99 seq_buf_putc(s: out, c: '\n');
100}
101
102static int allocinfo_show(struct seq_file *m, void *arg)
103{
104 struct allocinfo_private *priv = (struct allocinfo_private *)arg;
105 char *bufp;
106 size_t n = seq_get_buf(m, bufp: &bufp);
107 struct seq_buf buf;
108
109 seq_buf_init(s: &buf, buf: bufp, size: n);
110 if (priv->print_header) {
111 print_allocinfo_header(buf: &buf);
112 priv->print_header = false;
113 }
114 alloc_tag_to_text(out: &buf, ct: priv->iter.ct);
115 seq_commit(m, num: seq_buf_used(s: &buf));
116 return 0;
117}
118
119static const struct seq_operations allocinfo_seq_op = {
120 .start = allocinfo_start,
121 .next = allocinfo_next,
122 .stop = allocinfo_stop,
123 .show = allocinfo_show,
124};
125
126size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
127{
128 struct codetag_iterator iter;
129 struct codetag *ct;
130 struct codetag_bytes n;
131 unsigned int i, nr = 0;
132
133 if (IS_ERR_OR_NULL(ptr: alloc_tag_cttype))
134 return 0;
135
136 if (can_sleep)
137 codetag_lock_module_list(cttype: alloc_tag_cttype, lock: true);
138 else if (!codetag_trylock_module_list(cttype: alloc_tag_cttype))
139 return 0;
140
141 iter = codetag_get_ct_iter(cttype: alloc_tag_cttype);
142 while ((ct = codetag_next_ct(iter: &iter))) {
143 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
144
145 n.ct = ct;
146 n.bytes = counter.bytes;
147
148 for (i = 0; i < nr; i++)
149 if (n.bytes > tags[i].bytes)
150 break;
151
152 if (i < count) {
153 nr -= nr == count;
154 memmove(&tags[i + 1],
155 &tags[i],
156 sizeof(tags[0]) * (nr - i));
157 nr++;
158 tags[i] = n;
159 }
160 }
161
162 codetag_lock_module_list(cttype: alloc_tag_cttype, lock: false);
163
164 return nr;
165}
166
167void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
168{
169 int i;
170 struct alloc_tag *tag;
171 unsigned int nr_pages = 1 << new_order;
172
173 if (!mem_alloc_profiling_enabled())
174 return;
175
176 tag = __pgalloc_tag_get(&folio->page);
177 if (!tag)
178 return;
179
180 for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
181 union pgtag_ref_handle handle;
182 union codetag_ref ref;
183
184 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
185 /* Set new reference to point to the original tag */
186 alloc_tag_ref_set(&ref, tag);
187 update_page_tag_ref(handle, &ref);
188 put_page_tag_ref(handle);
189 }
190 }
191}
192
193void pgalloc_tag_swap(struct folio *new, struct folio *old)
194{
195 union pgtag_ref_handle handle_old, handle_new;
196 union codetag_ref ref_old, ref_new;
197 struct alloc_tag *tag_old, *tag_new;
198
199 if (!mem_alloc_profiling_enabled())
200 return;
201
202 tag_old = __pgalloc_tag_get(&old->page);
203 if (!tag_old)
204 return;
205 tag_new = __pgalloc_tag_get(&new->page);
206 if (!tag_new)
207 return;
208
209 if (!get_page_tag_ref(&old->page, &ref_old, &handle_old))
210 return;
211 if (!get_page_tag_ref(&new->page, &ref_new, &handle_new)) {
212 put_page_tag_ref(handle_old);
213 return;
214 }
215
216 /*
217 * Clear tag references to avoid debug warning when using
218 * __alloc_tag_ref_set() with non-empty reference.
219 */
220 set_codetag_empty(&ref_old);
221 set_codetag_empty(&ref_new);
222
223 /* swap tags */
224 __alloc_tag_ref_set(&ref_old, tag_new);
225 update_page_tag_ref(handle_old, &ref_old);
226 __alloc_tag_ref_set(&ref_new, tag_old);
227 update_page_tag_ref(handle_new, &ref_new);
228
229 put_page_tag_ref(handle_old);
230 put_page_tag_ref(handle_new);
231}
232
233static void shutdown_mem_profiling(bool remove_file)
234{
235 if (mem_alloc_profiling_enabled())
236 static_branch_disable(&mem_alloc_profiling_key);
237
238 if (!mem_profiling_support)
239 return;
240
241 if (remove_file)
242 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
243 mem_profiling_support = false;
244}
245
246void __init alloc_tag_sec_init(void)
247{
248 struct alloc_tag *last_codetag;
249
250 if (!mem_profiling_support)
251 return;
252
253 if (!static_key_enabled(&mem_profiling_compressed))
254 return;
255
256 kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
257 SECTION_START(ALLOC_TAG_SECTION_NAME));
258 last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
259 SECTION_STOP(ALLOC_TAG_SECTION_NAME));
260 kernel_tags.count = last_codetag - kernel_tags.first_tag;
261
262 /* Check if kernel tags fit into page flags */
263 if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
264 shutdown_mem_profiling(remove_file: false); /* allocinfo file does not exist yet */
265 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
266 kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
267 return;
268 }
269
270 alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
271 alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
272 pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
273 NR_UNUSED_PAGEFLAG_BITS);
274}
275
276#ifdef CONFIG_MODULES
277
278static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
279static struct vm_struct *vm_module_tags;
280/* A dummy object used to indicate an unloaded module */
281static struct module unloaded_mod;
282/* A dummy object used to indicate a module prepended area */
283static struct module prepend_mod;
284
285struct alloc_tag_module_section module_tags;
286
287static inline unsigned long alloc_tag_align(unsigned long val)
288{
289 if (!static_key_enabled(&mem_profiling_compressed)) {
290 /* No alignment requirements when we are not indexing the tags */
291 return val;
292 }
293
294 if (val % sizeof(struct alloc_tag) == 0)
295 return val;
296 return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
297}
298
299static bool ensure_alignment(unsigned long align, unsigned int *prepend)
300{
301 if (!static_key_enabled(&mem_profiling_compressed)) {
302 /* No alignment requirements when we are not indexing the tags */
303 return true;
304 }
305
306 /*
307 * If alloc_tag size is not a multiple of required alignment, tag
308 * indexing does not work.
309 */
310 if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
311 return false;
312
313 /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
314 if (*prepend)
315 *prepend = alloc_tag_align(val: *prepend);
316
317 return true;
318}
319
320static inline bool tags_addressable(void)
321{
322 unsigned long tag_idx_count;
323
324 if (!static_key_enabled(&mem_profiling_compressed))
325 return true; /* with page_ext tags are always addressable */
326
327 tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
328 module_tags.size / sizeof(struct alloc_tag);
329
330 return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
331}
332
333static bool needs_section_mem(struct module *mod, unsigned long size)
334{
335 if (!mem_profiling_support)
336 return false;
337
338 return size >= sizeof(struct alloc_tag);
339}
340
341static bool clean_unused_counters(struct alloc_tag *start_tag,
342 struct alloc_tag *end_tag)
343{
344 struct alloc_tag *tag;
345 bool ret = true;
346
347 for (tag = start_tag; tag <= end_tag; tag++) {
348 struct alloc_tag_counters counter;
349
350 if (!tag->counters)
351 continue;
352
353 counter = alloc_tag_read(tag);
354 if (!counter.bytes) {
355 free_percpu(pdata: tag->counters);
356 tag->counters = NULL;
357 } else {
358 ret = false;
359 }
360 }
361
362 return ret;
363}
364
365/* Called with mod_area_mt locked */
366static void clean_unused_module_areas_locked(void)
367{
368 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
369 struct module *val;
370
371 mas_for_each(&mas, val, module_tags.size) {
372 struct alloc_tag *start_tag;
373 struct alloc_tag *end_tag;
374
375 if (val != &unloaded_mod)
376 continue;
377
378 /* Release area if all tags are unused */
379 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
380 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
381 if (clean_unused_counters(start_tag, end_tag))
382 mas_erase(mas: &mas);
383 }
384}
385
386/* Called with mod_area_mt locked */
387static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
388 unsigned long size, unsigned int prepend, unsigned long align)
389{
390 bool cleanup_done = false;
391
392repeat:
393 /* Try finding exact size and hope the start is aligned */
394 if (!mas_empty_area(mas, min: 0, max: section_size - 1, size: prepend + size)) {
395 if (IS_ALIGNED(mas->index + prepend, align))
396 return true;
397
398 /* Try finding larger area to align later */
399 mas_reset(mas);
400 if (!mas_empty_area(mas, min: 0, max: section_size - 1,
401 size: size + prepend + align - 1))
402 return true;
403 }
404
405 /* No free area, try cleanup stale data and repeat the search once */
406 if (!cleanup_done) {
407 clean_unused_module_areas_locked();
408 cleanup_done = true;
409 mas_reset(mas);
410 goto repeat;
411 }
412
413 return false;
414}
415
416static int vm_module_tags_populate(void)
417{
418 unsigned long phys_end = ALIGN_DOWN(module_tags.start_addr, PAGE_SIZE) +
419 (vm_module_tags->nr_pages << PAGE_SHIFT);
420 unsigned long new_end = module_tags.start_addr + module_tags.size;
421
422 if (phys_end < new_end) {
423 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
424 unsigned long old_shadow_end = ALIGN(phys_end, MODULE_ALIGN);
425 unsigned long new_shadow_end = ALIGN(new_end, MODULE_ALIGN);
426 unsigned long more_pages;
427 unsigned long nr = 0;
428
429 more_pages = ALIGN(new_end - phys_end, PAGE_SIZE) >> PAGE_SHIFT;
430 while (nr < more_pages) {
431 unsigned long allocated;
432
433 allocated = alloc_pages_bulk_node(GFP_KERNEL | __GFP_NOWARN,
434 NUMA_NO_NODE, more_pages - nr, next_page + nr);
435
436 if (!allocated)
437 break;
438 nr += allocated;
439 }
440
441 if (nr < more_pages ||
442 vmap_pages_range(addr: phys_end, end: phys_end + (nr << PAGE_SHIFT), PAGE_KERNEL,
443 pages: next_page, PAGE_SHIFT) < 0) {
444 release_pages_arg arg = { .pages = next_page };
445
446 /* Clean up and error out */
447 release_pages(arg, nr);
448 return -ENOMEM;
449 }
450
451 vm_module_tags->nr_pages += nr;
452
453 /*
454 * Kasan allocates 1 byte of shadow for every 8 bytes of data.
455 * When kasan_alloc_module_shadow allocates shadow memory,
456 * its unit of allocation is a page.
457 * Therefore, here we need to align to MODULE_ALIGN.
458 */
459 if (old_shadow_end < new_shadow_end)
460 kasan_alloc_module_shadow(addr: (void *)old_shadow_end,
461 size: new_shadow_end - old_shadow_end,
462 GFP_KERNEL);
463 }
464
465 /*
466 * Mark the pages as accessible, now that they are mapped.
467 * With hardware tag-based KASAN, marking is skipped for
468 * non-VM_ALLOC mappings, see __kasan_unpoison_vmalloc().
469 */
470 kasan_unpoison_vmalloc(start: (void *)module_tags.start_addr,
471 size: new_end - module_tags.start_addr,
472 KASAN_VMALLOC_PROT_NORMAL);
473
474 return 0;
475}
476
477static void *reserve_module_tags(struct module *mod, unsigned long size,
478 unsigned int prepend, unsigned long align)
479{
480 unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
481 MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
482 unsigned long offset;
483 void *ret = NULL;
484
485 /* If no tags return error */
486 if (size < sizeof(struct alloc_tag))
487 return ERR_PTR(error: -EINVAL);
488
489 /*
490 * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
491 * align 0 or 1 means no alignment, to simplify set to 1.
492 */
493 if (!align)
494 align = 1;
495
496 if (!ensure_alignment(align, prepend: &prepend)) {
497 shutdown_mem_profiling(remove_file: true);
498 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
499 mod->name, align);
500 return ERR_PTR(error: -EINVAL);
501 }
502
503 mas_lock(&mas);
504 if (!find_aligned_area(mas: &mas, section_size, size, prepend, align)) {
505 ret = ERR_PTR(error: -ENOMEM);
506 goto unlock;
507 }
508
509 /* Mark found area as reserved */
510 offset = mas.index;
511 offset += prepend;
512 offset = ALIGN(offset, align);
513 if (offset != mas.index) {
514 unsigned long pad_start = mas.index;
515
516 mas.last = offset - 1;
517 mas_store(mas: &mas, entry: &prepend_mod);
518 if (mas_is_err(mas: &mas)) {
519 ret = ERR_PTR(error: xa_err(entry: mas.node));
520 goto unlock;
521 }
522 mas.index = offset;
523 mas.last = offset + size - 1;
524 mas_store(mas: &mas, entry: mod);
525 if (mas_is_err(mas: &mas)) {
526 mas.index = pad_start;
527 mas_erase(mas: &mas);
528 ret = ERR_PTR(error: xa_err(entry: mas.node));
529 }
530 } else {
531 mas.last = offset + size - 1;
532 mas_store(mas: &mas, entry: mod);
533 if (mas_is_err(mas: &mas))
534 ret = ERR_PTR(error: xa_err(entry: mas.node));
535 }
536unlock:
537 mas_unlock(&mas);
538
539 if (IS_ERR(ptr: ret))
540 return ret;
541
542 if (module_tags.size < offset + size) {
543 int grow_res;
544
545 module_tags.size = offset + size;
546 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
547 shutdown_mem_profiling(remove_file: true);
548 pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
549 mod->name, NR_UNUSED_PAGEFLAG_BITS);
550 }
551
552 grow_res = vm_module_tags_populate();
553 if (grow_res) {
554 shutdown_mem_profiling(remove_file: true);
555 pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
556 mod->name);
557 return ERR_PTR(error: grow_res);
558 }
559 }
560
561 return (struct alloc_tag *)(module_tags.start_addr + offset);
562}
563
564static void release_module_tags(struct module *mod, bool used)
565{
566 MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
567 struct alloc_tag *start_tag;
568 struct alloc_tag *end_tag;
569 struct module *val;
570
571 mas_lock(&mas);
572 mas_for_each_rev(&mas, val, 0)
573 if (val == mod)
574 break;
575
576 if (!val) /* module not found */
577 goto out;
578
579 if (!used)
580 goto release_area;
581
582 start_tag = (struct alloc_tag *)(module_tags.start_addr + mas.index);
583 end_tag = (struct alloc_tag *)(module_tags.start_addr + mas.last);
584 if (!clean_unused_counters(start_tag, end_tag)) {
585 struct alloc_tag *tag;
586
587 for (tag = start_tag; tag <= end_tag; tag++) {
588 struct alloc_tag_counters counter;
589
590 if (!tag->counters)
591 continue;
592
593 counter = alloc_tag_read(tag);
594 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
595 tag->ct.filename, tag->ct.lineno, tag->ct.modname,
596 tag->ct.function, counter.bytes);
597 }
598 } else {
599 used = false;
600 }
601release_area:
602 mas_store(mas: &mas, entry: used ? &unloaded_mod : NULL);
603 val = mas_prev_range(mas: &mas, max: 0);
604 if (val == &prepend_mod)
605 mas_store(mas: &mas, NULL);
606out:
607 mas_unlock(&mas);
608}
609
610static int load_module(struct module *mod, struct codetag *start, struct codetag *stop)
611{
612 /* Allocate module alloc_tag percpu counters */
613 struct alloc_tag *start_tag;
614 struct alloc_tag *stop_tag;
615 struct alloc_tag *tag;
616
617 /* percpu counters for core allocations are already statically allocated */
618 if (!mod)
619 return 0;
620
621 start_tag = ct_to_alloc_tag(start);
622 stop_tag = ct_to_alloc_tag(stop);
623 for (tag = start_tag; tag < stop_tag; tag++) {
624 WARN_ON(tag->counters);
625 tag->counters = alloc_percpu(struct alloc_tag_counters);
626 if (!tag->counters) {
627 while (--tag >= start_tag) {
628 free_percpu(pdata: tag->counters);
629 tag->counters = NULL;
630 }
631 pr_err("Failed to allocate memory for allocation tag percpu counters in the module %s\n",
632 mod->name);
633 return -ENOMEM;
634 }
635
636 /*
637 * Avoid a kmemleak false positive. The pointer to the counters is stored
638 * in the alloc_tag section of the module and cannot be directly accessed.
639 */
640 kmemleak_ignore_percpu(ptr: tag->counters);
641 }
642 return 0;
643}
644
645static void replace_module(struct module *mod, struct module *new_mod)
646{
647 MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
648 struct module *val;
649
650 mas_lock(&mas);
651 mas_for_each(&mas, val, module_tags.size) {
652 if (val != mod)
653 continue;
654
655 mas_store_gfp(mas: &mas, entry: new_mod, GFP_KERNEL);
656 break;
657 }
658 mas_unlock(&mas);
659}
660
661static int __init alloc_mod_tags_mem(void)
662{
663 /* Map space to copy allocation tags */
664 vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
665 if (!vm_module_tags) {
666 pr_err("Failed to map %lu bytes for module allocation tags\n",
667 MODULE_ALLOC_TAG_VMAP_SIZE);
668 module_tags.start_addr = 0;
669 return -ENOMEM;
670 }
671
672 vm_module_tags->pages = kmalloc_array(get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
673 sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
674 if (!vm_module_tags->pages) {
675 free_vm_area(area: vm_module_tags);
676 return -ENOMEM;
677 }
678
679 module_tags.start_addr = (unsigned long)vm_module_tags->addr;
680 module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
681 /* Ensure the base is alloc_tag aligned when required for indexing */
682 module_tags.start_addr = alloc_tag_align(val: module_tags.start_addr);
683
684 return 0;
685}
686
687static void __init free_mod_tags_mem(void)
688{
689 release_pages_arg arg = { .pages = vm_module_tags->pages };
690
691 module_tags.start_addr = 0;
692 release_pages(arg, nr: vm_module_tags->nr_pages);
693 kfree(objp: vm_module_tags->pages);
694 free_vm_area(area: vm_module_tags);
695}
696
697#else /* CONFIG_MODULES */
698
699static inline int alloc_mod_tags_mem(void) { return 0; }
700static inline void free_mod_tags_mem(void) {}
701
702#endif /* CONFIG_MODULES */
703
704/* See: Documentation/mm/allocation-profiling.rst */
705static int __init setup_early_mem_profiling(char *str)
706{
707 bool compressed = false;
708 bool enable;
709
710 if (!str || !str[0])
711 return -EINVAL;
712
713 if (!strncmp(str, "never", 5)) {
714 enable = false;
715 mem_profiling_support = false;
716 pr_info("Memory allocation profiling is disabled!\n");
717 } else {
718 char *token = strsep(&str, ",");
719
720 if (kstrtobool(s: token, res: &enable))
721 return -EINVAL;
722
723 if (str) {
724
725 if (strcmp(str, "compressed"))
726 return -EINVAL;
727
728 compressed = true;
729 }
730 mem_profiling_support = true;
731 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
732 compressed ? "with" : "without", str_on_off(enable));
733 }
734
735 if (enable != mem_alloc_profiling_enabled()) {
736 if (enable)
737 static_branch_enable(&mem_alloc_profiling_key);
738 else
739 static_branch_disable(&mem_alloc_profiling_key);
740 }
741 if (compressed != static_key_enabled(&mem_profiling_compressed)) {
742 if (compressed)
743 static_branch_enable(&mem_profiling_compressed);
744 else
745 static_branch_disable(&mem_profiling_compressed);
746 }
747
748 return 0;
749}
750early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
751
752static __init bool need_page_alloc_tagging(void)
753{
754 if (static_key_enabled(&mem_profiling_compressed))
755 return false;
756
757 return mem_profiling_support;
758}
759
760static __init void init_page_alloc_tagging(void)
761{
762}
763
764struct page_ext_operations page_alloc_tagging_ops = {
765 .size = sizeof(union codetag_ref),
766 .need = need_page_alloc_tagging,
767 .init = init_page_alloc_tagging,
768};
769EXPORT_SYMBOL(page_alloc_tagging_ops);
770
771#ifdef CONFIG_SYSCTL
772/*
773 * Not using proc_do_static_key() directly to prevent enabling profiling
774 * after it was shut down.
775 */
776static int proc_mem_profiling_handler(const struct ctl_table *table, int write,
777 void *buffer, size_t *lenp, loff_t *ppos)
778{
779 if (!mem_profiling_support && write)
780 return -EINVAL;
781
782 return proc_do_static_key(table, write, buffer, lenp, ppos);
783}
784
785
786static struct ctl_table memory_allocation_profiling_sysctls[] = {
787 {
788 .procname = "mem_profiling",
789 .data = &mem_alloc_profiling_key,
790#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
791 .mode = 0444,
792#else
793 .mode = 0644,
794#endif
795 .proc_handler = proc_mem_profiling_handler,
796 },
797};
798
799static void __init sysctl_init(void)
800{
801 if (!mem_profiling_support)
802 memory_allocation_profiling_sysctls[0].mode = 0444;
803
804 register_sysctl_init("vm", memory_allocation_profiling_sysctls);
805}
806#else /* CONFIG_SYSCTL */
807static inline void sysctl_init(void) {}
808#endif /* CONFIG_SYSCTL */
809
810static int __init alloc_tag_init(void)
811{
812 const struct codetag_type_desc desc = {
813 .section = ALLOC_TAG_SECTION_NAME,
814 .tag_size = sizeof(struct alloc_tag),
815#ifdef CONFIG_MODULES
816 .needs_section_mem = needs_section_mem,
817 .alloc_section_mem = reserve_module_tags,
818 .free_section_mem = release_module_tags,
819 .module_load = load_module,
820 .module_replaced = replace_module,
821#endif
822 };
823 int res;
824
825 sysctl_init();
826
827 if (!mem_profiling_support) {
828 pr_info("Memory allocation profiling is not supported!\n");
829 return 0;
830 }
831
832 if (!proc_create_seq_private(ALLOCINFO_FILE_NAME, mode: 0400, NULL, ops: &allocinfo_seq_op,
833 state_size: sizeof(struct allocinfo_private), NULL)) {
834 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
835 shutdown_mem_profiling(remove_file: false);
836 return -ENOMEM;
837 }
838
839 res = alloc_mod_tags_mem();
840 if (res) {
841 pr_err("Failed to reserve address space for module tags, errno = %d\n", res);
842 shutdown_mem_profiling(remove_file: true);
843 return res;
844 }
845
846 alloc_tag_cttype = codetag_register_type(desc: &desc);
847 if (IS_ERR(ptr: alloc_tag_cttype)) {
848 pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
849 free_mod_tags_mem();
850 shutdown_mem_profiling(remove_file: true);
851 return PTR_ERR(ptr: alloc_tag_cttype);
852 }
853
854 return 0;
855}
856module_init(alloc_tag_init);
857

source code of linux/lib/alloc_tag.c