1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Bootmem core functions.
4 *
5 * Copyright (c) 2020, Bytedance.
6 *
7 * Author: Muchun Song <songmuchun@bytedance.com>
8 *
9 */
10#include <linux/mm.h>
11#include <linux/compiler.h>
12#include <linux/memblock.h>
13#include <linux/bootmem_info.h>
14#include <linux/memory_hotplug.h>
15#include <linux/kmemleak.h>
16
17void get_page_bootmem(unsigned long info, struct page *page,
18 enum bootmem_type type)
19{
20 BUG_ON(type > 0xf);
21 BUG_ON(info > (ULONG_MAX >> 4));
22 SetPagePrivate(page);
23 set_page_private(page, private: info << 4 | type);
24 page_ref_inc(page);
25}
26
27void put_page_bootmem(struct page *page)
28{
29 enum bootmem_type type = bootmem_type(page);
30
31 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
32 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
33
34 if (page_ref_dec_return(page) == 1) {
35 ClearPagePrivate(page);
36 set_page_private(page, private: 0);
37 INIT_LIST_HEAD(list: &page->lru);
38 kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
39 free_reserved_page(page);
40 }
41}
42
43#ifndef CONFIG_SPARSEMEM_VMEMMAP
44static void __init register_page_bootmem_info_section(unsigned long start_pfn)
45{
46 unsigned long mapsize, section_nr, i;
47 struct mem_section *ms;
48 struct page *page, *memmap;
49 struct mem_section_usage *usage;
50
51 section_nr = pfn_to_section_nr(start_pfn);
52 ms = __nr_to_section(section_nr);
53
54 /* Get section's memmap address */
55 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
56
57 /*
58 * Get page for the memmap's phys address
59 * XXX: need more consideration for sparse_vmemmap...
60 */
61 page = virt_to_page(memmap);
62 mapsize = sizeof(struct page) * PAGES_PER_SECTION;
63 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
64
65 /* remember memmap's page */
66 for (i = 0; i < mapsize; i++, page++)
67 get_page_bootmem(section_nr, page, SECTION_INFO);
68
69 usage = ms->usage;
70 page = virt_to_page(usage);
71
72 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
73
74 for (i = 0; i < mapsize; i++, page++)
75 get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
76
77}
78#else /* CONFIG_SPARSEMEM_VMEMMAP */
79static void __init register_page_bootmem_info_section(unsigned long start_pfn)
80{
81 unsigned long mapsize, section_nr, i;
82 struct mem_section *ms;
83 struct page *page, *memmap;
84 struct mem_section_usage *usage;
85
86 section_nr = pfn_to_section_nr(pfn: start_pfn);
87 ms = __nr_to_section(nr: section_nr);
88
89 memmap = sparse_decode_mem_map(coded_mem_map: ms->section_mem_map, pnum: section_nr);
90
91 if (!preinited_vmemmap_section(section: ms))
92 register_page_bootmem_memmap(section_nr, map: memmap,
93 PAGES_PER_SECTION);
94
95 usage = ms->usage;
96 page = virt_to_page(usage);
97
98 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
99
100 for (i = 0; i < mapsize; i++, page++)
101 get_page_bootmem(info: section_nr, page, type: MIX_SECTION_INFO);
102}
103#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
104
105void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
106{
107 unsigned long i, pfn, end_pfn, nr_pages;
108 int node = pgdat->node_id;
109 struct page *page;
110
111 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
112 page = virt_to_page(pgdat);
113
114 for (i = 0; i < nr_pages; i++, page++)
115 get_page_bootmem(info: node, page, type: NODE_INFO);
116
117 pfn = pgdat->node_start_pfn;
118 end_pfn = pgdat_end_pfn(pgdat);
119
120 /* register section info */
121 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
122 /*
123 * Some platforms can assign the same pfn to multiple nodes - on
124 * node0 as well as nodeN. To avoid registering a pfn against
125 * multiple nodes we check that this pfn does not already
126 * reside in some other nodes.
127 */
128 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
129 register_page_bootmem_info_section(start_pfn: pfn);
130 }
131}
132

source code of linux/mm/bootmem_info.c