1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Memory subsystem support
4 *
5 * Written by Matt Tolentino <matthew.e.tolentino@intel.com>
6 * Dave Hansen <haveblue@us.ibm.com>
7 *
8 * This file provides the necessary infrastructure to represent
9 * a SPARSEMEM-memory-model system's physical memory in /sysfs.
10 * All arch-independent code that assumes MEMORY_HOTPLUG requires
11 * SPARSEMEM should be contained here, or in mm/memory_hotplug.c.
12 */
13
14#include <linux/module.h>
15#include <linux/init.h>
16#include <linux/topology.h>
17#include <linux/capability.h>
18#include <linux/device.h>
19#include <linux/memory.h>
20#include <linux/memory_hotplug.h>
21#include <linux/mm.h>
22#include <linux/stat.h>
23#include <linux/slab.h>
24#include <linux/xarray.h>
25#include <linux/export.h>
26
27#include <linux/atomic.h>
28#include <linux/uaccess.h>
29
30#define MEMORY_CLASS_NAME "memory"
31
32static const char *const online_type_to_str[] = {
33 [MMOP_OFFLINE] = "offline",
34 [MMOP_ONLINE] = "online",
35 [MMOP_ONLINE_KERNEL] = "online_kernel",
36 [MMOP_ONLINE_MOVABLE] = "online_movable",
37};
38
39int mhp_online_type_from_str(const char *str)
40{
41 int i;
42
43 for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) {
44 if (sysfs_streq(s1: str, s2: online_type_to_str[i]))
45 return i;
46 }
47 return -EINVAL;
48}
49
50#define to_memory_block(dev) container_of(dev, struct memory_block, dev)
51
52int sections_per_block;
53EXPORT_SYMBOL(sections_per_block);
54
55static int memory_subsys_online(struct device *dev);
56static int memory_subsys_offline(struct device *dev);
57
58static const struct bus_type memory_subsys = {
59 .name = MEMORY_CLASS_NAME,
60 .dev_name = MEMORY_CLASS_NAME,
61 .online = memory_subsys_online,
62 .offline = memory_subsys_offline,
63};
64
65/*
66 * Memory blocks are cached in a local radix tree to avoid
67 * a costly linear search for the corresponding device on
68 * the subsystem bus.
69 */
70static DEFINE_XARRAY(memory_blocks);
71
72/*
73 * Memory groups, indexed by memory group id (mgid).
74 */
75static DEFINE_XARRAY_FLAGS(memory_groups, XA_FLAGS_ALLOC);
76#define MEMORY_GROUP_MARK_DYNAMIC XA_MARK_1
77
78static BLOCKING_NOTIFIER_HEAD(memory_chain);
79
80int register_memory_notifier(struct notifier_block *nb)
81{
82 return blocking_notifier_chain_register(nh: &memory_chain, nb);
83}
84EXPORT_SYMBOL(register_memory_notifier);
85
86void unregister_memory_notifier(struct notifier_block *nb)
87{
88 blocking_notifier_chain_unregister(nh: &memory_chain, nb);
89}
90EXPORT_SYMBOL(unregister_memory_notifier);
91
92static void memory_block_release(struct device *dev)
93{
94 struct memory_block *mem = to_memory_block(dev);
95 /* Verify that the altmap is freed */
96 WARN_ON(mem->altmap);
97 kfree(objp: mem);
98}
99
100
101/* Max block size to be set by memory_block_advise_max_size */
102static unsigned long memory_block_advised_size;
103static bool memory_block_advised_size_queried;
104
105/**
106 * memory_block_advise_max_size() - advise memory hotplug on the max suggested
107 * block size, usually for alignment.
108 * @size: suggestion for maximum block size. must be aligned on power of 2.
109 *
110 * Early boot software (pre-allocator init) may advise archs on the max block
111 * size. This value can only decrease after initialization, as the intent is
112 * to identify the largest supported alignment for all sources.
113 *
114 * Use of this value is arch-defined, as is min/max block size.
115 *
116 * Return: 0 on success
117 * -EINVAL if size is 0 or not pow2 aligned
118 * -EBUSY if value has already been probed
119 */
120int __init memory_block_advise_max_size(unsigned long size)
121{
122 if (!size || !is_power_of_2(n: size))
123 return -EINVAL;
124
125 if (memory_block_advised_size_queried)
126 return -EBUSY;
127
128 if (memory_block_advised_size)
129 memory_block_advised_size = min(memory_block_advised_size, size);
130 else
131 memory_block_advised_size = size;
132
133 return 0;
134}
135
136/**
137 * memory_block_advised_max_size() - query advised max hotplug block size.
138 *
139 * After the first call, the value can never change. Callers looking for the
140 * actual block size should use memory_block_size_bytes. This interface is
141 * intended for use by arch-init when initializing the hotplug block size.
142 *
143 * Return: advised size in bytes, or 0 if never set.
144 */
145unsigned long memory_block_advised_max_size(void)
146{
147 memory_block_advised_size_queried = true;
148 return memory_block_advised_size;
149}
150
151unsigned long __weak memory_block_size_bytes(void)
152{
153 return MIN_MEMORY_BLOCK_SIZE;
154}
155EXPORT_SYMBOL_GPL(memory_block_size_bytes);
156
157/* Show the memory block ID, relative to the memory block size */
158static ssize_t phys_index_show(struct device *dev,
159 struct device_attribute *attr, char *buf)
160{
161 struct memory_block *mem = to_memory_block(dev);
162
163 return sysfs_emit(buf, fmt: "%08lx\n", memory_block_id(section_nr: mem->start_section_nr));
164}
165
166/*
167 * Legacy interface that we cannot remove. Always indicate "removable"
168 * with CONFIG_MEMORY_HOTREMOVE - bad heuristic.
169 */
170static ssize_t removable_show(struct device *dev, struct device_attribute *attr,
171 char *buf)
172{
173 return sysfs_emit(buf, fmt: "%d\n", (int)IS_ENABLED(CONFIG_MEMORY_HOTREMOVE));
174}
175
176/*
177 * online, offline, going offline, etc.
178 */
179static ssize_t state_show(struct device *dev, struct device_attribute *attr,
180 char *buf)
181{
182 struct memory_block *mem = to_memory_block(dev);
183 const char *output;
184
185 /*
186 * We can probably put these states in a nice little array
187 * so that they're not open-coded
188 */
189 switch (mem->state) {
190 case MEM_ONLINE:
191 output = "online";
192 break;
193 case MEM_OFFLINE:
194 output = "offline";
195 break;
196 case MEM_GOING_OFFLINE:
197 output = "going-offline";
198 break;
199 default:
200 WARN_ON(1);
201 return sysfs_emit(buf, fmt: "ERROR-UNKNOWN-%d\n", mem->state);
202 }
203
204 return sysfs_emit(buf, fmt: "%s\n", output);
205}
206
207int memory_notify(enum memory_block_state state, void *v)
208{
209 return blocking_notifier_call_chain(nh: &memory_chain, val: state, v);
210}
211
212#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)
213static unsigned long memblk_nr_poison(struct memory_block *mem);
214#else
215static inline unsigned long memblk_nr_poison(struct memory_block *mem)
216{
217 return 0;
218}
219#endif
220
221/*
222 * Must acquire mem_hotplug_lock in write mode.
223 */
224static int memory_block_online(struct memory_block *mem)
225{
226 unsigned long start_pfn = section_nr_to_pfn(sec: mem->start_section_nr);
227 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
228 unsigned long nr_vmemmap_pages = 0;
229 struct zone *zone;
230 int ret;
231
232 if (memblk_nr_poison(mem))
233 return -EHWPOISON;
234
235 zone = zone_for_pfn_range(online_type: mem->online_type, nid: mem->nid, group: mem->group,
236 start_pfn, nr_pages);
237
238 /*
239 * Although vmemmap pages have a different lifecycle than the pages
240 * they describe (they remain until the memory is unplugged), doing
241 * their initialization and accounting at memory onlining/offlining
242 * stage helps to keep accounting easier to follow - e.g vmemmaps
243 * belong to the same zone as the memory they backed.
244 */
245 if (mem->altmap)
246 nr_vmemmap_pages = mem->altmap->free;
247
248 mem_hotplug_begin();
249 if (nr_vmemmap_pages) {
250 ret = mhp_init_memmap_on_memory(pfn: start_pfn, nr_pages: nr_vmemmap_pages, zone);
251 if (ret)
252 goto out;
253 }
254
255 ret = online_pages(pfn: start_pfn + nr_vmemmap_pages,
256 nr_pages: nr_pages - nr_vmemmap_pages, zone, group: mem->group);
257 if (ret) {
258 if (nr_vmemmap_pages)
259 mhp_deinit_memmap_on_memory(pfn: start_pfn, nr_pages: nr_vmemmap_pages);
260 goto out;
261 }
262
263 /*
264 * Account once onlining succeeded. If the zone was unpopulated, it is
265 * now already properly populated.
266 */
267 if (nr_vmemmap_pages)
268 adjust_present_page_count(pfn_to_page(start_pfn), group: mem->group,
269 nr_pages: nr_vmemmap_pages);
270
271 mem->zone = zone;
272out:
273 mem_hotplug_done();
274 return ret;
275}
276
277/*
278 * Must acquire mem_hotplug_lock in write mode.
279 */
280static int memory_block_offline(struct memory_block *mem)
281{
282 unsigned long start_pfn = section_nr_to_pfn(sec: mem->start_section_nr);
283 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
284 unsigned long nr_vmemmap_pages = 0;
285 int ret;
286
287 if (!mem->zone)
288 return -EINVAL;
289
290 /*
291 * Unaccount before offlining, such that unpopulated zone and kthreads
292 * can properly be torn down in offline_pages().
293 */
294 if (mem->altmap)
295 nr_vmemmap_pages = mem->altmap->free;
296
297 mem_hotplug_begin();
298 if (nr_vmemmap_pages)
299 adjust_present_page_count(pfn_to_page(start_pfn), group: mem->group,
300 nr_pages: -nr_vmemmap_pages);
301
302 ret = offline_pages(start_pfn: start_pfn + nr_vmemmap_pages,
303 nr_pages: nr_pages - nr_vmemmap_pages, zone: mem->zone, group: mem->group);
304 if (ret) {
305 /* offline_pages() failed. Account back. */
306 if (nr_vmemmap_pages)
307 adjust_present_page_count(pfn_to_page(start_pfn),
308 group: mem->group, nr_pages: nr_vmemmap_pages);
309 goto out;
310 }
311
312 if (nr_vmemmap_pages)
313 mhp_deinit_memmap_on_memory(pfn: start_pfn, nr_pages: nr_vmemmap_pages);
314
315 mem->zone = NULL;
316out:
317 mem_hotplug_done();
318 return ret;
319}
320
321/*
322 * MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is
323 * OK to have direct references to sparsemem variables in here.
324 */
325static int
326memory_block_action(struct memory_block *mem, unsigned long action)
327{
328 int ret;
329
330 switch (action) {
331 case MEM_ONLINE:
332 ret = memory_block_online(mem);
333 break;
334 case MEM_OFFLINE:
335 ret = memory_block_offline(mem);
336 break;
337 default:
338 WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "
339 "%ld\n", __func__, mem->start_section_nr, action, action);
340 ret = -EINVAL;
341 }
342
343 return ret;
344}
345
346static int memory_block_change_state(struct memory_block *mem,
347 unsigned long to_state, unsigned long from_state_req)
348{
349 int ret = 0;
350
351 if (mem->state != from_state_req)
352 return -EINVAL;
353
354 if (to_state == MEM_OFFLINE)
355 mem->state = MEM_GOING_OFFLINE;
356
357 ret = memory_block_action(mem, action: to_state);
358 mem->state = ret ? from_state_req : to_state;
359
360 return ret;
361}
362
363/* The device lock serializes operations on memory_subsys_[online|offline] */
364static int memory_subsys_online(struct device *dev)
365{
366 struct memory_block *mem = to_memory_block(dev);
367 int ret;
368
369 if (mem->state == MEM_ONLINE)
370 return 0;
371
372 /*
373 * When called via device_online() without configuring the online_type,
374 * we want to default to MMOP_ONLINE.
375 */
376 if (mem->online_type == MMOP_OFFLINE)
377 mem->online_type = MMOP_ONLINE;
378
379 ret = memory_block_change_state(mem, to_state: MEM_ONLINE, from_state_req: MEM_OFFLINE);
380 mem->online_type = MMOP_OFFLINE;
381
382 return ret;
383}
384
385static int memory_subsys_offline(struct device *dev)
386{
387 struct memory_block *mem = to_memory_block(dev);
388
389 if (mem->state == MEM_OFFLINE)
390 return 0;
391
392 return memory_block_change_state(mem, to_state: MEM_OFFLINE, from_state_req: MEM_ONLINE);
393}
394
395static ssize_t state_store(struct device *dev, struct device_attribute *attr,
396 const char *buf, size_t count)
397{
398 const int online_type = mhp_online_type_from_str(str: buf);
399 struct memory_block *mem = to_memory_block(dev);
400 int ret;
401
402 if (online_type < 0)
403 return -EINVAL;
404
405 ret = lock_device_hotplug_sysfs();
406 if (ret)
407 return ret;
408
409 switch (online_type) {
410 case MMOP_ONLINE_KERNEL:
411 case MMOP_ONLINE_MOVABLE:
412 case MMOP_ONLINE:
413 /* mem->online_type is protected by device_hotplug_lock */
414 mem->online_type = online_type;
415 ret = device_online(dev: &mem->dev);
416 break;
417 case MMOP_OFFLINE:
418 ret = device_offline(dev: &mem->dev);
419 break;
420 default:
421 ret = -EINVAL; /* should never happen */
422 }
423
424 unlock_device_hotplug();
425
426 if (ret < 0)
427 return ret;
428 if (ret)
429 return -EINVAL;
430
431 return count;
432}
433
434/*
435 * Legacy interface that we cannot remove: s390x exposes the storage increment
436 * covered by a memory block, allowing for identifying which memory blocks
437 * comprise a storage increment. Since a memory block spans complete
438 * storage increments nowadays, this interface is basically unused. Other
439 * archs never exposed != 0.
440 */
441static ssize_t phys_device_show(struct device *dev,
442 struct device_attribute *attr, char *buf)
443{
444 struct memory_block *mem = to_memory_block(dev);
445 unsigned long start_pfn = section_nr_to_pfn(sec: mem->start_section_nr);
446
447 return sysfs_emit(buf, fmt: "%d\n",
448 arch_get_memory_phys_device(start_pfn));
449}
450
451#ifdef CONFIG_MEMORY_HOTREMOVE
452static int print_allowed_zone(char *buf, int len, int nid,
453 struct memory_group *group,
454 unsigned long start_pfn, unsigned long nr_pages,
455 int online_type, struct zone *default_zone)
456{
457 struct zone *zone;
458
459 zone = zone_for_pfn_range(online_type, nid, group, start_pfn, nr_pages);
460 if (zone == default_zone)
461 return 0;
462
463 return sysfs_emit_at(buf, at: len, fmt: " %s", zone->name);
464}
465
466static ssize_t valid_zones_show(struct device *dev,
467 struct device_attribute *attr, char *buf)
468{
469 struct memory_block *mem = to_memory_block(dev);
470 unsigned long start_pfn = section_nr_to_pfn(sec: mem->start_section_nr);
471 unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
472 struct memory_group *group = mem->group;
473 struct zone *default_zone;
474 int nid = mem->nid;
475 int len;
476
477 /*
478 * Check the existing zone. Make sure that we do that only on the
479 * online nodes otherwise the page_zone is not reliable
480 */
481 if (mem->state == MEM_ONLINE) {
482 /*
483 * If !mem->zone, the memory block spans multiple zones and
484 * cannot get offlined.
485 */
486 return sysfs_emit(buf, fmt: "%s\n",
487 mem->zone ? mem->zone->name : "none");
488 }
489
490 default_zone = zone_for_pfn_range(online_type: MMOP_ONLINE, nid, group,
491 start_pfn, nr_pages);
492
493 len = sysfs_emit(buf, fmt: "%s", default_zone->name);
494 len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages,
495 online_type: MMOP_ONLINE_KERNEL, default_zone);
496 len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages,
497 online_type: MMOP_ONLINE_MOVABLE, default_zone);
498 len += sysfs_emit_at(buf, at: len, fmt: "\n");
499 return len;
500}
501static DEVICE_ATTR_RO(valid_zones);
502#endif
503
504static DEVICE_ATTR_RO(phys_index);
505static DEVICE_ATTR_RW(state);
506static DEVICE_ATTR_RO(phys_device);
507static DEVICE_ATTR_RO(removable);
508
509/*
510 * Show the memory block size (shared by all memory blocks).
511 */
512static ssize_t block_size_bytes_show(struct device *dev,
513 struct device_attribute *attr, char *buf)
514{
515 return sysfs_emit(buf, fmt: "%lx\n", memory_block_size_bytes());
516}
517
518static DEVICE_ATTR_RO(block_size_bytes);
519
520/*
521 * Memory auto online policy.
522 */
523
524static ssize_t auto_online_blocks_show(struct device *dev,
525 struct device_attribute *attr, char *buf)
526{
527 return sysfs_emit(buf, fmt: "%s\n",
528 online_type_to_str[mhp_get_default_online_type()]);
529}
530
531static ssize_t auto_online_blocks_store(struct device *dev,
532 struct device_attribute *attr,
533 const char *buf, size_t count)
534{
535 const int online_type = mhp_online_type_from_str(str: buf);
536
537 if (online_type < 0)
538 return -EINVAL;
539
540 mhp_set_default_online_type(online_type);
541 return count;
542}
543
544static DEVICE_ATTR_RW(auto_online_blocks);
545
546#ifdef CONFIG_CRASH_HOTPLUG
547#include <linux/kexec.h>
548static ssize_t crash_hotplug_show(struct device *dev,
549 struct device_attribute *attr, char *buf)
550{
551 return sysfs_emit(buf, fmt: "%d\n", crash_check_hotplug_support());
552}
553static DEVICE_ATTR_RO(crash_hotplug);
554#endif
555
556/*
557 * Some architectures will have custom drivers to do this, and
558 * will not need to do it from userspace. The fake hot-add code
559 * as well as ppc64 will do all of their discovery in userspace
560 * and will require this interface.
561 */
562#ifdef CONFIG_ARCH_MEMORY_PROBE
563static ssize_t probe_store(struct device *dev, struct device_attribute *attr,
564 const char *buf, size_t count)
565{
566 u64 phys_addr;
567 int nid, ret;
568 unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;
569
570 ret = kstrtoull(s: buf, base: 0, res: &phys_addr);
571 if (ret)
572 return ret;
573
574 if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))
575 return -EINVAL;
576
577 ret = lock_device_hotplug_sysfs();
578 if (ret)
579 return ret;
580
581 nid = memory_add_physaddr_to_nid(start: phys_addr);
582 ret = __add_memory(nid, start: phys_addr,
583 MIN_MEMORY_BLOCK_SIZE * sections_per_block,
584 MHP_NONE);
585
586 if (ret)
587 goto out;
588
589 ret = count;
590out:
591 unlock_device_hotplug();
592 return ret;
593}
594
595static DEVICE_ATTR_WO(probe);
596#endif
597
598#ifdef CONFIG_MEMORY_FAILURE
599/*
600 * Support for offlining pages of memory
601 */
602
603/* Soft offline a page */
604static ssize_t soft_offline_page_store(struct device *dev,
605 struct device_attribute *attr,
606 const char *buf, size_t count)
607{
608 int ret;
609 u64 pfn;
610 if (!capable(CAP_SYS_ADMIN))
611 return -EPERM;
612 if (kstrtoull(s: buf, base: 0, res: &pfn) < 0)
613 return -EINVAL;
614 pfn >>= PAGE_SHIFT;
615 ret = soft_offline_page(pfn, flags: 0);
616 return ret == 0 ? count : ret;
617}
618
619/* Forcibly offline a page, including killing processes. */
620static ssize_t hard_offline_page_store(struct device *dev,
621 struct device_attribute *attr,
622 const char *buf, size_t count)
623{
624 int ret;
625 u64 pfn;
626 if (!capable(CAP_SYS_ADMIN))
627 return -EPERM;
628 if (kstrtoull(s: buf, base: 0, res: &pfn) < 0)
629 return -EINVAL;
630 pfn >>= PAGE_SHIFT;
631 ret = memory_failure(pfn, flags: MF_SW_SIMULATED);
632 if (ret == -EOPNOTSUPP)
633 ret = 0;
634 return ret ? ret : count;
635}
636
637static DEVICE_ATTR_WO(soft_offline_page);
638static DEVICE_ATTR_WO(hard_offline_page);
639#endif
640
641/* See phys_device_show(). */
642int __weak arch_get_memory_phys_device(unsigned long start_pfn)
643{
644 return 0;
645}
646
647/*
648 * A reference for the returned memory block device is acquired.
649 *
650 * Called under device_hotplug_lock.
651 */
652struct memory_block *find_memory_block_by_id(unsigned long block_id)
653{
654 struct memory_block *mem;
655
656 mem = xa_load(&memory_blocks, index: block_id);
657 if (mem)
658 get_device(dev: &mem->dev);
659 return mem;
660}
661
662/*
663 * Called under device_hotplug_lock.
664 */
665struct memory_block *find_memory_block(unsigned long section_nr)
666{
667 unsigned long block_id = memory_block_id(section_nr);
668
669 return find_memory_block_by_id(block_id);
670}
671
672static struct attribute *memory_memblk_attrs[] = {
673 &dev_attr_phys_index.attr,
674 &dev_attr_state.attr,
675 &dev_attr_phys_device.attr,
676 &dev_attr_removable.attr,
677#ifdef CONFIG_MEMORY_HOTREMOVE
678 &dev_attr_valid_zones.attr,
679#endif
680 NULL
681};
682
683static const struct attribute_group memory_memblk_attr_group = {
684 .attrs = memory_memblk_attrs,
685};
686
687static const struct attribute_group *memory_memblk_attr_groups[] = {
688 &memory_memblk_attr_group,
689 NULL,
690};
691
692static int __add_memory_block(struct memory_block *memory)
693{
694 int ret;
695
696 memory->dev.bus = &memory_subsys;
697 memory->dev.id = memory->start_section_nr / sections_per_block;
698 memory->dev.release = memory_block_release;
699 memory->dev.groups = memory_memblk_attr_groups;
700 memory->dev.offline = memory->state == MEM_OFFLINE;
701
702 ret = device_register(dev: &memory->dev);
703 if (ret) {
704 put_device(dev: &memory->dev);
705 return ret;
706 }
707 ret = xa_err(entry: xa_store(&memory_blocks, index: memory->dev.id, entry: memory,
708 GFP_KERNEL));
709 if (ret)
710 device_unregister(dev: &memory->dev);
711
712 return ret;
713}
714
715static struct zone *early_node_zone_for_memory_block(struct memory_block *mem,
716 int nid)
717{
718 const unsigned long start_pfn = section_nr_to_pfn(sec: mem->start_section_nr);
719 const unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;
720 struct zone *zone, *matching_zone = NULL;
721 pg_data_t *pgdat = NODE_DATA(nid);
722 int i;
723
724 /*
725 * This logic only works for early memory, when the applicable zones
726 * already span the memory block. We don't expect overlapping zones on
727 * a single node for early memory. So if we're told that some PFNs
728 * of a node fall into this memory block, we can assume that all node
729 * zones that intersect with the memory block are actually applicable.
730 * No need to look at the memmap.
731 */
732 for (i = 0; i < MAX_NR_ZONES; i++) {
733 zone = pgdat->node_zones + i;
734 if (!populated_zone(zone))
735 continue;
736 if (!zone_intersects(zone, start_pfn, nr_pages))
737 continue;
738 if (!matching_zone) {
739 matching_zone = zone;
740 continue;
741 }
742 /* Spans multiple zones ... */
743 matching_zone = NULL;
744 break;
745 }
746 return matching_zone;
747}
748
749#ifdef CONFIG_NUMA
750/**
751 * memory_block_add_nid_early() - Indicate that early system RAM falling into
752 * this memory block device (partially) belongs
753 * to the given node.
754 * @mem: The memory block device.
755 * @nid: The node id.
756 *
757 * Indicate that early system RAM falling into this memory block (partially)
758 * belongs to the given node. This will also properly set/adjust mem->zone based
759 * on the zone ranges of the given node.
760 *
761 * Memory hotplug handles this on memory block creation, where we can only have
762 * a single nid span a memory block.
763 */
764void memory_block_add_nid_early(struct memory_block *mem, int nid)
765{
766 if (mem->nid != nid) {
767 /*
768 * For early memory we have to determine the zone when setting
769 * the node id and handle multiple nodes spanning a single
770 * memory block by indicate via zone == NULL that we're not
771 * dealing with a single zone. So if we're setting the node id
772 * the first time, determine if there is a single zone. If we're
773 * setting the node id a second time to a different node,
774 * invalidate the single detected zone.
775 */
776 if (mem->nid == NUMA_NO_NODE)
777 mem->zone = early_node_zone_for_memory_block(mem, nid);
778 else
779 mem->zone = NULL;
780 /*
781 * If this memory block spans multiple nodes, we only indicate
782 * the last processed node. If we span multiple nodes (not applicable
783 * to hotplugged memory), zone == NULL will prohibit memory offlining
784 * and consequently unplug.
785 */
786 mem->nid = nid;
787 }
788}
789#endif
790
791static int add_memory_block(unsigned long block_id, int nid, unsigned long state,
792 struct vmem_altmap *altmap,
793 struct memory_group *group)
794{
795 struct memory_block *mem;
796 int ret = 0;
797
798 mem = find_memory_block_by_id(block_id);
799 if (mem) {
800 put_device(dev: &mem->dev);
801 return -EEXIST;
802 }
803 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
804 if (!mem)
805 return -ENOMEM;
806
807 mem->start_section_nr = block_id * sections_per_block;
808 mem->state = state;
809 mem->nid = nid;
810 mem->altmap = altmap;
811 INIT_LIST_HEAD(list: &mem->group_next);
812
813#ifndef CONFIG_NUMA
814 if (state == MEM_ONLINE)
815 /*
816 * MEM_ONLINE at this point implies early memory. With NUMA,
817 * we'll determine the zone when setting the node id via
818 * memory_block_add_nid(). Memory hotplug updated the zone
819 * manually when memory onlining/offlining succeeds.
820 */
821 mem->zone = early_node_zone_for_memory_block(mem, NUMA_NO_NODE);
822#endif /* CONFIG_NUMA */
823
824 ret = __add_memory_block(memory: mem);
825 if (ret)
826 return ret;
827
828 if (group) {
829 mem->group = group;
830 list_add(new: &mem->group_next, head: &group->memory_blocks);
831 }
832
833 return 0;
834}
835
836static void remove_memory_block(struct memory_block *memory)
837{
838 if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys))
839 return;
840
841 WARN_ON(xa_erase(&memory_blocks, memory->dev.id) == NULL);
842
843 if (memory->group) {
844 list_del(entry: &memory->group_next);
845 memory->group = NULL;
846 }
847
848 /* drop the ref. we got via find_memory_block() */
849 put_device(dev: &memory->dev);
850 device_unregister(dev: &memory->dev);
851}
852
853/*
854 * Create memory block devices for the given memory area. Start and size
855 * have to be aligned to memory block granularity. Memory block devices
856 * will be initialized as offline.
857 *
858 * Called under device_hotplug_lock.
859 */
860int create_memory_block_devices(unsigned long start, unsigned long size,
861 int nid, struct vmem_altmap *altmap,
862 struct memory_group *group)
863{
864 const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
865 unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
866 struct memory_block *mem;
867 unsigned long block_id;
868 int ret = 0;
869
870 if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
871 !IS_ALIGNED(size, memory_block_size_bytes())))
872 return -EINVAL;
873
874 for (block_id = start_block_id; block_id != end_block_id; block_id++) {
875 ret = add_memory_block(block_id, nid, state: MEM_OFFLINE, altmap, group);
876 if (ret)
877 break;
878 }
879 if (ret) {
880 end_block_id = block_id;
881 for (block_id = start_block_id; block_id != end_block_id;
882 block_id++) {
883 mem = find_memory_block_by_id(block_id);
884 if (WARN_ON_ONCE(!mem))
885 continue;
886 remove_memory_block(memory: mem);
887 }
888 }
889 return ret;
890}
891
892/*
893 * Remove memory block devices for the given memory area. Start and size
894 * have to be aligned to memory block granularity. Memory block devices
895 * have to be offline.
896 *
897 * Called under device_hotplug_lock.
898 */
899void remove_memory_block_devices(unsigned long start, unsigned long size)
900{
901 const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));
902 const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));
903 struct memory_block *mem;
904 unsigned long block_id;
905
906 if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||
907 !IS_ALIGNED(size, memory_block_size_bytes())))
908 return;
909
910 for (block_id = start_block_id; block_id != end_block_id; block_id++) {
911 mem = find_memory_block_by_id(block_id);
912 if (WARN_ON_ONCE(!mem))
913 continue;
914 num_poisoned_pages_sub(pfn: -1UL, i: memblk_nr_poison(mem));
915 unregister_memory_block_under_nodes(mem_blk: mem);
916 remove_memory_block(memory: mem);
917 }
918}
919
920static struct attribute *memory_root_attrs[] = {
921#ifdef CONFIG_ARCH_MEMORY_PROBE
922 &dev_attr_probe.attr,
923#endif
924
925#ifdef CONFIG_MEMORY_FAILURE
926 &dev_attr_soft_offline_page.attr,
927 &dev_attr_hard_offline_page.attr,
928#endif
929
930 &dev_attr_block_size_bytes.attr,
931 &dev_attr_auto_online_blocks.attr,
932#ifdef CONFIG_CRASH_HOTPLUG
933 &dev_attr_crash_hotplug.attr,
934#endif
935 NULL
936};
937
938static const struct attribute_group memory_root_attr_group = {
939 .attrs = memory_root_attrs,
940};
941
942static const struct attribute_group *memory_root_attr_groups[] = {
943 &memory_root_attr_group,
944 NULL,
945};
946
947/*
948 * Initialize the sysfs support for memory devices. At the time this function
949 * is called, we cannot have concurrent creation/deletion of memory block
950 * devices, the device_hotplug_lock is not needed.
951 */
952void __init memory_dev_init(void)
953{
954 int ret;
955 unsigned long block_sz, block_id, nr;
956
957 /* Validate the configured memory block size */
958 block_sz = memory_block_size_bytes();
959 if (!is_power_of_2(n: block_sz) || block_sz < MIN_MEMORY_BLOCK_SIZE)
960 panic(fmt: "Memory block size not suitable: 0x%lx\n", block_sz);
961 sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;
962
963 ret = subsys_system_register(subsys: &memory_subsys, groups: memory_root_attr_groups);
964 if (ret)
965 panic(fmt: "%s() failed to register subsystem: %d\n", __func__, ret);
966
967 /*
968 * Create entries for memory sections that were found during boot
969 * and have been initialized. Use @block_id to track the last
970 * handled block and initialize it to an invalid value (ULONG_MAX)
971 * to bypass the block ID matching check for the first present
972 * block so that it can be covered.
973 */
974 block_id = ULONG_MAX;
975 for_each_present_section_nr(0, nr) {
976 if (block_id != ULONG_MAX && memory_block_id(section_nr: nr) == block_id)
977 continue;
978
979 block_id = memory_block_id(section_nr: nr);
980 ret = add_memory_block(block_id, NUMA_NO_NODE, state: MEM_ONLINE, NULL, NULL);
981 if (ret) {
982 panic(fmt: "%s() failed to add memory block: %d\n",
983 __func__, ret);
984 }
985 }
986}
987
988/**
989 * walk_memory_blocks - walk through all present memory blocks overlapped
990 * by the range [start, start + size)
991 *
992 * @start: start address of the memory range
993 * @size: size of the memory range
994 * @arg: argument passed to func
995 * @func: callback for each memory section walked
996 *
997 * This function walks through all present memory blocks overlapped by the
998 * range [start, start + size), calling func on each memory block.
999 *
1000 * In case func() returns an error, walking is aborted and the error is
1001 * returned.
1002 *
1003 * Called under device_hotplug_lock.
1004 */
1005int walk_memory_blocks(unsigned long start, unsigned long size,
1006 void *arg, walk_memory_blocks_func_t func)
1007{
1008 const unsigned long start_block_id = phys_to_block_id(phys: start);
1009 const unsigned long end_block_id = phys_to_block_id(phys: start + size - 1);
1010 struct memory_block *mem;
1011 unsigned long block_id;
1012 int ret = 0;
1013
1014 if (!size)
1015 return 0;
1016
1017 for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
1018 mem = find_memory_block_by_id(block_id);
1019 if (!mem)
1020 continue;
1021
1022 ret = func(mem, arg);
1023 put_device(dev: &mem->dev);
1024 if (ret)
1025 break;
1026 }
1027 return ret;
1028}
1029
1030struct for_each_memory_block_cb_data {
1031 walk_memory_blocks_func_t func;
1032 void *arg;
1033};
1034
1035static int for_each_memory_block_cb(struct device *dev, void *data)
1036{
1037 struct memory_block *mem = to_memory_block(dev);
1038 struct for_each_memory_block_cb_data *cb_data = data;
1039
1040 return cb_data->func(mem, cb_data->arg);
1041}
1042
1043/**
1044 * for_each_memory_block - walk through all present memory blocks
1045 *
1046 * @arg: argument passed to func
1047 * @func: callback for each memory block walked
1048 *
1049 * This function walks through all present memory blocks, calling func on
1050 * each memory block.
1051 *
1052 * In case func() returns an error, walking is aborted and the error is
1053 * returned.
1054 */
1055int for_each_memory_block(void *arg, walk_memory_blocks_func_t func)
1056{
1057 struct for_each_memory_block_cb_data cb_data = {
1058 .func = func,
1059 .arg = arg,
1060 };
1061
1062 return bus_for_each_dev(bus: &memory_subsys, NULL, data: &cb_data,
1063 fn: for_each_memory_block_cb);
1064}
1065
1066/*
1067 * This is an internal helper to unify allocation and initialization of
1068 * memory groups. Note that the passed memory group will be copied to a
1069 * dynamically allocated memory group. After this call, the passed
1070 * memory group should no longer be used.
1071 */
1072static int memory_group_register(struct memory_group group)
1073{
1074 struct memory_group *new_group;
1075 uint32_t mgid;
1076 int ret;
1077
1078 if (!node_possible(group.nid))
1079 return -EINVAL;
1080
1081 new_group = kzalloc(sizeof(group), GFP_KERNEL);
1082 if (!new_group)
1083 return -ENOMEM;
1084 *new_group = group;
1085 INIT_LIST_HEAD(list: &new_group->memory_blocks);
1086
1087 ret = xa_alloc(xa: &memory_groups, id: &mgid, entry: new_group, xa_limit_31b,
1088 GFP_KERNEL);
1089 if (ret) {
1090 kfree(objp: new_group);
1091 return ret;
1092 } else if (group.is_dynamic) {
1093 xa_set_mark(&memory_groups, index: mgid, MEMORY_GROUP_MARK_DYNAMIC);
1094 }
1095 return mgid;
1096}
1097
1098/**
1099 * memory_group_register_static() - Register a static memory group.
1100 * @nid: The node id.
1101 * @max_pages: The maximum number of pages we'll have in this static memory
1102 * group.
1103 *
1104 * Register a new static memory group and return the memory group id.
1105 * All memory in the group belongs to a single unit, such as a DIMM. All
1106 * memory belonging to a static memory group is added in one go to be removed
1107 * in one go -- it's static.
1108 *
1109 * Returns an error if out of memory, if the node id is invalid, if no new
1110 * memory groups can be registered, or if max_pages is invalid (0). Otherwise,
1111 * returns the new memory group id.
1112 */
1113int memory_group_register_static(int nid, unsigned long max_pages)
1114{
1115 struct memory_group group = {
1116 .nid = nid,
1117 .s = {
1118 .max_pages = max_pages,
1119 },
1120 };
1121
1122 if (!max_pages)
1123 return -EINVAL;
1124 return memory_group_register(group);
1125}
1126EXPORT_SYMBOL_GPL(memory_group_register_static);
1127
1128/**
1129 * memory_group_register_dynamic() - Register a dynamic memory group.
1130 * @nid: The node id.
1131 * @unit_pages: Unit in pages in which is memory added/removed in this dynamic
1132 * memory group.
1133 *
1134 * Register a new dynamic memory group and return the memory group id.
1135 * Memory within a dynamic memory group is added/removed dynamically
1136 * in unit_pages.
1137 *
1138 * Returns an error if out of memory, if the node id is invalid, if no new
1139 * memory groups can be registered, or if unit_pages is invalid (0, not a
1140 * power of two, smaller than a single memory block). Otherwise, returns the
1141 * new memory group id.
1142 */
1143int memory_group_register_dynamic(int nid, unsigned long unit_pages)
1144{
1145 struct memory_group group = {
1146 .nid = nid,
1147 .is_dynamic = true,
1148 .d = {
1149 .unit_pages = unit_pages,
1150 },
1151 };
1152
1153 if (!unit_pages || !is_power_of_2(n: unit_pages) ||
1154 unit_pages < PHYS_PFN(memory_block_size_bytes()))
1155 return -EINVAL;
1156 return memory_group_register(group);
1157}
1158EXPORT_SYMBOL_GPL(memory_group_register_dynamic);
1159
1160/**
1161 * memory_group_unregister() - Unregister a memory group.
1162 * @mgid: the memory group id
1163 *
1164 * Unregister a memory group. If any memory block still belongs to this
1165 * memory group, unregistering will fail.
1166 *
1167 * Returns -EINVAL if the memory group id is invalid, returns -EBUSY if some
1168 * memory blocks still belong to this memory group and returns 0 if
1169 * unregistering succeeded.
1170 */
1171int memory_group_unregister(int mgid)
1172{
1173 struct memory_group *group;
1174
1175 if (mgid < 0)
1176 return -EINVAL;
1177
1178 group = xa_load(&memory_groups, index: mgid);
1179 if (!group)
1180 return -EINVAL;
1181 if (!list_empty(head: &group->memory_blocks))
1182 return -EBUSY;
1183 xa_erase(&memory_groups, index: mgid);
1184 kfree(objp: group);
1185 return 0;
1186}
1187EXPORT_SYMBOL_GPL(memory_group_unregister);
1188
1189/*
1190 * This is an internal helper only to be used in core memory hotplug code to
1191 * lookup a memory group. We don't care about locking, as we don't expect a
1192 * memory group to get unregistered while adding memory to it -- because
1193 * the group and the memory is managed by the same driver.
1194 */
1195struct memory_group *memory_group_find_by_id(int mgid)
1196{
1197 return xa_load(&memory_groups, index: mgid);
1198}
1199
1200/*
1201 * This is an internal helper only to be used in core memory hotplug code to
1202 * walk all dynamic memory groups excluding a given memory group, either
1203 * belonging to a specific node, or belonging to any node.
1204 */
1205int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func,
1206 struct memory_group *excluded, void *arg)
1207{
1208 struct memory_group *group;
1209 unsigned long index;
1210 int ret = 0;
1211
1212 xa_for_each_marked(&memory_groups, index, group,
1213 MEMORY_GROUP_MARK_DYNAMIC) {
1214 if (group == excluded)
1215 continue;
1216#ifdef CONFIG_NUMA
1217 if (nid != NUMA_NO_NODE && group->nid != nid)
1218 continue;
1219#endif /* CONFIG_NUMA */
1220 ret = func(group, arg);
1221 if (ret)
1222 break;
1223 }
1224 return ret;
1225}
1226
1227#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)
1228void memblk_nr_poison_inc(unsigned long pfn)
1229{
1230 const unsigned long block_id = pfn_to_block_id(pfn);
1231 struct memory_block *mem = find_memory_block_by_id(block_id);
1232
1233 if (mem)
1234 atomic_long_inc(v: &mem->nr_hwpoison);
1235}
1236
1237void memblk_nr_poison_sub(unsigned long pfn, long i)
1238{
1239 const unsigned long block_id = pfn_to_block_id(pfn);
1240 struct memory_block *mem = find_memory_block_by_id(block_id);
1241
1242 if (mem)
1243 atomic_long_sub(i, v: &mem->nr_hwpoison);
1244}
1245
1246static unsigned long memblk_nr_poison(struct memory_block *mem)
1247{
1248 return atomic_long_read(v: &mem->nr_hwpoison);
1249}
1250#endif
1251

source code of linux/drivers/base/memory.c