// SPDX-License-Identifier: GPL-2.0

/*

* Memory subsystem support

*

* Written by Matt Tolentino <matthew.e.tolentino@intel.com>

* Dave Hansen <haveblue@us.ibm.com>

*

* This file provides the necessary infrastructure to represent

* a SPARSEMEM-memory-model system's physical memory in /sysfs.

* All arch-independent code that assumes MEMORY_HOTPLUG requires

* SPARSEMEM should be contained here, or in mm/memory_hotplug.c.

*/

#include <linux/module.h>

#include <linux/init.h>

#include <linux/topology.h>

#include <linux/capability.h>

#include <linux/device.h>

#include <linux/memory.h>

#include <linux/memory_hotplug.h>

#include <linux/mm.h>

#include <linux/stat.h>

#include <linux/slab.h>

#include <linux/xarray.h>

#include <linux/export.h>

#include <linux/atomic.h>

#include <linux/uaccess.h>

#define MEMORY_CLASS_NAME "memory"

static const char *const online_type_to_str[] = {

[MMOP_OFFLINE] = "offline",

[MMOP_ONLINE] = "online",

[MMOP_ONLINE_KERNEL] = "online_kernel",

[MMOP_ONLINE_MOVABLE] = "online_movable",

};

int mhp_online_type_from_str(const char *str)

{

int i;

for (i = 0; i < ARRAY_SIZE(online_type_to_str); i++) {

if (sysfs_streq(str, online_type_to_str[i]))

return i;

}

return -EINVAL;

}

#define to_memory_block(dev) container_of(dev, struct memory_block, dev)

int sections_per_block;

EXPORT_SYMBOL(sections_per_block);

static int memory_subsys_online(struct device *dev);

static int memory_subsys_offline(struct device *dev);

static const struct bus_type memory_subsys = {

.name = MEMORY_CLASS_NAME,

.dev_name = MEMORY_CLASS_NAME,

.online = memory_subsys_online,

.offline = memory_subsys_offline,

};

/*

* Memory blocks are cached in a local radix tree to avoid

* a costly linear search for the corresponding device on

* the subsystem bus.

*/

static DEFINE_XARRAY(memory_blocks);

/*

* Memory groups, indexed by memory group id (mgid).

*/

static DEFINE_XARRAY_FLAGS(memory_groups, XA_FLAGS_ALLOC);

#define MEMORY_GROUP_MARK_DYNAMIC XA_MARK_1

static BLOCKING_NOTIFIER_HEAD(memory_chain);

int register_memory_notifier(struct notifier_block *nb)

{

return blocking_notifier_chain_register(&memory_chain, nb);

}

EXPORT_SYMBOL(register_memory_notifier);

void unregister_memory_notifier(struct notifier_block *nb)

{

blocking_notifier_chain_unregister(&memory_chain, nb);

}

EXPORT_SYMBOL(unregister_memory_notifier);

static void memory_block_release(struct device *dev)

{

struct memory_block *mem = to_memory_block(dev);

/* Verify that the altmap is freed */

WARN_ON(mem->altmap);

kfree(mem);

}

/* Max block size to be set by memory_block_advise_max_size */

static unsigned long memory_block_advised_size;

static bool memory_block_advised_size_queried;

/**

* memory_block_advise_max_size() - advise memory hotplug on the max suggested

* block size, usually for alignment.

* @size: suggestion for maximum block size. must be aligned on power of 2.

*

* Early boot software (pre-allocator init) may advise archs on the max block

* size. This value can only decrease after initialization, as the intent is

* to identify the largest supported alignment for all sources.

*

* Use of this value is arch-defined, as is min/max block size.

*

* Return: 0 on success

* -EINVAL if size is 0 or not pow2 aligned

* -EBUSY if value has already been probed

*/

int __init memory_block_advise_max_size(unsigned long size)

{

if (!size || !is_power_of_2(size))

return -EINVAL;

if (memory_block_advised_size_queried)

return -EBUSY;

if (memory_block_advised_size)

memory_block_advised_size = min(memory_block_advised_size, size);

else

memory_block_advised_size = size;

return 0;

}

/**

* memory_block_advised_max_size() - query advised max hotplug block size.

*

* After the first call, the value can never change. Callers looking for the

* actual block size should use memory_block_size_bytes. This interface is

* intended for use by arch-init when initializing the hotplug block size.

*

* Return: advised size in bytes, or 0 if never set.

*/

unsigned long memory_block_advised_max_size(void)

{

memory_block_advised_size_queried = true;

return memory_block_advised_size;

}

unsigned long __weak memory_block_size_bytes(void)

{

return MIN_MEMORY_BLOCK_SIZE;

}

EXPORT_SYMBOL_GPL(memory_block_size_bytes);

/* Show the memory block ID, relative to the memory block size */

static ssize_t phys_index_show(struct device *dev,

struct device_attribute *attr, char *buf)

{

struct memory_block *mem = to_memory_block(dev);

return sysfs_emit(buf, "%08lx\n", memory_block_id(mem->start_section_nr));

}

/*

* Legacy interface that we cannot remove. Always indicate "removable"

* with CONFIG_MEMORY_HOTREMOVE - bad heuristic.

*/

static ssize_t removable_show(struct device *dev, struct device_attribute *attr,

char *buf)

{

return sysfs_emit(buf, "%d\n", (int)IS_ENABLED(CONFIG_MEMORY_HOTREMOVE));

}

/*

* online, offline, going offline, etc.

*/

static ssize_t state_show(struct device *dev, struct device_attribute *attr,

char *buf)

{

struct memory_block *mem = to_memory_block(dev);

const char *output;

/*

* We can probably put these states in a nice little array

* so that they're not open-coded

*/

switch (mem->state) {

case MEM_ONLINE:

output = "online";

break;

case MEM_OFFLINE:

output = "offline";

break;

case MEM_GOING_OFFLINE:

output = "going-offline";

break;

default:

WARN_ON(1);

return sysfs_emit(buf, "ERROR-UNKNOWN-%d\n", mem->state);

}

return sysfs_emit(buf, "%s\n", output);

}

int memory_notify(enum memory_block_state state, void *v)

{

return blocking_notifier_call_chain(&memory_chain, state, v);

}

#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG)

static unsigned long memblk_nr_poison(struct memory_block *mem);

#else

static inline unsigned long memblk_nr_poison(struct memory_block *mem)

{

return 0;

}

#endif

/*

* Must acquire mem_hotplug_lock in write mode.

*/

static int memory_block_online(struct memory_block *mem)

{

unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);

unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;

unsigned long nr_vmemmap_pages = 0;

struct zone *zone;

int ret;

if (memblk_nr_poison(mem))

return -EHWPOISON;

zone = zone_for_pfn_range(mem->online_type, mem->nid, mem->group,

start_pfn, nr_pages);

/*

* Although vmemmap pages have a different lifecycle than the pages

* they describe (they remain until the memory is unplugged), doing

* their initialization and accounting at memory onlining/offlining

* stage helps to keep accounting easier to follow - e.g vmemmaps

* belong to the same zone as the memory they backed.

*/

if (mem->altmap)

nr_vmemmap_pages = mem->altmap->free;

mem_hotplug_begin();

if (nr_vmemmap_pages) {

ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone);

if (ret)

goto out;

}

ret = online_pages(start_pfn + nr_vmemmap_pages,

nr_pages - nr_vmemmap_pages, zone, mem->group);

if (ret) {

if (nr_vmemmap_pages)

mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages);

goto out;

}

/*

* Account once onlining succeeded. If the zone was unpopulated, it is

* now already properly populated.

*/

if (nr_vmemmap_pages)

adjust_present_page_count(pfn_to_page(start_pfn), mem->group,

nr_vmemmap_pages);

mem->zone = zone;

out:

mem_hotplug_done();

return ret;

}

/*

* Must acquire mem_hotplug_lock in write mode.

*/

static int memory_block_offline(struct memory_block *mem)

{

unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);

unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;

unsigned long nr_vmemmap_pages = 0;

int ret;

if (!mem->zone)

return -EINVAL;

/*

* Unaccount before offlining, such that unpopulated zone and kthreads

* can properly be torn down in offline_pages().

*/

if (mem->altmap)

nr_vmemmap_pages = mem->altmap->free;

mem_hotplug_begin();

if (nr_vmemmap_pages)

adjust_present_page_count(pfn_to_page(start_pfn), mem->group,

-nr_vmemmap_pages);

ret = offline_pages(start_pfn + nr_vmemmap_pages,

nr_pages - nr_vmemmap_pages, mem->zone, mem->group);

if (ret) {

/* offline_pages() failed. Account back. */

if (nr_vmemmap_pages)

adjust_present_page_count(pfn_to_page(start_pfn),

mem->group, nr_vmemmap_pages);

goto out;

}

if (nr_vmemmap_pages)

mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages);

mem->zone = NULL;

out:

mem_hotplug_done();

return ret;

}

/*

* MEMORY_HOTPLUG depends on SPARSEMEM in mm/Kconfig, so it is

* OK to have direct references to sparsemem variables in here.

*/

static int

memory_block_action(struct memory_block *mem, unsigned long action)

{

int ret;

switch (action) {

case MEM_ONLINE:

ret = memory_block_online(mem);

break;

case MEM_OFFLINE:

ret = memory_block_offline(mem);

break;

default:

WARN(1, KERN_WARNING "%s(%ld, %ld) unknown action: "

"%ld\n", __func__, mem->start_section_nr, action, action);

ret = -EINVAL;

}

return ret;

}

static int memory_block_change_state(struct memory_block *mem,

unsigned long to_state, unsigned long from_state_req)

{

int ret = 0;

if (mem->state != from_state_req)

return -EINVAL;

if (to_state == MEM_OFFLINE)

mem->state = MEM_GOING_OFFLINE;

ret = memory_block_action(mem, to_state);

mem->state = ret ? from_state_req : to_state;

return ret;

}

/* The device lock serializes operations on memory_subsys_[online|offline] */

static int memory_subsys_online(struct device *dev)

{

struct memory_block *mem = to_memory_block(dev);

int ret;

if (mem->state == MEM_ONLINE)

return 0;

/*

* When called via device_online() without configuring the online_type,

* we want to default to MMOP_ONLINE.

*/

if (mem->online_type == MMOP_OFFLINE)

mem->online_type = MMOP_ONLINE;

ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE);

mem->online_type = MMOP_OFFLINE;

return ret;

}

static int memory_subsys_offline(struct device *dev)

{

struct memory_block *mem = to_memory_block(dev);

if (mem->state == MEM_OFFLINE)

return 0;

return memory_block_change_state(mem, MEM_OFFLINE, MEM_ONLINE);

}

static ssize_t state_store(struct device *dev, struct device_attribute *attr,

const char *buf, size_t count)

{

const int online_type = mhp_online_type_from_str(buf);

struct memory_block *mem = to_memory_block(dev);

int ret;

if (online_type < 0)

return -EINVAL;

ret = lock_device_hotplug_sysfs();

if (ret)

return ret;

switch (online_type) {

case MMOP_ONLINE_KERNEL:

case MMOP_ONLINE_MOVABLE:

case MMOP_ONLINE:

/* mem->online_type is protected by device_hotplug_lock */

mem->online_type = online_type;

ret = device_online(&mem->dev);

break;

case MMOP_OFFLINE:

ret = device_offline(&mem->dev);

break;

default:

ret = -EINVAL; /* should never happen */

}

unlock_device_hotplug();

if (ret < 0)

return ret;

if (ret)

return -EINVAL;

return count;

}

/*

* Legacy interface that we cannot remove: s390x exposes the storage increment

* covered by a memory block, allowing for identifying which memory blocks

* comprise a storage increment. Since a memory block spans complete

* storage increments nowadays, this interface is basically unused. Other

* archs never exposed != 0.

*/

static ssize_t phys_device_show(struct device *dev,

struct device_attribute *attr, char *buf)

{

struct memory_block *mem = to_memory_block(dev);

unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);

return sysfs_emit(buf, "%d\n",

arch_get_memory_phys_device(start_pfn));

}

#ifdef CONFIG_MEMORY_HOTREMOVE

static int print_allowed_zone(char *buf, int len, int nid,

struct memory_group *group,

unsigned long start_pfn, unsigned long nr_pages,

enum mmop online_type, struct zone *default_zone)

{

struct zone *zone;

zone = zone_for_pfn_range(online_type, nid, group, start_pfn, nr_pages);

if (zone == default_zone)

return 0;

return sysfs_emit_at(buf, len, " %s", zone->name);

}

static ssize_t valid_zones_show(struct device *dev,

struct device_attribute *attr, char *buf)

{

struct memory_block *mem = to_memory_block(dev);

unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);

unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;

struct memory_group *group = mem->group;

struct zone *default_zone;

int nid = mem->nid;

int len;

/*

* Check the existing zone. Make sure that we do that only on the

* online nodes otherwise the page_zone is not reliable

*/

if (mem->state == MEM_ONLINE) {

/*

* If !mem->zone, the memory block spans multiple zones and

* cannot get offlined.

*/

return sysfs_emit(buf, "%s\n",

mem->zone ? mem->zone->name : "none");

}

default_zone = zone_for_pfn_range(MMOP_ONLINE, nid, group,

start_pfn, nr_pages);

len = sysfs_emit(buf, "%s", default_zone->name);

len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages,

MMOP_ONLINE_KERNEL, default_zone);

len += print_allowed_zone(buf, len, nid, group, start_pfn, nr_pages,

MMOP_ONLINE_MOVABLE, default_zone);

len += sysfs_emit_at(buf, len, "\n");

return len;

}

static DEVICE_ATTR_RO(valid_zones);

#endif

static DEVICE_ATTR_RO(phys_index);

static DEVICE_ATTR_RW(state);

static DEVICE_ATTR_RO(phys_device);

static DEVICE_ATTR_RO(removable);

/*

* Show the memory block size (shared by all memory blocks).

*/

static ssize_t block_size_bytes_show(struct device *dev,

struct device_attribute *attr, char *buf)

{

return sysfs_emit(buf, "%lx\n", memory_block_size_bytes());

}

static DEVICE_ATTR_RO(block_size_bytes);

/*

* Memory auto online policy.

*/

static ssize_t auto_online_blocks_show(struct device *dev,

struct device_attribute *attr, char *buf)

{

return sysfs_emit(buf, "%s\n",

online_type_to_str[mhp_get_default_online_type()]);

}

static ssize_t auto_online_blocks_store(struct device *dev,

struct device_attribute *attr,

const char *buf, size_t count)

{

const int online_type = mhp_online_type_from_str(buf);

if (online_type < 0)

return -EINVAL;

mhp_set_default_online_type(online_type);

return count;

}

static DEVICE_ATTR_RW(auto_online_blocks);

#ifdef CONFIG_CRASH_HOTPLUG

#include <linux/kexec.h>

static ssize_t crash_hotplug_show(struct device *dev,

struct device_attribute *attr, char *buf)

{

return sysfs_emit(buf, "%d\n", crash_check_hotplug_support());

}

static DEVICE_ATTR_RO(crash_hotplug);

#endif

/*

* Some architectures will have custom drivers to do this, and

* will not need to do it from userspace. The fake hot-add code

* as well as ppc64 will do all of their discovery in userspace

* and will require this interface.

*/

#ifdef CONFIG_ARCH_MEMORY_PROBE

static ssize_t probe_store(struct device *dev, struct device_attribute *attr,

const char *buf, size_t count)

{

u64 phys_addr;

int nid, ret;

unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block;

ret = kstrtoull(buf, 0, &phys_addr);

if (ret)

return ret;

if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1))

return -EINVAL;

ret = lock_device_hotplug_sysfs();

if (ret)

return ret;

nid = memory_add_physaddr_to_nid(phys_addr);

ret = __add_memory(nid, phys_addr,

MIN_MEMORY_BLOCK_SIZE * sections_per_block,

MHP_NONE);

if (ret)

goto out;

ret = count;

out:

unlock_device_hotplug();

return ret;

}

static DEVICE_ATTR_WO(probe);

#endif

#ifdef CONFIG_MEMORY_FAILURE

/*

* Support for offlining pages of memory

*/

/* Soft offline a page */

static ssize_t soft_offline_page_store(struct device *dev,

struct device_attribute *attr,

const char *buf, size_t count)

{

int ret;

u64 pfn;

if (!capable(CAP_SYS_ADMIN))

return -EPERM;

if (kstrtoull(buf, 0, &pfn) < 0)

return -EINVAL;

pfn >>= PAGE_SHIFT;

ret = soft_offline_page(pfn, 0);

return ret == 0 ? count : ret;

}

/* Forcibly offline a page, including killing processes. */

static ssize_t hard_offline_page_store(struct device *dev,

struct device_attribute *attr,

const char *buf, size_t count)

{

int ret;

u64 pfn;

if (!capable(CAP_SYS_ADMIN))

return -EPERM;

if (kstrtoull(buf, 0, &pfn) < 0)

return -EINVAL;

pfn >>= PAGE_SHIFT;

ret = memory_failure(pfn, MF_SW_SIMULATED);

if (ret == -EOPNOTSUPP)

ret = 0;

return ret ? ret : count;

}

static DEVICE_ATTR_WO(soft_offline_page);

static DEVICE_ATTR_WO(hard_offline_page);

#endif

/* See phys_device_show(). */

int __weak arch_get_memory_phys_device(unsigned long start_pfn)

{

return 0;

}

/*

* A reference for the returned memory block device is acquired.

*

* Called under device_hotplug_lock.

*/

struct memory_block *memory_block_get(unsigned long block_id)

{

struct memory_block *mem;

mem = xa_load(&memory_blocks, block_id);

if (mem)

get_device(&mem->dev);

return mem;

}

static struct attribute *memory_memblk_attrs[] = {

&dev_attr_phys_index.attr,

&dev_attr_state.attr,

&dev_attr_phys_device.attr,

&dev_attr_removable.attr,

#ifdef CONFIG_MEMORY_HOTREMOVE

&dev_attr_valid_zones.attr,

#endif

NULL

};

static const struct attribute_group memory_memblk_attr_group = {

.attrs = memory_memblk_attrs,

};

static const struct attribute_group *memory_memblk_attr_groups[] = {

&memory_memblk_attr_group,

NULL,

};

static int __add_memory_block(struct memory_block *memory)

{

int ret;

memory->dev.bus = &memory_subsys;

memory->dev.id = memory->start_section_nr / sections_per_block;

memory->dev.release = memory_block_release;

memory->dev.groups = memory_memblk_attr_groups;

dev_assign_offline(&memory->dev, memory->state == MEM_OFFLINE);

ret = device_register(&memory->dev);

if (ret) {

memory_block_put(memory);

return ret;

}

ret = xa_err(xa_store(&memory_blocks, memory->dev.id, memory,

GFP_KERNEL));

if (ret)

device_unregister(&memory->dev);

return ret;

}

static struct zone *early_node_zone_for_memory_block(struct memory_block *mem,

int nid)

{

const unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);

const unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block;

struct zone *zone, *matching_zone = NULL;

pg_data_t *pgdat = NODE_DATA(nid);

int i;

/*

* This logic only works for early memory, when the applicable zones

* already span the memory block. We don't expect overlapping zones on

* a single node for early memory. So if we're told that some PFNs

* of a node fall into this memory block, we can assume that all node

* zones that intersect with the memory block are actually applicable.

* No need to look at the memmap.

*/

for (i = 0; i < MAX_NR_ZONES; i++) {

zone = pgdat->node_zones + i;

if (!populated_zone(zone))

continue;

if (!zone_intersects(zone, start_pfn, nr_pages))

continue;

if (!matching_zone) {

matching_zone = zone;

continue;

}

/* Spans multiple zones ... */

matching_zone = NULL;

break;

}

return matching_zone;

}

#ifdef CONFIG_NUMA

/**

* memory_block_add_nid_early() - Indicate that early system RAM falling into

* this memory block device (partially) belongs

* to the given node.

* @mem: The memory block device.

* @nid: The node id.

*

* Indicate that early system RAM falling into this memory block (partially)

* belongs to the given node. This will also properly set/adjust mem->zone based

* on the zone ranges of the given node.

*

* Memory hotplug handles this on memory block creation, where we can only have

* a single nid span a memory block.

*/

void memory_block_add_nid_early(struct memory_block *mem, int nid)

{

if (mem->nid != nid) {

/*

* For early memory we have to determine the zone when setting

* the node id and handle multiple nodes spanning a single

* memory block by indicate via zone == NULL that we're not

* dealing with a single zone. So if we're setting the node id

* the first time, determine if there is a single zone. If we're

* setting the node id a second time to a different node,

* invalidate the single detected zone.

*/

if (mem->nid == NUMA_NO_NODE)

mem->zone = early_node_zone_for_memory_block(mem, nid);

else

mem->zone = NULL;

/*

* If this memory block spans multiple nodes, we only indicate

* the last processed node. If we span multiple nodes (not applicable

* to hotplugged memory), zone == NULL will prohibit memory offlining

* and consequently unplug.

*/

mem->nid = nid;

}

}

#endif

static int add_memory_block(unsigned long block_id, int nid, unsigned long state,

struct vmem_altmap *altmap,

struct memory_group *group)

{

struct memory_block *mem;

int ret = 0;

mem = memory_block_get(block_id);

if (mem) {

memory_block_put(mem);

return -EEXIST;

}

mem = kzalloc_obj(*mem);

if (!mem)

return -ENOMEM;

mem->start_section_nr = block_id * sections_per_block;

mem->state = state;

mem->nid = nid;

INIT_LIST_HEAD(&mem->group_next);

#ifndef CONFIG_NUMA

if (state == MEM_ONLINE)

/*

* MEM_ONLINE at this point implies early memory. With NUMA,

* we'll determine the zone when setting the node id via

* memory_block_add_nid_early(). Memory hotplug updated the zone

* manually when memory onlining/offlining succeeds.

*/

mem->zone = early_node_zone_for_memory_block(mem, NUMA_NO_NODE);

#endif /* CONFIG_NUMA */

ret = __add_memory_block(mem);

if (ret)

return ret;

mem->altmap = altmap;

if (group) {

mem->group = group;

list_add(&mem->group_next, &group->memory_blocks);

}

return 0;

}

static void remove_memory_block(struct memory_block *memory)

{

if (WARN_ON_ONCE(memory->dev.bus != &memory_subsys))

return;

WARN_ON(xa_erase(&memory_blocks, memory->dev.id) == NULL);

if (memory->group) {

list_del(&memory->group_next);

memory->group = NULL;

}

/* drop the ref. we got via memory_block_get() */

memory_block_put(memory);

device_unregister(&memory->dev);

}

/*

* Create memory block devices for the given memory area. Start and size

* have to be aligned to memory block granularity. Memory block devices

* will be initialized as offline.

*

* Called under device_hotplug_lock.

*/

int create_memory_block_devices(unsigned long start, unsigned long size,

int nid, struct vmem_altmap *altmap,

struct memory_group *group)

{

const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));

unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));

struct memory_block *mem;

unsigned long block_id;

int ret = 0;

if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||

!IS_ALIGNED(size, memory_block_size_bytes())))

return -EINVAL;

for (block_id = start_block_id; block_id != end_block_id; block_id++) {

ret = add_memory_block(block_id, nid, MEM_OFFLINE, altmap, group);

if (ret)

break;

}

if (ret) {

end_block_id = block_id;

for (block_id = start_block_id; block_id != end_block_id;

block_id++) {

mem = memory_block_get(block_id);

if (WARN_ON_ONCE(!mem))

continue;

remove_memory_block(mem);

}

}

return ret;

}

/*

* Remove memory block devices for the given memory area. Start and size

* have to be aligned to memory block granularity. Memory block devices

* have to be offline.

*

* Called under device_hotplug_lock.

*/

void remove_memory_block_devices(unsigned long start, unsigned long size)

{

const unsigned long start_block_id = pfn_to_block_id(PFN_DOWN(start));

const unsigned long end_block_id = pfn_to_block_id(PFN_DOWN(start + size));

struct memory_block *mem;

unsigned long block_id;

if (WARN_ON_ONCE(!IS_ALIGNED(start, memory_block_size_bytes()) ||

!IS_ALIGNED(size, memory_block_size_bytes())))

return;

for (block_id = start_block_id; block_id != end_block_id; block_id++) {

mem = memory_block_get(block_id);

if (WARN_ON_ONCE(!mem))

continue;

num_poisoned_pages_sub(-1UL, memblk_nr_poison(mem));

unregister_memory_block_under_nodes(mem);

remove_memory_block(mem);

}

}

static struct attribute *memory_root_attrs[] = {

#ifdef CONFIG_ARCH_MEMORY_PROBE

&dev_attr_probe.attr,

#endif

#ifdef CONFIG_MEMORY_FAILURE

&dev_attr_soft_offline_page.attr,

&dev_attr_hard_offline_page.attr,

#endif

&dev_attr_block_size_bytes.attr,

&dev_attr_auto_online_blocks.attr,

#ifdef CONFIG_CRASH_HOTPLUG

&dev_attr_crash_hotplug.attr,

#endif

NULL

};

static const struct attribute_group memory_root_attr_group = {

.attrs = memory_root_attrs,

};

static const struct attribute_group *memory_root_attr_groups[] = {

&memory_root_attr_group,

NULL,

};

/*

* Initialize the sysfs support for memory devices. At the time this function

* is called, we cannot have concurrent creation/deletion of memory block

* devices, the device_hotplug_lock is not needed.

*/

void __init memory_dev_init(void)

{

int ret;

unsigned long block_sz, block_id, nr;

/* Validate the configured memory block size */

block_sz = memory_block_size_bytes();

if (!is_power_of_2(block_sz) || block_sz < MIN_MEMORY_BLOCK_SIZE)

panic("Memory block size not suitable: 0x%lx\n", block_sz);

sections_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE;

ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);

if (ret)

panic("%s() failed to register subsystem: %d\n", __func__, ret);

/*

* Create entries for memory sections that were found during boot

* and have been initialized. Use @block_id to track the last

* handled block and initialize it to an invalid value (ULONG_MAX)

* to bypass the block ID matching check for the first present

* block so that it can be covered.

*/

block_id = ULONG_MAX;

for_each_present_section_nr(0, nr) {

if (block_id != ULONG_MAX && memory_block_id(nr) == block_id)

continue;

block_id = memory_block_id(nr);

ret = add_memory_block(block_id, NUMA_NO_NODE, MEM_ONLINE, NULL, NULL);

if (ret) {

panic("%s() failed to add memory block: %d\n",

__func__, ret);

}

}

}

/**

* walk_memory_blocks - walk through all present memory blocks overlapped

* by the range [start, start + size)

*

* @start: start address of the memory range

* @size: size of the memory range

* @arg: argument passed to func

* @func: callback for each memory section walked

*

* This function walks through all present memory blocks overlapped by the

* range [start, start + size), calling func on each memory block.

*

* In case func() returns an error, walking is aborted and the error is

* returned.

*

* Called under device_hotplug_lock.

*/

int walk_memory_blocks(unsigned long start, unsigned long size,

void *arg, walk_memory_blocks_func_t func)

{

const unsigned long start_block_id = phys_to_block_id(start);

const unsigned long end_block_id = phys_to_block_id(start + size - 1);