// SPDX-License-Identifier: GPL-2.0-only

/*

* Simple NUMA memory policy for the Linux kernel.

*

* Copyright 2003,2004 Andi Kleen, SuSE Labs.

* (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.

*

* NUMA policy allows the user to give hints in which node(s) memory should

* be allocated.

*

* Support six policies per VMA and per process:

*

* The VMA policy has priority over the process policy for a page fault.

*

* interleave Allocate memory interleaved over a set of nodes,

* with normal fallback if it fails.

* For VMA based allocations this interleaves based on the

* offset into the backing object or offset into the mapping

* for anonymous memory. For process policy an process counter

* is used.

*

* weighted interleave

* Allocate memory interleaved over a set of nodes based on

* a set of weights (per-node), with normal fallback if it

* fails. Otherwise operates the same as interleave.

* Example: nodeset(0,1) & weights (2,1) - 2 pages allocated

* on node 0 for every 1 page allocated on node 1.

*

* bind Only allocate memory on a specific set of nodes,

* no fallback.

* FIXME: memory is allocated starting with the first node

* to the last. It would be better if bind would truly restrict

* the allocation to memory nodes instead

*

* preferred Try a specific node first before normal fallback.

* As a special case NUMA_NO_NODE here means do the allocation

* on the local CPU. This is normally identical to default,

* but useful to set in a VMA when you have a non default

* process policy.

*

* preferred many Try a set of nodes first before normal fallback. This is

* similar to preferred without the special case.

*

* default Allocate on the local node first, or when on a VMA

* use the process policy. This is what Linux always did

* in a NUMA aware kernel and still does by, ahem, default.

*

* The process policy is applied for most non interrupt memory allocations

* in that process' context. Interrupts ignore the policies and always

* try to allocate on the local CPU. The VMA policy is only applied for memory

* allocations for a VMA in the VM.

*

* Currently there are a few corner cases in swapping where the policy

* is not applied, but the majority should be handled. When process policy

* is used it is not remembered over swap outs/swap ins.

*

* Only the highest zone in the zone hierarchy gets policied. Allocations

* requesting a lower zone just use default policy. This implies that

* on systems with highmem kernel lowmem allocation don't get policied.

* Same with GFP_DMA allocations.

*

* For shmem/tmpfs shared memory the policy is shared between

* all users and remembered even when nobody has memory mapped.

*/

/* Notebook:

fix mmap readahead to honour policy and enable policy for any page cache

object

statistics for bigpages

global policy for page cache? currently it uses process policy. Requires

first item above.

handle mremap for shared memory (currently ignored for the policy)

grows down?

make bind policy root only? It can trigger oom much faster and the

kernel is not always grateful with that.

*/

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/mempolicy.h>

#include <linux/pagewalk.h>

#include <linux/highmem.h>

#include <linux/hugetlb.h>

#include <linux/kernel.h>

#include <linux/sched.h>

#include <linux/sched/mm.h>

#include <linux/sched/numa_balancing.h>

#include <linux/sched/sysctl.h>

#include <linux/sched/task.h>

#include <linux/nodemask.h>

#include <linux/cpuset.h>

#include <linux/slab.h>

#include <linux/string.h>

#include <linux/export.h>

#include <linux/nsproxy.h>

#include <linux/interrupt.h>

#include <linux/init.h>

#include <linux/compat.h>

#include <linux/ptrace.h>

#include <linux/swap.h>

#include <linux/seq_file.h>

#include <linux/proc_fs.h>

#include <linux/memory-tiers.h>

#include <linux/migrate.h>

#include <linux/ksm.h>

#include <linux/rmap.h>

#include <linux/security.h>

#include <linux/syscalls.h>

#include <linux/ctype.h>

#include <linux/mm_inline.h>

#include <linux/mmu_notifier.h>

#include <linux/printk.h>

#include <linux/leafops.h>

#include <linux/gcd.h>

#include <asm/tlbflush.h>

#include <asm/tlb.h>

#include <linux/uaccess.h>

#include <linux/memory.h>

#include "internal.h"

/* Internal flags */

#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0) /* Skip checks for continuous vmas */

#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1) /* Invert check for nodemask */

#define MPOL_MF_WRLOCK (MPOL_MF_INTERNAL << 2) /* Write-lock walked vmas */

static struct kmem_cache *policy_cache;

static struct kmem_cache *sn_cache;

/* Highest zone. An specific allocation for a zone below that is not

policied. */

enum zone_type policy_zone = 0;

/*

* run-time system-wide default policy => local allocation

*/

static struct mempolicy default_policy = {

.refcnt = ATOMIC_INIT(1), /* never free it */

.mode = MPOL_LOCAL,

};

static struct mempolicy preferred_node_policy[MAX_NUMNODES];

/*

* weightiness balances the tradeoff between small weights (cycles through nodes

* faster, more fair/even distribution) and large weights (smaller errors

* between actual bandwidth ratios and weight ratios). 32 is a number that has

* been found to perform at a reasonable compromise between the two goals.

*/

static const int weightiness = 32;

/*

* A null weighted_interleave_state is interpreted as having .mode="auto",

* and .iw_table is interpreted as an array of 1s with length nr_node_ids.

*/

struct weighted_interleave_state {

bool mode_auto;

u8 iw_table[];

};

static struct weighted_interleave_state __rcu *wi_state;

static unsigned int *node_bw_table;

/*

* wi_state_lock protects both wi_state and node_bw_table.

* node_bw_table is only used by writers to update wi_state.

*/

static DEFINE_MUTEX(wi_state_lock);

static u8 get_il_weight(int node)

{

struct weighted_interleave_state *state;

u8 weight = 1;

rcu_read_lock();

state = rcu_dereference(wi_state);

if (state)

weight = state->iw_table[node];

rcu_read_unlock();

return weight;

}

/*

* Convert bandwidth values into weighted interleave weights.

* Call with wi_state_lock.

*/

static void reduce_interleave_weights(unsigned int *bw, u8 *new_iw)

{

u64 sum_bw = 0;

unsigned int cast_sum_bw, scaling_factor = 1, iw_gcd = 0;

int nid;

for_each_node_state(nid, N_MEMORY)

sum_bw += bw[nid];

/* Scale bandwidths to whole numbers in the range [1, weightiness] */

for_each_node_state(nid, N_MEMORY) {

/*

* Try not to perform 64-bit division.

* If sum_bw < scaling_factor, then sum_bw < U32_MAX.

* If sum_bw > scaling_factor, then round the weight up to 1.

*/

scaling_factor = weightiness * bw[nid];

if (bw[nid] && sum_bw < scaling_factor) {

cast_sum_bw = (unsigned int)sum_bw;

new_iw[nid] = scaling_factor / cast_sum_bw;

} else {

new_iw[nid] = 1;

}

if (!iw_gcd)

iw_gcd = new_iw[nid];

iw_gcd = gcd(iw_gcd, new_iw[nid]);

}

/* 1:2 is strictly better than 16:32. Reduce by the weights' GCD. */

for_each_node_state(nid, N_MEMORY)

new_iw[nid] /= iw_gcd;

}

int mempolicy_set_node_perf(unsigned int node, struct access_coordinate *coords)

{

struct weighted_interleave_state *new_wi_state, *old_wi_state = NULL;

unsigned int *old_bw, *new_bw;

unsigned int bw_val;

int i;

bw_val = min(coords->read_bandwidth, coords->write_bandwidth);

new_bw = kcalloc(nr_node_ids, sizeof(unsigned int), GFP_KERNEL);

if (!new_bw)

return -ENOMEM;

new_wi_state = kmalloc_flex(*new_wi_state, iw_table, nr_node_ids);

if (!new_wi_state) {

kfree(new_bw);

return -ENOMEM;

}

new_wi_state->mode_auto = true;

for (i = 0; i < nr_node_ids; i++)

new_wi_state->iw_table[i] = 1;

/*

* Update bandwidth info, even in manual mode. That way, when switching

* to auto mode in the future, iw_table can be overwritten using

* accurate bw data.

*/

mutex_lock(&wi_state_lock);

old_bw = node_bw_table;

if (old_bw)

memcpy(new_bw, old_bw, nr_node_ids * sizeof(*old_bw));

new_bw[node] = bw_val;

node_bw_table = new_bw;

old_wi_state = rcu_dereference_protected(wi_state,

lockdep_is_held(&wi_state_lock));

if (old_wi_state && !old_wi_state->mode_auto) {

/* Manual mode; skip reducing weights and updating wi_state */

mutex_unlock(&wi_state_lock);

kfree(new_wi_state);

goto out;

}

/* NULL wi_state assumes auto=true; reduce weights and update wi_state*/

reduce_interleave_weights(new_bw, new_wi_state->iw_table);

rcu_assign_pointer(wi_state, new_wi_state);

mutex_unlock(&wi_state_lock);

if (old_wi_state) {

synchronize_rcu();

kfree(old_wi_state);

}

out:

kfree(old_bw);

return 0;

}

/**

* numa_nearest_node - Find nearest node by state

* @node: Node id to start the search

* @state: State to filter the search

*

* Lookup the closest node by distance if @nid is not in state.

*

* Return: this @node if it is in state, otherwise the closest node by distance

*/

int numa_nearest_node(int node, unsigned int state)

{

int min_dist = INT_MAX, dist, n, min_node;

if (state >= NR_NODE_STATES)

return -EINVAL;

if (node == NUMA_NO_NODE || node_state(node, state))

return node;

min_node = node;

for_each_node_state(n, state) {

dist = node_distance(node, n);

if (dist < min_dist) {

min_dist = dist;

min_node = n;

}

}

return min_node;

}

EXPORT_SYMBOL_GPL(numa_nearest_node);

/**

* nearest_node_nodemask - Find the node in @mask at the nearest distance

* from @node.

*

* @node: a valid node ID to start the search from.

* @mask: a pointer to a nodemask representing the allowed nodes.

*

* This function iterates over all nodes in @mask and calculates the

* distance from the starting @node, then it returns the node ID that is

* the closest to @node, or MAX_NUMNODES if no node is found.

*

* Note that @node must be a valid node ID usable with node_distance(),

* providing an invalid node ID (e.g., NUMA_NO_NODE) may result in crashes

* or unexpected behavior.

*/

int nearest_node_nodemask(int node, nodemask_t *mask)

{

int dist, n, min_dist = INT_MAX, min_node = MAX_NUMNODES;

for_each_node_mask(n, *mask) {

dist = node_distance(node, n);

if (dist < min_dist) {

min_dist = dist;

min_node = n;

}

}

return min_node;

}

EXPORT_SYMBOL_GPL(nearest_node_nodemask);

struct mempolicy *get_task_policy(struct task_struct *p)

{

struct mempolicy *pol = p->mempolicy;

int node;

if (pol)

return pol;

node = numa_node_id();

if (node != NUMA_NO_NODE) {

pol = &preferred_node_policy[node];

/* preferred_node_policy is not initialised early in boot */

if (pol->mode)

return pol;

}

return &default_policy;

}

EXPORT_SYMBOL_FOR_MODULES(get_task_policy, "kvm");

static const struct mempolicy_operations {

int (*create)(struct mempolicy *pol, const nodemask_t *nodes);

void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);

} mpol_ops[MPOL_MAX];

static inline int mpol_store_user_nodemask(const struct mempolicy *pol)

{

return pol->flags & MPOL_USER_NODEMASK_FLAGS;

}

static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,

const nodemask_t *rel)

{

nodemask_t tmp;

nodes_fold(tmp, *orig, nodes_weight(*rel));

nodes_onto(*ret, tmp, *rel);

}

static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)

{

if (nodes_empty(*nodes))

return -EINVAL;

pol->nodes = *nodes;

return 0;

}

static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)

{

if (nodes_empty(*nodes))

return -EINVAL;

nodes_clear(pol->nodes);

node_set(first_node(*nodes), pol->nodes);

return 0;

}

/*

* mpol_set_nodemask is called after mpol_new() to set up the nodemask, if

* any, for the new policy. mpol_new() has already validated the nodes

* parameter with respect to the policy mode and flags.

*

* Must be called holding task's alloc_lock to protect task's mems_allowed

* and mempolicy. May also be called holding the mmap_lock for write.

*/

static int mpol_set_nodemask(struct mempolicy *pol,

const nodemask_t *nodes, struct nodemask_scratch *nsc)

{

int ret;

/*

* Default (pol==NULL) resp. local memory policies are not a

* subject of any remapping. They also do not need any special

* constructor.

*/

if (!pol || pol->mode == MPOL_LOCAL)

return 0;

/* Check N_MEMORY */

nodes_and(nsc->mask1,

cpuset_current_mems_allowed, node_states[N_MEMORY]);

VM_BUG_ON(!nodes);

if (pol->flags & MPOL_F_RELATIVE_NODES)

mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);

else

nodes_and(nsc->mask2, *nodes, nsc->mask1);

if (mpol_store_user_nodemask(pol))

pol->w.user_nodemask = *nodes;

else

pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed;

ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);

return ret;

}

/*

* This function just creates a new policy, does some check and simple

* initialization. You must invoke mpol_set_nodemask() to set nodes.

*/

static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,

nodemask_t *nodes)

{

struct mempolicy *policy;

if (mode == MPOL_DEFAULT) {

if (nodes && !nodes_empty(*nodes))

return ERR_PTR(-EINVAL);

return NULL;

}

VM_BUG_ON(!nodes);

/*

* MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or

* MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).

* All other modes require a valid pointer to a non-empty nodemask.

*/

if (mode == MPOL_PREFERRED) {

if (nodes_empty(*nodes)) {

if (((flags & MPOL_F_STATIC_NODES) ||

(flags & MPOL_F_RELATIVE_NODES)))

return ERR_PTR(-EINVAL);

mode = MPOL_LOCAL;

}

} else if (mode == MPOL_LOCAL) {

if (!nodes_empty(*nodes) ||

(flags & MPOL_F_STATIC_NODES) ||

(flags & MPOL_F_RELATIVE_NODES))

return ERR_PTR(-EINVAL);

} else if (nodes_empty(*nodes))

return ERR_PTR(-EINVAL);

policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);

if (!policy)

return ERR_PTR(-ENOMEM);

atomic_set(&policy->refcnt, 1);

policy->mode = mode;

policy->flags = flags;

policy->home_node = NUMA_NO_NODE;

return policy;

}

/* Slow path of a mpol destructor. */

void __mpol_put(struct mempolicy *pol)

{

if (!atomic_dec_and_test(&pol->refcnt))

return;

/*

* Required to allow mmap_lock_speculative*() access, see for example

* futex_key_to_node_opt(). All accesses are serialized by mmap_lock,

* however the speculative lock section unbound by the normal lock

* boundaries, requiring RCU freeing.

*/

kfree_rcu(pol, rcu);

}

EXPORT_SYMBOL_FOR_MODULES(__mpol_put, "kvm");

static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)

{

}

static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)

{

nodemask_t tmp;

if (pol->flags & MPOL_F_STATIC_NODES)

nodes_and(tmp, pol->w.user_nodemask, *nodes);

else if (pol->flags & MPOL_F_RELATIVE_NODES)

mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);

else {

nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,

*nodes);

pol->w.cpuset_mems_allowed = *nodes;

}

if (nodes_empty(tmp))

tmp = *nodes;

pol->nodes = tmp;

}

static void mpol_rebind_preferred(struct mempolicy *pol,

const nodemask_t *nodes)

{

pol->w.cpuset_mems_allowed = *nodes;

}

/*

* mpol_rebind_policy - Migrate a policy to a different set of nodes

*

* Per-vma policies are protected by mmap_lock. Allocations using per-task

* policies are protected by task->mems_allowed_seq to prevent a premature

* OOM/allocation failure due to parallel nodemask modification.

*/

static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)

{

if (!pol || pol->mode == MPOL_LOCAL)

return;

if (!mpol_store_user_nodemask(pol) &&

nodes_equal(pol->w.cpuset_mems_allowed, *newmask))

return;

mpol_ops[pol->mode].rebind(pol, newmask);

}

/*

* Wrapper for mpol_rebind_policy() that just requires task

* pointer, and updates task mempolicy.

*

* Called with task's alloc_lock held.

*/

void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)

{

mpol_rebind_policy(tsk->mempolicy, new);

}

/*

* Rebind each vma in mm to new nodemask.

*

* Call holding a reference to mm. Takes mm->mmap_lock during call.

*/

void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)

{

struct vm_area_struct *vma;

VMA_ITERATOR(vmi, mm, 0);

mmap_write_lock(mm);

for_each_vma(vmi, vma) {

vma_start_write(vma);

mpol_rebind_policy(vma->vm_policy, new);

}

mmap_write_unlock(mm);

}

static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {

[MPOL_DEFAULT] = {

.rebind = mpol_rebind_default,

},

[MPOL_INTERLEAVE] = {

.create = mpol_new_nodemask,

.rebind = mpol_rebind_nodemask,

},

[MPOL_PREFERRED] = {

.create = mpol_new_preferred,

.rebind = mpol_rebind_preferred,

},

[MPOL_BIND] = {

.create = mpol_new_nodemask,

.rebind = mpol_rebind_nodemask,

},

[MPOL_LOCAL] = {

.rebind = mpol_rebind_default,

},

[MPOL_PREFERRED_MANY] = {

.create = mpol_new_nodemask,

.rebind = mpol_rebind_preferred,

},

[MPOL_WEIGHTED_INTERLEAVE] = {

.create = mpol_new_nodemask,

.rebind = mpol_rebind_nodemask,

},

};

static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,

unsigned long flags);

static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol,

pgoff_t ilx, int *nid);

static bool strictly_unmovable(unsigned long flags)

{

/*

* STRICT without MOVE flags lets do_mbind() fail immediately with -EIO

* if any misplaced page is found.

*/

return (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ==

MPOL_MF_STRICT;

}

struct migration_mpol { /* for alloc_migration_target_by_mpol() */

struct mempolicy *pol;

pgoff_t ilx;

};

struct queue_pages {

struct list_head *pagelist;

unsigned long flags;

nodemask_t *nmask;

unsigned long start;

unsigned long end;

struct vm_area_struct *first;

struct folio *large; /* note last large folio encountered */

long nr_failed; /* could not be isolated at this time */

};

/*

* Check if the folio's nid is in qp->nmask.

*

* If MPOL_MF_INVERT is set in qp->flags, check if the nid is

* in the invert of qp->nmask.

*/

static inline bool queue_folio_required(struct folio *folio,

struct queue_pages *qp)

{

int nid = folio_nid(folio);

unsigned long flags = qp->flags;

return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);

}

static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)

{

struct folio *folio;

struct queue_pages *qp = walk->private;

if (unlikely(pmd_is_migration_entry(*pmd))) {

qp->nr_failed++;

return;

}

folio = pmd_folio(*pmd);

if (is_huge_zero_folio(folio)) {

walk->action = ACTION_CONTINUE;

return;

}

if (!queue_folio_required(folio, qp))

return;

if (!(qp->flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||

!vma_migratable(walk->vma) ||

!migrate_folio_add(folio, qp->pagelist, qp->flags))

qp->nr_failed++;

}

/*

* Scan through folios, checking if they satisfy the required conditions,

* moving them from LRU to local pagelist for migration if they do (or not).

*

* queue_folios_pte_range() has two possible return values:

* 0 - continue walking to scan for more, even if an existing folio on the

* wrong node could not be isolated and queued for migration.

* -EIO - only MPOL_MF_STRICT was specified, without MPOL_MF_MOVE or ..._ALL,

* and an existing folio was on a node that does not follow the policy.

*/

static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,

unsigned long end, struct mm_walk *walk)

{

struct vm_area_struct *vma = walk->vma;

struct folio *folio;

struct queue_pages *qp = walk->private;

unsigned long flags = qp->flags;

pte_t *pte, *mapped_pte;

pte_t ptent;

spinlock_t *ptl;

int max_nr, nr;

ptl = pmd_trans_huge_lock(pmd, vma);

if (ptl) {

queue_folios_pmd(pmd, walk);

spin_unlock(ptl);

goto out;

}

mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);

if (!pte) {

walk->action = ACTION_AGAIN;

return 0;

}

for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) {

max_nr = (end - addr) >> PAGE_SHIFT;

nr = 1;

ptent = ptep_get(pte);

if (pte_none(ptent))

continue;

if (!pte_present(ptent)) {

const softleaf_t entry = softleaf_from_pte(ptent);

if (softleaf_is_migration(entry))

qp->nr_failed++;

continue;

}

folio = vm_normal_folio(vma, addr, ptent);

if (!folio || folio_is_zone_device(folio))

continue;

if (folio_test_large(folio) && max_nr != 1)

nr = folio_pte_batch(folio, pte, ptent, max_nr);

/*

* vm_normal_folio() filters out zero pages, but there might

* still be reserved folios to skip, perhaps in a VDSO.

*/

if (folio_test_reserved(folio))

continue;

if (!queue_folio_required(folio, qp))

continue;

if (folio_test_large(folio)) {

/*

* A large folio can only be isolated from LRU once,

* but may be mapped by many PTEs (and Copy-On-Write may

* intersperse PTEs of other, order 0, folios). This is

* a common case, so don't mistake it for failure (but

* there can be other cases of multi-mapped pages which

* this quick check does not help to filter out - and a

* search of the pagelist might grow to be prohibitive).

*

* migrate_pages(&pagelist) returns nr_failed folios, so

* check "large" now so that queue_pages_range() returns

* a comparable nr_failed folios. This does imply that

* if folio could not be isolated for some racy reason

* at its first PTE, later PTEs will not give it another

* chance of isolation; but keeps the accounting simple.

*/

if (folio == qp->large)

continue;

qp->large = folio;

}

if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||

!vma_migratable(vma) ||

!migrate_folio_add(folio, qp->pagelist, flags)) {

qp->nr_failed += nr;

if (strictly_unmovable(flags))

break;

}

}

pte_unmap_unlock(mapped_pte, ptl);

cond_resched();

out:

if (qp->nr_failed && strictly_unmovable(flags))

return -EIO;

return 0;

}

static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask,

unsigned long addr, unsigned long end,

struct mm_walk *walk)

{

#ifdef CONFIG_HUGETLB_PAGE

struct queue_pages *qp = walk->private;

unsigned long flags = qp->flags;

struct folio *folio;

spinlock_t *ptl;

pte_t ptep;

ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);

ptep = huge_ptep_get(walk->mm, addr, pte);

if (!pte_present(ptep)) {

if (!huge_pte_none(ptep)) {

const softleaf_t entry = softleaf_from_pte(ptep);

if (unlikely(softleaf_is_migration(entry)))

qp->nr_failed++;

}

goto unlock;

}

folio = pfn_folio(pte_pfn(ptep));

if (!queue_folio_required(folio, qp))

goto unlock;

if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||

!vma_migratable(walk->vma)) {

qp->nr_failed++;

goto unlock;

}

/*

* Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio.

* Choosing not to migrate a shared folio is not counted as a failure.

*

* See folio_maybe_mapped_shared() on possible imprecision when we

* cannot easily detect if a folio is shared.

*/

if ((flags & MPOL_MF_MOVE_ALL) ||

(!folio_maybe_mapped_shared(folio) && !hugetlb_pmd_shared(pte)))

if (!folio_isolate_hugetlb(folio, qp->pagelist))

qp->nr_failed++;

unlock:

spin_unlock(ptl);

if (qp->nr_failed && strictly_unmovable(flags))

return -EIO;

#endif

return 0;

}

#ifdef CONFIG_NUMA_BALANCING

/**

* folio_can_map_prot_numa() - check whether the folio can map prot numa

* @folio: The folio whose mapping considered for being made NUMA hintable

* @vma: The VMA that the folio belongs to.

* @is_private_single_threaded: Is this a single-threaded private VMA or not

*

* This function checks to see if the folio actually indicates that

* we need to make the mapping one which causes a NUMA hinting fault,

* as there are cases where it's simply unnecessary, and the folio's

* access time is adjusted for memory tiering if prot numa needed.

*

* Return: True if the mapping of the folio needs to be changed, false otherwise.

*/

bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma,

bool is_private_single_threaded)

{

int nid;

if (!folio || folio_is_zone_device(folio) || folio_test_ksm(folio))

return false;

/* Also skip shared copy-on-write folios */

if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio))

return false;

/* Folios are pinned and can't be migrated */

if (folio_maybe_dma_pinned(folio))

return false;

/*

* While migration can move some dirty folios,

* it cannot move them all from MIGRATE_ASYNC

* context.

*/

if (folio_is_file_lru(folio) && folio_test_dirty(folio))

return false;

/*

* Don't mess with PTEs if folio is already on the node

* a single-threaded process is running on.

*/

nid = folio_nid(folio);

if (is_private_single_threaded && (nid == numa_node_id()))

return false;

/*

* Skip scanning top tier node if normal numa

* balancing is disabled

*/

if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&

node_is_toptier(nid))

return false;

if (folio_use_access_time(folio))

folio_xchg_access_time(folio, jiffies_to_msecs(jiffies));

return true;

}

/*

* This is used to mark a range of virtual addresses to be inaccessible.

* These are later cleared by a NUMA hinting fault. Depending on these

* faults, pages may be migrated for better NUMA placement.

*

* This is assuming that NUMA faults are handled using PROT_NONE. If

* an architecture makes a different choice, it will need further

* changes to the core.

*/

unsigned long change_prot_numa(struct vm_area_struct *vma,

unsigned long addr, unsigned long end)

{

struct mmu_gather tlb;

long nr_updated;

tlb_gather_mmu(&tlb, vma->vm_mm);

nr_updated = change_protection(&tlb, vma, addr, end, MM_CP_PROT_NUMA);

if (nr_updated > 0) {

count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);

count_memcg_events_mm(vma->vm_mm, NUMA_PTE_UPDATES, nr_updated);

}

tlb_finish_mmu(&tlb);

return nr_updated;

}

#endif /* CONFIG_NUMA_BALANCING */

static int queue_pages_test_walk(unsigned long start, unsigned long end,

struct mm_walk *walk)

{

struct vm_area_struct *next, *vma = walk->vma;

struct queue_pages *qp = walk->private;

unsigned long flags = qp->flags;

/* range check first */

VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma);

if (!qp->first) {

qp->first = vma;

if (!(flags & MPOL_MF_DISCONTIG_OK) &&

(qp->start < vma->vm_start))

/* hole at head side of range */

return -EFAULT;

}

next = find_vma(vma->vm_mm, vma->vm_end);

if (!(flags & MPOL_MF_DISCONTIG_OK) &&

((vma->vm_end < qp->end) &&

(!next || vma->vm_end < next->vm_start)))

/* hole at middle or tail of range */

return -EFAULT;

/*

* Need check MPOL_MF_STRICT to return -EIO if possible

* regardless of vma_migratable

*/

if (!vma_migratable(vma) &&

!(flags & MPOL_MF_STRICT))

return 1;

/*

* Check page nodes, and queue pages to move, in the current vma.

* But if no moving, and no strict checking, the scan can be skipped.

*/

if (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))

return 0;

return 1;

}

static const struct mm_walk_ops queue_pages_walk_ops = {

.hugetlb_entry = queue_folios_hugetlb,

.pmd_entry = queue_folios_pte_range,

.test_walk = queue_pages_test_walk,

.walk_lock = PGWALK_RDLOCK,

};

static const struct mm_walk_ops queue_pages_lock_vma_walk_ops = {

.hugetlb_entry = queue_folios_hugetlb,

.pmd_entry = queue_folios_pte_range,

.test_walk = queue_pages_test_walk,

.walk_lock = PGWALK_WRLOCK,

};

/*

* Walk through page tables and collect pages to be migrated.

*

* If pages found in a given range are not on the required set of @nodes,

* and migration is allowed, they are isolated and queued to @pagelist.

*

* queue_pages_range() may return:

* 0 - all pages already on the right node, or successfully queued for moving

* (or neither strict checking nor moving requested: only range checking).

* >0 - this number of misplaced folios could not be queued for moving

* (a hugetlbfs page or a transparent huge page being counted as 1).

* -EIO - a misplaced page found, when MPOL_MF_STRICT specified without MOVEs.

* -EFAULT - a hole in the memory range, when MPOL_MF_DISCONTIG_OK unspecified.

*/

static long

queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,

nodemask_t *nodes, unsigned long flags,

struct list_head *pagelist)

{

int err;

struct queue_pages qp = {

.pagelist = pagelist,

.flags = flags,

.nmask = nodes,

.start = start,

.end = end,

.first = NULL,

};

const struct mm_walk_ops *ops = (flags & MPOL_MF_WRLOCK) ?

&queue_pages_lock_vma_walk_ops : &queue_pages_walk_ops;

err = walk_page_range(mm, start, end, ops, &qp);

if (!qp.first)

/* whole range in hole */

err = -EFAULT;