/* SPDX-License-Identifier: GPL-2.0-or-later */

/* internal.h: mm/ internal definitions

*

* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.

* Written by David Howells (dhowells@redhat.com)

*/

#ifndef __MM_INTERNAL_H

#define __MM_INTERNAL_H

#include <linux/fs.h>

#include <linux/khugepaged.h>

#include <linux/mm.h>

#include <linux/mm_inline.h>

#include <linux/mmu_notifier.h>

#include <linux/pagemap.h>

#include <linux/pagewalk.h>

#include <linux/rmap.h>

#include <linux/swap.h>

#include <linux/leafops.h>

#include <linux/swap_cgroup.h>

#include <linux/tracepoint-defs.h>

/* Internal core VMA manipulation functions. */

#include "vma.h"

struct folio_batch;

/*

* Maintains state across a page table move. The operation assumes both source

* and destination VMAs already exist and are specified by the user.

*

* Partial moves are permitted, but the old and new ranges must both reside

* within a VMA.

*

* mmap lock must be held in write and VMA write locks must be held on any VMA

* that is visible.

*

* Use the PAGETABLE_MOVE() macro to initialise this struct.

*

* The old_addr and new_addr fields are updated as the page table move is

* executed.

*

* NOTE: The page table move is affected by reading from [old_addr, old_end),

* and old_addr may be updated for better page table alignment, so len_in

* represents the length of the range being copied as specified by the user.

*/

struct pagetable_move_control {

struct vm_area_struct *old; /* Source VMA. */

struct vm_area_struct *new; /* Destination VMA. */

unsigned long old_addr; /* Address from which the move begins. */

unsigned long old_end; /* Exclusive address at which old range ends. */

unsigned long new_addr; /* Address to move page tables to. */

unsigned long len_in; /* Bytes to remap specified by user. */

bool need_rmap_locks; /* Do rmap locks need to be taken? */

bool for_stack; /* Is this an early temp stack being moved? */

};

#define PAGETABLE_MOVE(name, old_, new_, old_addr_, new_addr_, len_) \

struct pagetable_move_control name = { \

.old = old_, \

.new = new_, \

.old_addr = old_addr_, \

.old_end = (old_addr_) + (len_), \

.new_addr = new_addr_, \

.len_in = len_, \

}

/*

* The set of flags that only affect watermark checking and reclaim

* behaviour. This is used by the MM to obey the caller constraints

* about IO, FS and watermark checking while ignoring placement

* hints such as HIGHMEM usage.

*/

#define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\

__GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\

__GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\

__GFP_NOLOCKDEP)

/* The GFP flags allowed during early boot */

#define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS))

/* Control allocation cpuset and node placement constraints */

#define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE)

/* Do not use these with a slab allocator */

#define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)

/*

* Different from WARN_ON_ONCE(), no warning will be issued

* when we specify __GFP_NOWARN.

*/

#define WARN_ON_ONCE_GFP(cond, gfp) ({ \

static bool __section(".data..once") __warned; \

int __ret_warn_once = !!(cond); \

\

if (unlikely(!(gfp & __GFP_NOWARN) && __ret_warn_once && !__warned)) { \

__warned = true; \

WARN_ON(1); \

} \

unlikely(__ret_warn_once); \

})

void page_writeback_init(void);

/*

* If a 16GB hugetlb folio were mapped by PTEs of all of its 4kB pages,

* its nr_pages_mapped would be 0x400000: choose the ENTIRELY_MAPPED bit

* above that range, instead of 2*(PMD_SIZE/PAGE_SIZE). Hugetlb currently

* leaves nr_pages_mapped at 0, but avoid surprise if it participates later.

*/

#define ENTIRELY_MAPPED 0x800000

#define FOLIO_PAGES_MAPPED (ENTIRELY_MAPPED - 1)

/*

* Flags passed to __show_mem() and show_free_areas() to suppress output in

* various contexts.

*/

#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */

/*

* How many individual pages have an elevated _mapcount. Excludes

* the folio's entire_mapcount.

*

* Don't use this function outside of debugging code.

*/

static inline int folio_nr_pages_mapped(const struct folio *folio)

{

if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT))

return -1;

return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED;

}

/*

* Retrieve the first entry of a folio based on a provided entry within the

* folio. We cannot rely on folio->swap as there is no guarantee that it has

* been initialized. Used for calling arch_swap_restore()

*/

static inline swp_entry_t folio_swap(swp_entry_t entry,

const struct folio *folio)

{

swp_entry_t swap = {

.val = ALIGN_DOWN(entry.val, folio_nr_pages(folio)),

};

return swap;

}

static inline void *folio_raw_mapping(const struct folio *folio)

{

unsigned long mapping = (unsigned long)folio->mapping;

return (void *)(mapping & ~FOLIO_MAPPING_FLAGS);

}

/*

* This is a file-backed mapping, and is about to be memory mapped - invoke its

* mmap hook and safely handle error conditions. On error, VMA hooks will be

* mutated.

*

* @file: File which backs the mapping.

* @vma: VMA which we are mapping.

*

* Returns: 0 if success, error otherwise.

*/

static inline int mmap_file(struct file *file, struct vm_area_struct *vma)

{

int err = vfs_mmap(file, vma);

if (likely(!err))

return 0;

/*

* OK, we tried to call the file hook for mmap(), but an error

* arose. The mapping is in an inconsistent state and we must not invoke

* any further hooks on it.

*/

vma->vm_ops = &vma_dummy_vm_ops;

return err;

}

/*

* If the VMA has a close hook then close it, and since closing it might leave

* it in an inconsistent state which makes the use of any hooks suspect, clear

* them down by installing dummy empty hooks.

*/

static inline void vma_close(struct vm_area_struct *vma)

{

if (vma->vm_ops && vma->vm_ops->close) {

vma->vm_ops->close(vma);

/*

* The mapping is in an inconsistent state, and no further hooks

* may be invoked upon it.

*/

vma->vm_ops = &vma_dummy_vm_ops;

}

}

/* unmap_vmas is in mm/memory.c */

void unmap_vmas(struct mmu_gather *tlb, struct unmap_desc *unmap);

#ifdef CONFIG_MMU

static inline void get_anon_vma(struct anon_vma *anon_vma)

{

atomic_inc(&anon_vma->refcount);

}

void __put_anon_vma(struct anon_vma *anon_vma);

static inline void put_anon_vma(struct anon_vma *anon_vma)

{

if (atomic_dec_and_test(&anon_vma->refcount))

__put_anon_vma(anon_vma);

}

static inline void anon_vma_lock_write(struct anon_vma *anon_vma)

{

down_write(&anon_vma->root->rwsem);

}

static inline int anon_vma_trylock_write(struct anon_vma *anon_vma)

{

return down_write_trylock(&anon_vma->root->rwsem);

}

static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)

{

up_write(&anon_vma->root->rwsem);

}

static inline void anon_vma_lock_read(struct anon_vma *anon_vma)

{

down_read(&anon_vma->root->rwsem);

}

static inline int anon_vma_trylock_read(struct anon_vma *anon_vma)

{

return down_read_trylock(&anon_vma->root->rwsem);

}

static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)

{

up_read(&anon_vma->root->rwsem);

}

struct anon_vma *folio_get_anon_vma(const struct folio *folio);

/* Operations which modify VMAs. */

enum vma_operation {

VMA_OP_SPLIT,

VMA_OP_MERGE_UNFAULTED,

VMA_OP_REMAP,

VMA_OP_FORK,

};

int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src,

enum vma_operation operation);

int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma);

int __anon_vma_prepare(struct vm_area_struct *vma);

void unlink_anon_vmas(struct vm_area_struct *vma);

static inline int anon_vma_prepare(struct vm_area_struct *vma)

{

if (likely(vma->anon_vma))

return 0;

return __anon_vma_prepare(vma);

}

/* Flags for folio_pte_batch(). */

typedef int __bitwise fpb_t;

/* Compare PTEs respecting the dirty bit. */

#define FPB_RESPECT_DIRTY ((__force fpb_t)BIT(0))

/* Compare PTEs respecting the soft-dirty bit. */

#define FPB_RESPECT_SOFT_DIRTY ((__force fpb_t)BIT(1))

/* Compare PTEs respecting the writable bit. */

#define FPB_RESPECT_WRITE ((__force fpb_t)BIT(2))

/*

* Merge PTE write bits: if any PTE in the batch is writable, modify the

* PTE at @ptentp to be writable.

*/

#define FPB_MERGE_WRITE ((__force fpb_t)BIT(3))

/*

* Merge PTE young and dirty bits: if any PTE in the batch is young or dirty,

* modify the PTE at @ptentp to be young or dirty, respectively.

*/

#define FPB_MERGE_YOUNG_DIRTY ((__force fpb_t)BIT(4))

static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)

{

if (!(flags & FPB_RESPECT_DIRTY))

pte = pte_mkclean(pte);

if (likely(!(flags & FPB_RESPECT_SOFT_DIRTY)))

pte = pte_clear_soft_dirty(pte);

if (likely(!(flags & FPB_RESPECT_WRITE)))

pte = pte_wrprotect(pte);

return pte_mkold(pte);

}

/**

* folio_pte_batch_flags - detect a PTE batch for a large folio

* @folio: The large folio to detect a PTE batch for.

* @vma: The VMA. Only relevant with FPB_MERGE_WRITE, otherwise can be NULL.

* @ptep: Page table pointer for the first entry.

* @ptentp: Pointer to a COPY of the first page table entry whose flags this

* function updates based on @flags if appropriate.

* @max_nr: The maximum number of table entries to consider.

* @flags: Flags to modify the PTE batch semantics.

*

* Detect a PTE batch: consecutive (present) PTEs that map consecutive

* pages of the same large folio in a single VMA and a single page table.

*

* All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,

* the accessed bit, writable bit, dirty bit (unless FPB_RESPECT_DIRTY is set)

* and soft-dirty bit (unless FPB_RESPECT_SOFT_DIRTY is set).

*

* @ptep must map any page of the folio. max_nr must be at least one and

* must be limited by the caller so scanning cannot exceed a single VMA and

* a single page table.

*

* Depending on the FPB_MERGE_* flags, the pte stored at @ptentp will

* be updated: it's crucial that a pointer to a COPY of the first

* page table entry, obtained through ptep_get(), is provided as @ptentp.

*

* This function will be inlined to optimize based on the input parameters;

* consider using folio_pte_batch() instead if applicable.

*

* Return: the number of table entries in the batch.

*/

static inline unsigned int folio_pte_batch_flags(struct folio *folio,

struct vm_area_struct *vma, pte_t *ptep, pte_t *ptentp,

unsigned int max_nr, fpb_t flags)

{

bool any_writable = false, any_young = false, any_dirty = false;

pte_t expected_pte, pte = *ptentp;

unsigned int nr, cur_nr;

VM_WARN_ON_FOLIO(!pte_present(pte), folio);

VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio);

VM_WARN_ON_FOLIO(page_folio(pfn_to_page(pte_pfn(pte))) != folio, folio);

/*

* Ensure this is a pointer to a copy not a pointer into a page table.

* If this is a stack value, it won't be a valid virtual address, but

* that's fine because it also cannot be pointing into the page table.

*/

VM_WARN_ON(virt_addr_valid(ptentp) && PageTable(virt_to_page(ptentp)));

/* Limit max_nr to the actual remaining PFNs in the folio we could batch. */

max_nr = min_t(unsigned long, max_nr,

folio_pfn(folio) + folio_nr_pages(folio) - pte_pfn(pte));

nr = pte_batch_hint(ptep, pte);

expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags);

ptep = ptep + nr;

while (nr < max_nr) {

pte = ptep_get(ptep);

if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected_pte))

break;

if (flags & FPB_MERGE_WRITE)

any_writable |= pte_write(pte);

if (flags & FPB_MERGE_YOUNG_DIRTY) {

any_young |= pte_young(pte);

any_dirty |= pte_dirty(pte);

}

cur_nr = pte_batch_hint(ptep, pte);

expected_pte = pte_advance_pfn(expected_pte, cur_nr);

ptep += cur_nr;

nr += cur_nr;

}

if (any_writable)

*ptentp = pte_mkwrite(*ptentp, vma);

if (any_young)

*ptentp = pte_mkyoung(*ptentp);

if (any_dirty)

*ptentp = pte_mkdirty(*ptentp);

return min(nr, max_nr);

}

unsigned int folio_pte_batch(struct folio *folio, pte_t *ptep, pte_t pte,

unsigned int max_nr);

/**

* pte_move_swp_offset - Move the swap entry offset field of a swap pte

* forward or backward by delta

* @pte: The initial pte state; must be a swap entry

* @delta: The direction and the offset we are moving; forward if delta

* is positive; backward if delta is negative

*

* Moves the swap offset, while maintaining all other fields, including

* swap type, and any swp pte bits. The resulting pte is returned.

*/

static inline pte_t pte_move_swp_offset(pte_t pte, long delta)

{

const softleaf_t entry = softleaf_from_pte(pte);

pte_t new = __swp_entry_to_pte(__swp_entry(swp_type(entry),

(swp_offset(entry) + delta)));

if (pte_swp_soft_dirty(pte))

new = pte_swp_mksoft_dirty(new);

if (pte_swp_exclusive(pte))

new = pte_swp_mkexclusive(new);

if (pte_swp_uffd_wp(pte))

new = pte_swp_mkuffd_wp(new);

return new;

}

/**

* pte_next_swp_offset - Increment the swap entry offset field of a swap pte.

* @pte: The initial pte state; must be a swap entry.

*

* Increments the swap offset, while maintaining all other fields, including

* swap type, and any swp pte bits. The resulting pte is returned.

*/

static inline pte_t pte_next_swp_offset(pte_t pte)

{

return pte_move_swp_offset(pte, 1);

}

/**

* swap_pte_batch - detect a PTE batch for a set of contiguous swap entries

* @start_ptep: Page table pointer for the first entry.

* @max_nr: The maximum number of table entries to consider.

* @pte: Page table entry for the first entry.

*

* Detect a batch of contiguous swap entries: consecutive (non-present) PTEs

* containing swap entries all with consecutive offsets and targeting the same

* swap type, all with matching swp pte bits.

*

* max_nr must be at least one and must be limited by the caller so scanning

* cannot exceed a single page table.

*

* Return: the number of table entries in the batch.

*/

static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte)

{

pte_t expected_pte = pte_next_swp_offset(pte);

const pte_t *end_ptep = start_ptep + max_nr;

const softleaf_t entry = softleaf_from_pte(pte);

pte_t *ptep = start_ptep + 1;

unsigned short cgroup_id;

VM_WARN_ON(max_nr < 1);

VM_WARN_ON(!softleaf_is_swap(entry));

cgroup_id = lookup_swap_cgroup_id(entry);

while (ptep < end_ptep) {

softleaf_t entry;

pte = ptep_get(ptep);

if (!pte_same(pte, expected_pte))

break;

entry = softleaf_from_pte(pte);

if (lookup_swap_cgroup_id(entry) != cgroup_id)

break;

expected_pte = pte_next_swp_offset(expected_pte);

ptep++;

}

return ptep - start_ptep;

}

#endif /* CONFIG_MMU */

void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,

int nr_throttled);

static inline void acct_reclaim_writeback(struct folio *folio)

{

pg_data_t *pgdat = folio_pgdat(folio);

int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled);

if (nr_throttled)

__acct_reclaim_writeback(pgdat, folio, nr_throttled);

}

static inline void wake_throttle_isolated(pg_data_t *pgdat)

{

wait_queue_head_t *wqh;

wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED];

if (waitqueue_active(wqh))

wake_up(wqh);

}

vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf);

static inline vm_fault_t vmf_anon_prepare(struct vm_fault *vmf)

{

vm_fault_t ret = __vmf_anon_prepare(vmf);

if (unlikely(ret & VM_FAULT_RETRY))

vma_end_read(vmf->vma);

return ret;

}

vm_fault_t do_swap_page(struct vm_fault *vmf);

void folio_rotate_reclaimable(struct folio *folio);

bool __folio_end_writeback(struct folio *folio);

void deactivate_file_folio(struct folio *folio);

void folio_activate(struct folio *folio);

void free_pgtables(struct mmu_gather *tlb, struct unmap_desc *desc);

void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);

/**

* sync_with_folio_pmd_zap - sync with concurrent zapping of a folio PMD

* @mm: The mm_struct.

* @pmdp: Pointer to the pmd that was found to be pmd_none().

*

* When we find a pmd_none() while unmapping a folio without holding the PTL,

* zap_huge_pmd() may have cleared the PMD but not yet modified the folio to

* indicate that it's unmapped. Skipping the PMD without synchronization could

* make folio unmapping code assume that unmapping failed.

*

* Wait for concurrent zapping to complete by grabbing the PTL.

*/

static inline void sync_with_folio_pmd_zap(struct mm_struct *mm, pmd_t *pmdp)

{

spinlock_t *ptl = pmd_lock(mm, pmdp);

spin_unlock(ptl);

}

struct zap_details;

void zap_vma_range_batched(struct mmu_gather *tlb,

struct vm_area_struct *vma, unsigned long addr,

unsigned long size, struct zap_details *details);

int zap_vma_for_reaping(struct vm_area_struct *vma);

int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio,

gfp_t gfp);

void page_cache_ra_order(struct readahead_control *, struct file_ra_state *);

void force_page_cache_ra(struct readahead_control *, unsigned long nr);

static inline void force_page_cache_readahead(struct address_space *mapping,

struct file *file, pgoff_t index, unsigned long nr_to_read)

{

DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, index);

force_page_cache_ra(&ractl, nr_to_read);

}

unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,

pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);

unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,

pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);

int truncate_inode_folio(struct address_space *mapping, struct folio *folio);

bool truncate_inode_partial_folio(struct folio *folio, loff_t start,

loff_t end);

long mapping_evict_folio(struct address_space *mapping, struct folio *folio);

unsigned long mapping_try_invalidate(struct address_space *mapping,

pgoff_t start, pgoff_t end, unsigned long *nr_failed);

/**

* folio_evictable - Test whether a folio is evictable.

* @folio: The folio to test.

*

* Test whether @folio is evictable -- i.e., should be placed on

* active/inactive lists vs unevictable list.

*

* Reasons folio might not be evictable:

* 1. folio's mapping marked unevictable

* 2. One of the pages in the folio is part of an mlocked VMA

*/

static inline bool folio_evictable(struct folio *folio)

{

bool ret;

/* Prevent address_space of inode and swap cache from being freed */

rcu_read_lock();

ret = !mapping_unevictable(folio_mapping(folio)) &&

!folio_test_mlocked(folio);

rcu_read_unlock();

return ret;

}

/*

* Turn a non-refcounted page (->_refcount == 0) into refcounted with

* a count of one.

*/

static inline void set_page_refcounted(struct page *page)

{

VM_BUG_ON_PAGE(PageTail(page), page);

VM_BUG_ON_PAGE(page_ref_count(page), page);

set_page_count(page, 1);

}

static inline void set_pages_refcounted(struct page *page, unsigned long nr_pages)

{

unsigned long pfn = page_to_pfn(page);

for (; nr_pages--; pfn++)

set_page_refcounted(pfn_to_page(pfn));

}

/*

* Return true if a folio needs ->release_folio() calling upon it.

*/

static inline bool folio_needs_release(struct folio *folio)

{

struct address_space *mapping = folio_mapping(folio);

return folio_has_private(folio) ||

(mapping && mapping_release_always(mapping));

}

extern unsigned long highest_memmap_pfn;

/*

* Maximum number of reclaim retries without progress before the OOM

* killer is consider the only way forward.

*/

#define MAX_RECLAIM_RETRIES 16

/*

* in mm/vmscan.c:

*/

bool folio_isolate_lru(struct folio *folio);

void folio_putback_lru(struct folio *folio);

extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason);

int user_proactive_reclaim(char *buf,

struct mem_cgroup *memcg, pg_data_t *pgdat);

/*

* in mm/rmap.c:

*/

pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);

/*

* in mm/khugepaged.c

*/

void set_recommended_min_free_kbytes(void);

/*

* in mm/page_alloc.c

*/

#define K(x) ((x) << (PAGE_SHIFT-10))

extern char * const zone_names[MAX_NR_ZONES];

/* perform sanity checks on struct pages being allocated or freed */

DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);

extern int min_free_kbytes;

extern int defrag_mode;

void setup_per_zone_wmarks(void);

void calculate_min_free_kbytes(void);

int __meminit init_per_zone_wmark_min(void);

void page_alloc_sysctl_init(void);

/*

* Structure for holding the mostly immutable allocation parameters passed

* between functions involved in allocations, including the alloc_pages*

* family of functions.

*

* nodemask, migratetype and highest_zoneidx are initialized only once in

* __alloc_pages() and then never change.

*

* zonelist, preferred_zone and highest_zoneidx are set first in

* __alloc_pages() for the fast path, and might be later changed

* in __alloc_pages_slowpath(). All other functions pass the whole structure

* by a const pointer.

*/

struct alloc_context {

struct zonelist *zonelist;

nodemask_t *nodemask;

struct zoneref *preferred_zoneref;

int migratetype;

/*

* highest_zoneidx represents highest usable zone index of

* the allocation request. Due to the nature of the zone,

* memory on lower zone than the highest_zoneidx will be

* protected by lowmem_reserve[highest_zoneidx].

*

* highest_zoneidx is also used by reclaim/compaction to limit

* the target zone since higher zone than this index cannot be

* usable for this allocation request.

*/

enum zone_type highest_zoneidx;

bool spread_dirty_pages;

};

/*

* This function returns the order of a free page in the buddy system. In

* general, page_zone(page)->lock must be held by the caller to prevent the

* page from being allocated in parallel and returning garbage as the order.

* If a caller does not hold page_zone(page)->lock, it must guarantee that the

* page cannot be allocated or merged in parallel. Alternatively, it must

* handle invalid values gracefully, and use buddy_order_unsafe() below.

*/

static inline unsigned int buddy_order(struct page *page)

{

/* PageBuddy() must be checked by the caller */

return page_private(page);

}

/*

* Like buddy_order(), but for callers who cannot afford to hold the zone lock.

* PageBuddy() should be checked first by the caller to minimize race window,

* and invalid values must be handled gracefully.

*

* READ_ONCE is used so that if the caller assigns the result into a local

* variable and e.g. tests it for valid range before using, the compiler cannot

* decide to remove the variable and inline the page_private(page) multiple

* times, potentially observing different values in the tests and the actual

* use of the result.

*/

#define buddy_order_unsafe(page) READ_ONCE(page_private(page))

/*

* This function checks whether a page is free && is the buddy

* we can coalesce a page and its buddy if

* (a) the buddy is not in a hole (check before calling!) &&

* (b) the buddy is in the buddy system &&

* (c) a page and its buddy have the same order &&

* (d) a page and its buddy are in the same zone.

*

* For recording whether a page is in the buddy system, we set PageBuddy.

* Setting, clearing, and testing PageBuddy is serialized by zone->lock.

*

* For recording page's order, we use page_private(page).

*/

static inline bool page_is_buddy(struct page *page, struct page *buddy,

unsigned int order)

{

if (!page_is_guard(buddy) && !PageBuddy(buddy))

return false;

if (buddy_order(buddy) != order)

return false;

/*

* zone check is done late to avoid uselessly calculating

* zone/node ids for pages that could never merge.

*/

if (page_zone_id(page) != page_zone_id(buddy))

return false;

VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

return true;

}

/*

* Locate the struct page for both the matching buddy in our

* pair (buddy1) and the combined O(n+1) page they form (page).

*

* 1) Any buddy B1 will have an order O twin B2 which satisfies

* the following equation:

* B2 = B1 ^ (1 << O)

* For example, if the starting buddy (buddy2) is #8 its order

* 1 buddy is #10:

* B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10

*

* 2) Any buddy B will have an order O+1 parent P which

* satisfies the following equation:

* P = B & ~(1 << O)

*

* Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER

*/

static inline unsigned long

__find_buddy_pfn(unsigned long page_pfn, unsigned int order)

{

return page_pfn ^ (1 << order);

}

/*

* Find the buddy of @page and validate it.

* @page: The input page

* @pfn: The pfn of the page, it saves a call to page_to_pfn() when the

* function is used in the performance-critical __free_one_page().

* @order: The order of the page

* @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to

* page_to_pfn().

*

* The found buddy can be a non PageBuddy, out of @page's zone, or its order is

* not the same as @page. The validation is necessary before use it.

*

* Return: the found buddy page or NULL if not found.

*/

static inline struct page *find_buddy_page_pfn(struct page *page,

unsigned long pfn, unsigned int order, unsigned long *buddy_pfn)

{

unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order);

struct page *buddy;

buddy = page + (__buddy_pfn - pfn);

if (buddy_pfn)

*buddy_pfn = __buddy_pfn;

if (page_is_buddy(page, buddy, order))

return buddy;

return NULL;

}

extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,

unsigned long end_pfn, struct zone *zone);

static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,

unsigned long end_pfn, struct zone *zone)

{

if (zone->contiguous)

return pfn_to_page(start_pfn);

return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);

}

void set_zone_contiguous(struct zone *zone);

bool pfn_range_intersects_zones(int nid, unsigned long start_pfn,

unsigned long nr_pages);

static inline void clear_zone_contiguous(struct zone *zone)

{

zone->contiguous = false;

}

extern int __isolate_free_page(struct page *page, unsigned int order);

extern void __putback_isolated_page(struct page *page, unsigned int order,

int mt);

extern void memblock_free_pages(unsigned long pfn, unsigned int order);

extern void __free_pages_core(struct page *page, unsigned int order,

enum meminit_context context);

/*

* This will have no effect, other than possibly generating a warning, if the

* caller passes in a non-large folio.

*/

static inline void folio_set_order(struct folio *folio, unsigned int order)

{

if (WARN_ON_ONCE(!order || !folio_test_large(folio)))

return;

VM_WARN_ON_ONCE(order > MAX_FOLIO_ORDER);

folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order;

#ifdef NR_PAGES_IN_LARGE_FOLIO

folio->_nr_pages = 1U << order;

#endif

}

bool __folio_unqueue_deferred_split(struct folio *folio);

static inline bool folio_unqueue_deferred_split(struct folio *folio)

{

if (folio_order(folio) <= 1 || !folio_test_large_rmappable(folio))

return false;

/*

* At this point, there is no one trying to add the folio to

* deferred_list. If folio is not in deferred_list, it's safe

* to check without acquiring the split_queue_lock.

*/

if (data_race(list_empty(&folio->_deferred_list)))

return false;

return __folio_unqueue_deferred_split(folio);

}

static inline struct folio *page_rmappable_folio(struct page *page)

{

struct folio *folio = (struct folio *)page;

if (folio && folio_test_large(folio))

folio_set_large_rmappable(folio);

return folio;

}

static inline void prep_compound_head(struct page *page, unsigned int order)

{

struct folio *folio = (struct folio *)page;

folio_set_order(folio, order);

atomic_set(&folio->_large_mapcount, -1);

if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))

atomic_set(&folio->_nr_pages_mapped, 0);

if (IS_ENABLED(CONFIG_MM_ID)) {

folio->_mm_ids = 0;

folio->_mm_id_mapcount[0] = -1;

folio->_mm_id_mapcount[1] = -1;

}

if (IS_ENABLED(CONFIG_64BIT) || order > 1) {

atomic_set(&folio->_pincount, 0);

atomic_set(&folio->_entire_mapcount, -1);

}

if (order > 1)

INIT_LIST_HEAD(&folio->_deferred_list);

}

static inline void prep_compound_tail(struct page *tail,

const struct page *head, unsigned int order)

{

tail->mapping = TAIL_MAPPING;

set_compound_head(tail, head, order);

set_page_private(tail, 0);

}

static inline void init_compound_tail(struct page *tail,

const struct page *head, unsigned int order, struct zone *zone)

{

atomic_set(&tail->_mapcount, -1);

set_page_node(tail, zone_to_nid(zone));

set_page_zone(tail, zone_idx(zone));

prep_compound_tail(tail, head, order);

}

void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags);

extern bool free_pages_prepare(struct page *page, unsigned int order);

extern int user_min_free_kbytes;

struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid,

nodemask_t *);

#define __alloc_frozen_pages(...) \

alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))

void free_frozen_pages(struct page *page, unsigned int order);

void free_unref_folios(struct folio_batch *fbatch);

#ifdef CONFIG_NUMA

struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);

#else

static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)

{

return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL);

}

#endif

#define alloc_frozen_pages(...) \

alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__))

struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);

#define alloc_frozen_pages_nolock(...) \

alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__))

void free_frozen_pages_nolock(struct page *page, unsigned int order);

extern void zone_pcp_reset(struct zone *zone);

extern void zone_pcp_disable(struct zone *zone);

extern void zone_pcp_enable(struct zone *zone);

extern void zone_pcp_init(struct zone *zone);

extern void *memmap_alloc(phys_addr_t size, phys_addr_t align,

phys_addr_t min_addr,

int nid, bool exact_nid);

void memmap_init_range(unsigned long, int, unsigned long, unsigned long,

unsigned long, enum meminit_context, struct vmem_altmap *, int,

bool);

/*

* mm/sparse.c

*/

#ifdef CONFIG_SPARSEMEM

void sparse_init(void);

int sparse_index_init(unsigned long section_nr, int nid);

static inline void sparse_init_one_section(struct mem_section *ms,

unsigned long pnum, struct page *mem_map,

struct mem_section_usage *usage, unsigned long flags)

{

unsigned long coded_mem_map;

BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT);

/*

* We encode the start PFN of the section into the mem_map such that

* page_to_pfn() on !CONFIG_SPARSEMEM_VMEMMAP can simply subtract it

* from the page pointer to obtain the PFN.

*/

coded_mem_map = (unsigned long)(mem_map - section_nr_to_pfn(pnum));

VM_WARN_ON_ONCE(coded_mem_map & ~SECTION_MAP_MASK);

ms->section_mem_map &= ~SECTION_MAP_MASK;

ms->section_mem_map |= coded_mem_map;

ms->section_mem_map |= flags | SECTION_HAS_MEM_MAP;

ms->usage = usage;

}

static inline void __section_mark_present(struct mem_section *ms,

unsigned long section_nr)

{

if (section_nr > __highest_present_section_nr)

__highest_present_section_nr = section_nr;

ms->section_mem_map |= SECTION_MARKED_PRESENT;

}

#else

static inline void sparse_init(void) {}

#endif /* CONFIG_SPARSEMEM */