1// SPDX-License-Identifier: GPL-2.0
2/*
3 * mm/pgtable-generic.c
4 *
5 * Generic pgtable methods declared in linux/pgtable.h
6 *
7 * Copyright (C) 2010 Linus Torvalds
8 */
9
10#include <linux/pagemap.h>
11#include <linux/hugetlb.h>
12#include <linux/pgtable.h>
13#include <linux/swap.h>
14#include <linux/swapops.h>
15#include <linux/mm_inline.h>
16#include <linux/iommu.h>
17#include <linux/pgalloc.h>
18
19#include <asm/tlb.h>
20
21/*
22 * If a p?d_bad entry is found while walking page tables, report
23 * the error, before resetting entry to p?d_none. Usually (but
24 * very seldom) called out from the p?d_none_or_clear_bad macros.
25 */
26
27void pgd_clear_bad(pgd_t *pgd)
28{
29 pgd_ERROR(*pgd);
30 pgd_clear(pgd);
31}
32
33#ifndef __PAGETABLE_P4D_FOLDED
34void p4d_clear_bad(p4d_t *p4d)
35{
36 p4d_ERROR(*p4d);
37 p4d_clear(p4dp: p4d);
38}
39#endif
40
41#ifndef __PAGETABLE_PUD_FOLDED
42void pud_clear_bad(pud_t *pud)
43{
44 pud_ERROR(*pud);
45 pud_clear(pudp: pud);
46}
47#endif
48
49/*
50 * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
51 * above. pmd folding is special and typically pmd_* macros refer to upper
52 * level even when folded
53 */
54void pmd_clear_bad(pmd_t *pmd)
55{
56 pmd_ERROR(*pmd);
57 pmd_clear(pmdp: pmd);
58}
59
60#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
61/*
62 * Only sets the access flags (dirty, accessed), as well as write
63 * permission. Furthermore, we know it always gets set to a "more
64 * permissive" setting, which allows most architectures to optimize
65 * this. We return whether the PTE actually changed, which in turn
66 * instructs the caller to do things like update__mmu_cache. This
67 * used to be done in the caller, but sparc needs minor faults to
68 * force that call on sun4c so we changed this macro slightly
69 */
70int ptep_set_access_flags(struct vm_area_struct *vma,
71 unsigned long address, pte_t *ptep,
72 pte_t entry, int dirty)
73{
74 int changed = !pte_same(ptep_get(ptep), entry);
75 if (changed) {
76 set_pte_at(vma->vm_mm, address, ptep, entry);
77 flush_tlb_fix_spurious_fault(vma, address, ptep);
78 }
79 return changed;
80}
81#endif
82
83#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
84int ptep_clear_flush_young(struct vm_area_struct *vma,
85 unsigned long address, pte_t *ptep)
86{
87 int young;
88 young = ptep_test_and_clear_young(vma, address, ptep);
89 if (young)
90 flush_tlb_page(vma, address);
91 return young;
92}
93#endif
94
95#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
96pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
97 pte_t *ptep)
98{
99 struct mm_struct *mm = (vma)->vm_mm;
100 pte_t pte;
101 pte = ptep_get_and_clear(mm, addr: address, ptep);
102 if (pte_accessible(mm, a: pte))
103 flush_tlb_page(vma, a: address);
104 return pte;
105}
106#endif
107
108#ifdef CONFIG_TRANSPARENT_HUGEPAGE
109
110#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
111int pmdp_set_access_flags(struct vm_area_struct *vma,
112 unsigned long address, pmd_t *pmdp,
113 pmd_t entry, int dirty)
114{
115 int changed = !pmd_same(*pmdp, entry);
116 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
117 if (changed) {
118 set_pmd_at(vma->vm_mm, address, pmdp, entry);
119 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
120 }
121 return changed;
122}
123#endif
124
125#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
126int pmdp_clear_flush_young(struct vm_area_struct *vma,
127 unsigned long address, pmd_t *pmdp)
128{
129 int young;
130 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
131 young = pmdp_test_and_clear_young(vma, address, pmdp);
132 if (young)
133 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
134 return young;
135}
136#endif
137
138#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
139pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
140 pmd_t *pmdp)
141{
142 pmd_t pmd;
143 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
144 VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp));
145 pmd = pmdp_huge_get_and_clear(mm: vma->vm_mm, addr: address, pmdp);
146 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
147 return pmd;
148}
149
150#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
151pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
152 pud_t *pudp)
153{
154 pud_t pud;
155
156 VM_BUG_ON(address & ~HPAGE_PUD_MASK);
157 VM_BUG_ON(!pud_trans_huge(*pudp));
158 pud = pudp_huge_get_and_clear(mm: vma->vm_mm, addr: address, pudp);
159 flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);
160 return pud;
161}
162#endif
163#endif
164
165#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
166void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
167 pgtable_t pgtable)
168{
169 assert_spin_locked(pmd_lockptr(mm, pmdp));
170
171 /* FIFO */
172 if (!pmd_huge_pte(mm, pmdp))
173 INIT_LIST_HEAD(list: &pgtable->lru);
174 else
175 list_add(new: &pgtable->lru, head: &pmd_huge_pte(mm, pmdp)->lru);
176 pmd_huge_pte(mm, pmdp) = pgtable;
177}
178#endif
179
180#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
181/* no "address" argument so destroys page coloring of some arch */
182pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
183{
184 pgtable_t pgtable;
185
186 assert_spin_locked(pmd_lockptr(mm, pmdp));
187
188 /* FIFO */
189 pgtable = pmd_huge_pte(mm, pmdp);
190 pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
191 struct page, lru);
192 if (pmd_huge_pte(mm, pmdp))
193 list_del(entry: &pgtable->lru);
194 return pgtable;
195}
196#endif
197
198#ifndef __HAVE_ARCH_PMDP_INVALIDATE
199pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
200 pmd_t *pmdp)
201{
202 VM_WARN_ON_ONCE(!pmd_present(*pmdp));
203 pmd_t old = pmdp_establish(vma, address, pmdp, pmd: pmd_mkinvalid(pmd: *pmdp));
204 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
205 return old;
206}
207#endif
208
209#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
210pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
211 pmd_t *pmdp)
212{
213 VM_WARN_ON_ONCE(!pmd_present(*pmdp));
214 return pmdp_invalidate(vma, address, pmdp);
215}
216#endif
217
218#ifndef pmdp_collapse_flush
219pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
220 pmd_t *pmdp)
221{
222 /*
223 * pmd and hugepage pte format are same. So we could
224 * use the same function.
225 */
226 pmd_t pmd;
227
228 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
229 VM_BUG_ON(pmd_trans_huge(*pmdp));
230 pmd = pmdp_huge_get_and_clear(mm: vma->vm_mm, addr: address, pmdp);
231
232 /* collapse entails shooting down ptes not pmd */
233 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
234 return pmd;
235}
236#endif
237
238/* arch define pte_free_defer in asm/pgalloc.h for its own implementation */
239#ifndef pte_free_defer
240static void pte_free_now(struct rcu_head *head)
241{
242 struct page *page;
243
244 page = container_of(head, struct page, rcu_head);
245 pte_free(NULL /* mm not passed and not used */, pte_page: (pgtable_t)page);
246}
247
248void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
249{
250 struct page *page;
251
252 page = pgtable;
253 call_rcu(head: &page->rcu_head, func: pte_free_now);
254}
255#endif /* pte_free_defer */
256#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
257
258#if defined(CONFIG_GUP_GET_PXX_LOW_HIGH) && \
259 (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RCU))
260/*
261 * See the comment above ptep_get_lockless() in include/linux/pgtable.h:
262 * the barriers in pmdp_get_lockless() cannot guarantee that the value in
263 * pmd_high actually belongs with the value in pmd_low; but holding interrupts
264 * off blocks the TLB flush between present updates, which guarantees that a
265 * successful __pte_offset_map() points to a page from matched halves.
266 */
267static unsigned long pmdp_get_lockless_start(void)
268{
269 unsigned long irqflags;
270
271 local_irq_save(irqflags);
272 return irqflags;
273}
274static void pmdp_get_lockless_end(unsigned long irqflags)
275{
276 local_irq_restore(irqflags);
277}
278#else
279static unsigned long pmdp_get_lockless_start(void) { return 0; }
280static void pmdp_get_lockless_end(unsigned long irqflags) { }
281#endif
282
283pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
284{
285 unsigned long irqflags;
286 pmd_t pmdval;
287
288 rcu_read_lock();
289 irqflags = pmdp_get_lockless_start();
290 pmdval = pmdp_get_lockless(pmdp: pmd);
291 pmdp_get_lockless_end(irqflags);
292
293 if (pmdvalp)
294 *pmdvalp = pmdval;
295 if (unlikely(pmd_none(pmdval) || !pmd_present(pmdval)))
296 goto nomap;
297 if (unlikely(pmd_trans_huge(pmdval)))
298 goto nomap;
299 if (unlikely(pmd_bad(pmdval))) {
300 pmd_clear_bad(pmd);
301 goto nomap;
302 }
303 return __pte_map(pmd: &pmdval, address: addr);
304nomap:
305 rcu_read_unlock();
306 return NULL;
307}
308
309pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
310 unsigned long addr, spinlock_t **ptlp)
311{
312 pmd_t pmdval;
313 pte_t *pte;
314
315 pte = __pte_offset_map(pmd, addr, pmdvalp: &pmdval);
316 if (likely(pte))
317 *ptlp = pte_lockptr(mm, pmd: &pmdval);
318 return pte;
319}
320
321pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
322 unsigned long addr, pmd_t *pmdvalp,
323 spinlock_t **ptlp)
324{
325 pte_t *pte;
326
327 VM_WARN_ON_ONCE(!pmdvalp);
328 pte = __pte_offset_map(pmd, addr, pmdvalp);
329 if (likely(pte))
330 *ptlp = pte_lockptr(mm, pmd: pmdvalp);
331 return pte;
332}
333
334/*
335 * pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation
336 * __pte_offset_map_lock() below, is usually called with the pmd pointer for
337 * addr, reached by walking down the mm's pgd, p4d, pud for addr: either while
338 * holding mmap_lock or vma lock for read or for write; or in truncate or rmap
339 * context, while holding file's i_mmap_lock or anon_vma lock for read (or for
340 * write). In a few cases, it may be used with pmd pointing to a pmd_t already
341 * copied to or constructed on the stack.
342 *
343 * When successful, it returns the pte pointer for addr, with its page table
344 * kmapped if necessary (when CONFIG_HIGHPTE), and locked against concurrent
345 * modification by software, with a pointer to that spinlock in ptlp (in some
346 * configs mm->page_table_lock, in SPLIT_PTLOCK configs a spinlock in table's
347 * struct page). pte_unmap_unlock(pte, ptl) to unlock and unmap afterwards.
348 *
349 * But it is unsuccessful, returning NULL with *ptlp unchanged, if there is no
350 * page table at *pmd: if, for example, the page table has just been removed,
351 * or replaced by the huge pmd of a THP. (When successful, *pmd is rechecked
352 * after acquiring the ptlock, and retried internally if it changed: so that a
353 * page table can be safely removed or replaced by THP while holding its lock.)
354 *
355 * pte_offset_map(pmd, addr), and its internal helper __pte_offset_map() above,
356 * just returns the pte pointer for addr, its page table kmapped if necessary;
357 * or NULL if there is no page table at *pmd. It does not attempt to lock the
358 * page table, so cannot normally be used when the page table is to be updated,
359 * or when entries read must be stable. But it does take rcu_read_lock(): so
360 * that even when page table is racily removed, it remains a valid though empty
361 * and disconnected table. Until pte_unmap(pte) unmaps and rcu_read_unlock()s
362 * afterwards.
363 *
364 * pte_offset_map_ro_nolock(mm, pmd, addr, ptlp), above, is like pte_offset_map();
365 * but when successful, it also outputs a pointer to the spinlock in ptlp - as
366 * pte_offset_map_lock() does, but in this case without locking it. This helps
367 * the caller to avoid a later pte_lockptr(mm, *pmd), which might by that time
368 * act on a changed *pmd: pte_offset_map_ro_nolock() provides the correct spinlock
369 * pointer for the page table that it returns. Even after grabbing the spinlock,
370 * we might be looking either at a page table that is still mapped or one that
371 * was unmapped and is about to get freed. But for R/O access this is sufficient.
372 * So it is only applicable for read-only cases where any modification operations
373 * to the page table are not allowed even if the corresponding spinlock is held
374 * afterwards.
375 *
376 * pte_offset_map_rw_nolock(mm, pmd, addr, pmdvalp, ptlp), above, is like
377 * pte_offset_map_ro_nolock(); but when successful, it also outputs the pdmval.
378 * It is applicable for may-write cases where any modification operations to the
379 * page table may happen after the corresponding spinlock is held afterwards.
380 * But the users should make sure the page table is stable like checking pte_same()
381 * or checking pmd_same() by using the output pmdval before performing the write
382 * operations.
383 *
384 * Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will
385 * be read-only/read-write protected.
386 *
387 * Note that free_pgtables(), used after unmapping detached vmas, or when
388 * exiting the whole mm, does not take page table lock before freeing a page
389 * table, and may not use RCU at all: "outsiders" like khugepaged should avoid
390 * pte_offset_map() and co once the vma is detached from mm or mm_users is zero.
391 */
392pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
393 unsigned long addr, spinlock_t **ptlp)
394{
395 spinlock_t *ptl;
396 pmd_t pmdval;
397 pte_t *pte;
398again:
399 pte = __pte_offset_map(pmd, addr, pmdvalp: &pmdval);
400 if (unlikely(!pte))
401 return pte;
402 ptl = pte_lockptr(mm, pmd: &pmdval);
403 spin_lock(lock: ptl);
404 if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
405 *ptlp = ptl;
406 return pte;
407 }
408 pte_unmap_unlock(pte, ptl);
409 goto again;
410}
411
412#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE
413static void kernel_pgtable_work_func(struct work_struct *work);
414
415static struct {
416 struct list_head list;
417 /* protect above ptdesc lists */
418 spinlock_t lock;
419 struct work_struct work;
420} kernel_pgtable_work = {
421 .list = LIST_HEAD_INIT(kernel_pgtable_work.list),
422 .lock = __SPIN_LOCK_UNLOCKED(kernel_pgtable_work.lock),
423 .work = __WORK_INITIALIZER(kernel_pgtable_work.work, kernel_pgtable_work_func),
424};
425
426static void kernel_pgtable_work_func(struct work_struct *work)
427{
428 struct ptdesc *pt, *next;
429 LIST_HEAD(page_list);
430
431 spin_lock(lock: &kernel_pgtable_work.lock);
432 list_splice_tail_init(list: &kernel_pgtable_work.list, head: &page_list);
433 spin_unlock(lock: &kernel_pgtable_work.lock);
434
435 iommu_sva_invalidate_kva_range(PAGE_OFFSET, TLB_FLUSH_ALL);
436 list_for_each_entry_safe(pt, next, &page_list, pt_list)
437 __pagetable_free(pt);
438}
439
440void pagetable_free_kernel(struct ptdesc *pt)
441{
442 spin_lock(lock: &kernel_pgtable_work.lock);
443 list_add(new: &pt->pt_list, head: &kernel_pgtable_work.list);
444 spin_unlock(lock: &kernel_pgtable_work.lock);
445
446 schedule_work(work: &kernel_pgtable_work.work);
447}
448#endif
449

source code of linux/mm/pgtable-generic.c