1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */
7#include <linux/kstrtox.h>
8#include <linux/mm.h>
9#include <linux/page_table_check.h>
10#include <linux/swap.h>
11#include <linux/leafops.h>
12
13#undef pr_fmt
14#define pr_fmt(fmt) "page_table_check: " fmt
15
16struct page_table_check {
17 atomic_t anon_map_count;
18 atomic_t file_map_count;
19};
20
21static bool __page_table_check_enabled __initdata =
22 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
23
24DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
25EXPORT_SYMBOL(page_table_check_disabled);
26
27static int __init early_page_table_check_param(char *buf)
28{
29 return kstrtobool(s: buf, res: &__page_table_check_enabled);
30}
31
32early_param("page_table_check", early_page_table_check_param);
33
34static bool __init need_page_table_check(void)
35{
36 return __page_table_check_enabled;
37}
38
39static void __init init_page_table_check(void)
40{
41 if (!__page_table_check_enabled)
42 return;
43 static_branch_disable(&page_table_check_disabled);
44}
45
46struct page_ext_operations page_table_check_ops = {
47 .size = sizeof(struct page_table_check),
48 .need = need_page_table_check,
49 .init = init_page_table_check,
50 .need_shared_flags = false,
51};
52
53static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
54{
55 BUG_ON(!page_ext);
56 return page_ext_data(page_ext, ops: &page_table_check_ops);
57}
58
59/*
60 * An entry is removed from the page table, decrement the counters for that page
61 * verify that it is of correct type and counters do not become negative.
62 */
63static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
64{
65 struct page_ext_iter iter;
66 struct page_ext *page_ext;
67 struct page *page;
68 bool anon;
69
70 if (!pfn_valid(pfn))
71 return;
72
73 page = pfn_to_page(pfn);
74 BUG_ON(PageSlab(page));
75 anon = PageAnon(page);
76
77 rcu_read_lock();
78 for_each_page_ext(page, pgcnt, page_ext, iter) {
79 struct page_table_check *ptc = get_page_table_check(page_ext);
80
81 if (anon) {
82 BUG_ON(atomic_read(&ptc->file_map_count));
83 BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
84 } else {
85 BUG_ON(atomic_read(&ptc->anon_map_count));
86 BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
87 }
88 }
89 rcu_read_unlock();
90}
91
92/*
93 * A new entry is added to the page table, increment the counters for that page
94 * verify that it is of correct type and is not being mapped with a different
95 * type to a different process.
96 */
97static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
98 bool rw)
99{
100 struct page_ext_iter iter;
101 struct page_ext *page_ext;
102 struct page *page;
103 bool anon;
104
105 if (!pfn_valid(pfn))
106 return;
107
108 page = pfn_to_page(pfn);
109 BUG_ON(PageSlab(page));
110 anon = PageAnon(page);
111
112 rcu_read_lock();
113 for_each_page_ext(page, pgcnt, page_ext, iter) {
114 struct page_table_check *ptc = get_page_table_check(page_ext);
115
116 if (anon) {
117 BUG_ON(atomic_read(&ptc->file_map_count));
118 BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
119 } else {
120 BUG_ON(atomic_read(&ptc->anon_map_count));
121 BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
122 }
123 }
124 rcu_read_unlock();
125}
126
127/*
128 * page is on free list, or is being allocated, verify that counters are zeroes
129 * crash if they are not.
130 */
131void __page_table_check_zero(struct page *page, unsigned int order)
132{
133 struct page_ext_iter iter;
134 struct page_ext *page_ext;
135
136 BUG_ON(PageSlab(page));
137
138 rcu_read_lock();
139 for_each_page_ext(page, 1 << order, page_ext, iter) {
140 struct page_table_check *ptc = get_page_table_check(page_ext);
141
142 BUG_ON(atomic_read(&ptc->anon_map_count));
143 BUG_ON(atomic_read(&ptc->file_map_count));
144 }
145 rcu_read_unlock();
146}
147
148void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
149{
150 if (&init_mm == mm)
151 return;
152
153 if (pte_user_accessible_page(pte)) {
154 page_table_check_clear(pfn: pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
155 }
156}
157EXPORT_SYMBOL(__page_table_check_pte_clear);
158
159void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
160{
161 if (&init_mm == mm)
162 return;
163
164 if (pmd_user_accessible_page(pmd)) {
165 page_table_check_clear(pfn: pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
166 }
167}
168EXPORT_SYMBOL(__page_table_check_pmd_clear);
169
170void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
171{
172 if (&init_mm == mm)
173 return;
174
175 if (pud_user_accessible_page(pud)) {
176 page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
177 }
178}
179EXPORT_SYMBOL(__page_table_check_pud_clear);
180
181/* Whether the swap entry cached writable information */
182static inline bool softleaf_cached_writable(softleaf_t entry)
183{
184 return softleaf_is_device_private_write(entry) ||
185 softleaf_is_migration_write(entry);
186}
187
188static void page_table_check_pte_flags(pte_t pte)
189{
190 if (pte_present(a: pte)) {
191 WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte));
192 } else if (pte_swp_uffd_wp(pte)) {
193 const softleaf_t entry = softleaf_from_pte(pte);
194
195 WARN_ON_ONCE(softleaf_cached_writable(entry));
196 }
197}
198
199void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
200 unsigned int nr)
201{
202 unsigned int i;
203
204 if (&init_mm == mm)
205 return;
206
207 page_table_check_pte_flags(pte);
208
209 for (i = 0; i < nr; i++)
210 __page_table_check_pte_clear(mm, ptep_get(ptep: ptep + i));
211 if (pte_user_accessible_page(pte))
212 page_table_check_set(pfn: pte_pfn(pte), pgcnt: nr, rw: pte_write(pte));
213}
214EXPORT_SYMBOL(__page_table_check_ptes_set);
215
216static inline void page_table_check_pmd_flags(pmd_t pmd)
217{
218 if (pmd_present(pmd)) {
219 if (pmd_uffd_wp(pmd))
220 WARN_ON_ONCE(pmd_write(pmd));
221 } else if (pmd_swp_uffd_wp(pmd)) {
222 const softleaf_t entry = softleaf_from_pmd(pmd);
223
224 WARN_ON_ONCE(softleaf_cached_writable(entry));
225 }
226}
227
228void __page_table_check_pmds_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd,
229 unsigned int nr)
230{
231 unsigned long stride = PMD_SIZE >> PAGE_SHIFT;
232 unsigned int i;
233
234 if (&init_mm == mm)
235 return;
236
237 page_table_check_pmd_flags(pmd);
238
239 for (i = 0; i < nr; i++)
240 __page_table_check_pmd_clear(mm, *(pmdp + i));
241 if (pmd_user_accessible_page(pmd))
242 page_table_check_set(pfn: pmd_pfn(pmd), pgcnt: stride * nr, pmd_write(pmd));
243}
244EXPORT_SYMBOL(__page_table_check_pmds_set);
245
246void __page_table_check_puds_set(struct mm_struct *mm, pud_t *pudp, pud_t pud,
247 unsigned int nr)
248{
249 unsigned long stride = PUD_SIZE >> PAGE_SHIFT;
250 unsigned int i;
251
252 if (&init_mm == mm)
253 return;
254
255 for (i = 0; i < nr; i++)
256 __page_table_check_pud_clear(mm, *(pudp + i));
257 if (pud_user_accessible_page(pud))
258 page_table_check_set(pud_pfn(pud), pgcnt: stride * nr, pud_write(pud));
259}
260EXPORT_SYMBOL(__page_table_check_puds_set);
261
262void __page_table_check_pte_clear_range(struct mm_struct *mm,
263 unsigned long addr,
264 pmd_t pmd)
265{
266 if (&init_mm == mm)
267 return;
268
269 if (!pmd_bad(pmd) && !pmd_leaf(pte: pmd)) {
270 pte_t *ptep = pte_offset_map(pmd: &pmd, addr);
271 unsigned long i;
272
273 if (WARN_ON(!ptep))
274 return;
275 for (i = 0; i < PTRS_PER_PTE; i++) {
276 __page_table_check_pte_clear(mm, ptep_get(ptep));
277 addr += PAGE_SIZE;
278 ptep++;
279 }
280 pte_unmap(pte: ptep - PTRS_PER_PTE);
281 }
282}
283

source code of linux/mm/page_table_check.c