1/* SPDX-License-Identifier: GPL-2.0 */
2
3/*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */
7#ifndef __LINUX_PAGE_TABLE_CHECK_H
8#define __LINUX_PAGE_TABLE_CHECK_H
9
10#ifdef CONFIG_PAGE_TABLE_CHECK
11#include <linux/jump_label.h>
12
13extern struct static_key_true page_table_check_disabled;
14extern struct page_ext_operations page_table_check_ops;
15
16void __page_table_check_zero(struct page *page, unsigned int order);
17void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte);
18void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd);
19void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud);
20void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
21 unsigned int nr);
22void __page_table_check_pmds_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd,
23 unsigned int nr);
24void __page_table_check_puds_set(struct mm_struct *mm, pud_t *pudp, pud_t pud,
25 unsigned int nr);
26void __page_table_check_pte_clear_range(struct mm_struct *mm,
27 unsigned long addr,
28 pmd_t pmd);
29
30static inline void page_table_check_alloc(struct page *page, unsigned int order)
31{
32 if (static_branch_likely(&page_table_check_disabled))
33 return;
34
35 __page_table_check_zero(page, order);
36}
37
38static inline void page_table_check_free(struct page *page, unsigned int order)
39{
40 if (static_branch_likely(&page_table_check_disabled))
41 return;
42
43 __page_table_check_zero(page, order);
44}
45
46static inline void page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
47{
48 if (static_branch_likely(&page_table_check_disabled))
49 return;
50
51 __page_table_check_pte_clear(mm, pte);
52}
53
54static inline void page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
55{
56 if (static_branch_likely(&page_table_check_disabled))
57 return;
58
59 __page_table_check_pmd_clear(mm, pmd);
60}
61
62static inline void page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
63{
64 if (static_branch_likely(&page_table_check_disabled))
65 return;
66
67 __page_table_check_pud_clear(mm, pud);
68}
69
70static inline void page_table_check_ptes_set(struct mm_struct *mm,
71 pte_t *ptep, pte_t pte, unsigned int nr)
72{
73 if (static_branch_likely(&page_table_check_disabled))
74 return;
75
76 __page_table_check_ptes_set(mm, ptep, pte, nr);
77}
78
79static inline void page_table_check_pmds_set(struct mm_struct *mm,
80 pmd_t *pmdp, pmd_t pmd, unsigned int nr)
81{
82 if (static_branch_likely(&page_table_check_disabled))
83 return;
84
85 __page_table_check_pmds_set(mm, pmdp, pmd, nr);
86}
87
88static inline void page_table_check_puds_set(struct mm_struct *mm,
89 pud_t *pudp, pud_t pud, unsigned int nr)
90{
91 if (static_branch_likely(&page_table_check_disabled))
92 return;
93
94 __page_table_check_puds_set(mm, pudp, pud, nr);
95}
96
97static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
98 unsigned long addr,
99 pmd_t pmd)
100{
101 if (static_branch_likely(&page_table_check_disabled))
102 return;
103
104 __page_table_check_pte_clear_range(mm, addr, pmd);
105}
106
107#else
108
109static inline void page_table_check_alloc(struct page *page, unsigned int order)
110{
111}
112
113static inline void page_table_check_free(struct page *page, unsigned int order)
114{
115}
116
117static inline void page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
118{
119}
120
121static inline void page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
122{
123}
124
125static inline void page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
126{
127}
128
129static inline void page_table_check_ptes_set(struct mm_struct *mm,
130 pte_t *ptep, pte_t pte, unsigned int nr)
131{
132}
133
134static inline void page_table_check_pmds_set(struct mm_struct *mm,
135 pmd_t *pmdp, pmd_t pmd, unsigned int nr)
136{
137}
138
139static inline void page_table_check_puds_set(struct mm_struct *mm,
140 pud_t *pudp, pud_t pud, unsigned int nr)
141{
142}
143
144static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
145 unsigned long addr,
146 pmd_t pmd)
147{
148}
149
150#endif /* CONFIG_PAGE_TABLE_CHECK */
151
152#define page_table_check_pmd_set(mm, pmdp, pmd) page_table_check_pmds_set(mm, pmdp, pmd, 1)
153#define page_table_check_pud_set(mm, pudp, pud) page_table_check_puds_set(mm, pudp, pud, 1)
154
155#endif /* __LINUX_PAGE_TABLE_CHECK_H */
156

source code of linux/include/linux/page_table_check.h