1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * HugeTLB sysfs interfaces.
4 * (C) Nadia Yvette Chambers, April 2004
5 */
6
7#include <linux/swap.h>
8#include <linux/page_owner.h>
9#include <linux/page-isolation.h>
10
11#include "hugetlb_vmemmap.h"
12#include "hugetlb_internal.h"
13
14#define HSTATE_ATTR_RO(_name) \
15 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
16
17#define HSTATE_ATTR_WO(_name) \
18 static struct kobj_attribute _name##_attr = __ATTR_WO(_name)
19
20#define HSTATE_ATTR(_name) \
21 static struct kobj_attribute _name##_attr = __ATTR_RW(_name)
22
23static struct kobject *hugepages_kobj;
24static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
25
26static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
27
28static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
29{
30 int i;
31
32 for (i = 0; i < HUGE_MAX_HSTATE; i++)
33 if (hstate_kobjs[i] == kobj) {
34 if (nidp)
35 *nidp = NUMA_NO_NODE;
36 return &hstates[i];
37 }
38
39 return kobj_to_node_hstate(kobj, nidp);
40}
41
42static ssize_t nr_hugepages_show_common(struct kobject *kobj,
43 struct kobj_attribute *attr, char *buf)
44{
45 struct hstate *h;
46 unsigned long nr_huge_pages;
47 int nid;
48
49 h = kobj_to_hstate(kobj, nidp: &nid);
50 if (nid == NUMA_NO_NODE)
51 nr_huge_pages = h->nr_huge_pages;
52 else
53 nr_huge_pages = h->nr_huge_pages_node[nid];
54
55 return sysfs_emit(buf, fmt: "%lu\n", nr_huge_pages);
56}
57
58static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
59 struct kobject *kobj, const char *buf,
60 size_t len)
61{
62 struct hstate *h;
63 unsigned long count;
64 int nid;
65 int err;
66
67 err = kstrtoul(s: buf, base: 10, res: &count);
68 if (err)
69 return err;
70
71 h = kobj_to_hstate(kobj, nidp: &nid);
72 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
73}
74
75static ssize_t nr_hugepages_show(struct kobject *kobj,
76 struct kobj_attribute *attr, char *buf)
77{
78 return nr_hugepages_show_common(kobj, attr, buf);
79}
80
81static ssize_t nr_hugepages_store(struct kobject *kobj,
82 struct kobj_attribute *attr, const char *buf, size_t len)
83{
84 return nr_hugepages_store_common(obey_mempolicy: false, kobj, buf, len);
85}
86HSTATE_ATTR(nr_hugepages);
87
88#ifdef CONFIG_NUMA
89
90/*
91 * hstate attribute for optionally mempolicy-based constraint on persistent
92 * huge page alloc/free.
93 */
94static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
95 struct kobj_attribute *attr,
96 char *buf)
97{
98 return nr_hugepages_show_common(kobj, attr, buf);
99}
100
101static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
102 struct kobj_attribute *attr, const char *buf, size_t len)
103{
104 return nr_hugepages_store_common(obey_mempolicy: true, kobj, buf, len);
105}
106HSTATE_ATTR(nr_hugepages_mempolicy);
107#endif
108
109
110static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
111 struct kobj_attribute *attr, char *buf)
112{
113 struct hstate *h = kobj_to_hstate(kobj, NULL);
114 return sysfs_emit(buf, fmt: "%lu\n", h->nr_overcommit_huge_pages);
115}
116
117static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
118 struct kobj_attribute *attr, const char *buf, size_t count)
119{
120 int err;
121 unsigned long input;
122 struct hstate *h = kobj_to_hstate(kobj, NULL);
123
124 if (hstate_is_gigantic_no_runtime(h))
125 return -EINVAL;
126
127 err = kstrtoul(s: buf, base: 10, res: &input);
128 if (err)
129 return err;
130
131 spin_lock_irq(lock: &hugetlb_lock);
132 h->nr_overcommit_huge_pages = input;
133 spin_unlock_irq(lock: &hugetlb_lock);
134
135 return count;
136}
137HSTATE_ATTR(nr_overcommit_hugepages);
138
139static ssize_t free_hugepages_show(struct kobject *kobj,
140 struct kobj_attribute *attr, char *buf)
141{
142 struct hstate *h;
143 unsigned long free_huge_pages;
144 int nid;
145
146 h = kobj_to_hstate(kobj, nidp: &nid);
147 if (nid == NUMA_NO_NODE)
148 free_huge_pages = h->free_huge_pages;
149 else
150 free_huge_pages = h->free_huge_pages_node[nid];
151
152 return sysfs_emit(buf, fmt: "%lu\n", free_huge_pages);
153}
154HSTATE_ATTR_RO(free_hugepages);
155
156static ssize_t resv_hugepages_show(struct kobject *kobj,
157 struct kobj_attribute *attr, char *buf)
158{
159 struct hstate *h = kobj_to_hstate(kobj, NULL);
160 return sysfs_emit(buf, fmt: "%lu\n", h->resv_huge_pages);
161}
162HSTATE_ATTR_RO(resv_hugepages);
163
164static ssize_t surplus_hugepages_show(struct kobject *kobj,
165 struct kobj_attribute *attr, char *buf)
166{
167 struct hstate *h;
168 unsigned long surplus_huge_pages;
169 int nid;
170
171 h = kobj_to_hstate(kobj, nidp: &nid);
172 if (nid == NUMA_NO_NODE)
173 surplus_huge_pages = h->surplus_huge_pages;
174 else
175 surplus_huge_pages = h->surplus_huge_pages_node[nid];
176
177 return sysfs_emit(buf, fmt: "%lu\n", surplus_huge_pages);
178}
179HSTATE_ATTR_RO(surplus_hugepages);
180
181static ssize_t demote_store(struct kobject *kobj,
182 struct kobj_attribute *attr, const char *buf, size_t len)
183{
184 unsigned long nr_demote;
185 unsigned long nr_available;
186 nodemask_t nodes_allowed, *n_mask;
187 struct hstate *h;
188 int err;
189 int nid;
190
191 err = kstrtoul(s: buf, base: 10, res: &nr_demote);
192 if (err)
193 return err;
194 h = kobj_to_hstate(kobj, nidp: &nid);
195
196 if (nid != NUMA_NO_NODE) {
197 init_nodemask_of_node(mask: &nodes_allowed, node: nid);
198 n_mask = &nodes_allowed;
199 } else {
200 n_mask = &node_states[N_MEMORY];
201 }
202
203 /* Synchronize with other sysfs operations modifying huge pages */
204 mutex_lock(&h->resize_lock);
205 spin_lock_irq(lock: &hugetlb_lock);
206
207 while (nr_demote) {
208 long rc;
209
210 /*
211 * Check for available pages to demote each time thorough the
212 * loop as demote_pool_huge_page will drop hugetlb_lock.
213 */
214 if (nid != NUMA_NO_NODE)
215 nr_available = h->free_huge_pages_node[nid];
216 else
217 nr_available = h->free_huge_pages;
218 nr_available -= h->resv_huge_pages;
219 if (!nr_available)
220 break;
221
222 rc = demote_pool_huge_page(src: h, nodes_allowed: n_mask, nr_to_demote: nr_demote);
223 if (rc < 0) {
224 err = rc;
225 break;
226 }
227
228 nr_demote -= rc;
229 }
230
231 spin_unlock_irq(lock: &hugetlb_lock);
232 mutex_unlock(lock: &h->resize_lock);
233
234 if (err)
235 return err;
236 return len;
237}
238HSTATE_ATTR_WO(demote);
239
240static ssize_t demote_size_show(struct kobject *kobj,
241 struct kobj_attribute *attr, char *buf)
242{
243 struct hstate *h = kobj_to_hstate(kobj, NULL);
244 unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;
245
246 return sysfs_emit(buf, fmt: "%lukB\n", demote_size);
247}
248
249static ssize_t demote_size_store(struct kobject *kobj,
250 struct kobj_attribute *attr,
251 const char *buf, size_t count)
252{
253 struct hstate *h, *demote_hstate;
254 unsigned long demote_size;
255 unsigned int demote_order;
256
257 demote_size = (unsigned long)memparse(ptr: buf, NULL);
258
259 demote_hstate = size_to_hstate(size: demote_size);
260 if (!demote_hstate)
261 return -EINVAL;
262 demote_order = demote_hstate->order;
263 if (demote_order < HUGETLB_PAGE_ORDER)
264 return -EINVAL;
265
266 /* demote order must be smaller than hstate order */
267 h = kobj_to_hstate(kobj, NULL);
268 if (demote_order >= h->order)
269 return -EINVAL;
270
271 /* resize_lock synchronizes access to demote size and writes */
272 mutex_lock(&h->resize_lock);
273 h->demote_order = demote_order;
274 mutex_unlock(lock: &h->resize_lock);
275
276 return count;
277}
278HSTATE_ATTR(demote_size);
279
280static struct attribute *hstate_attrs[] = {
281 &nr_hugepages_attr.attr,
282 &nr_overcommit_hugepages_attr.attr,
283 &free_hugepages_attr.attr,
284 &resv_hugepages_attr.attr,
285 &surplus_hugepages_attr.attr,
286#ifdef CONFIG_NUMA
287 &nr_hugepages_mempolicy_attr.attr,
288#endif
289 NULL,
290};
291
292static const struct attribute_group hstate_attr_group = {
293 .attrs = hstate_attrs,
294};
295
296static struct attribute *hstate_demote_attrs[] = {
297 &demote_size_attr.attr,
298 &demote_attr.attr,
299 NULL,
300};
301
302static const struct attribute_group hstate_demote_attr_group = {
303 .attrs = hstate_demote_attrs,
304};
305
306static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
307 struct kobject **hstate_kobjs,
308 const struct attribute_group *hstate_attr_group)
309{
310 int retval;
311 int hi = hstate_index(h);
312
313 hstate_kobjs[hi] = kobject_create_and_add(name: h->name, parent);
314 if (!hstate_kobjs[hi])
315 return -ENOMEM;
316
317 retval = sysfs_create_group(kobj: hstate_kobjs[hi], grp: hstate_attr_group);
318 if (retval) {
319 kobject_put(kobj: hstate_kobjs[hi]);
320 hstate_kobjs[hi] = NULL;
321 return retval;
322 }
323
324 if (h->demote_order) {
325 retval = sysfs_create_group(kobj: hstate_kobjs[hi],
326 grp: &hstate_demote_attr_group);
327 if (retval) {
328 pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
329 sysfs_remove_group(kobj: hstate_kobjs[hi], grp: hstate_attr_group);
330 kobject_put(kobj: hstate_kobjs[hi]);
331 hstate_kobjs[hi] = NULL;
332 return retval;
333 }
334 }
335
336 return 0;
337}
338
339#ifdef CONFIG_NUMA
340static bool hugetlb_sysfs_initialized __ro_after_init;
341
342/*
343 * node_hstate/s - associate per node hstate attributes, via their kobjects,
344 * with node devices in node_devices[] using a parallel array. The array
345 * index of a node device or _hstate == node id.
346 * This is here to avoid any static dependency of the node device driver, in
347 * the base kernel, on the hugetlb module.
348 */
349struct node_hstate {
350 struct kobject *hugepages_kobj;
351 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
352};
353static struct node_hstate node_hstates[MAX_NUMNODES];
354
355/*
356 * A subset of global hstate attributes for node devices
357 */
358static struct attribute *per_node_hstate_attrs[] = {
359 &nr_hugepages_attr.attr,
360 &free_hugepages_attr.attr,
361 &surplus_hugepages_attr.attr,
362 NULL,
363};
364
365static const struct attribute_group per_node_hstate_attr_group = {
366 .attrs = per_node_hstate_attrs,
367};
368
369/*
370 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
371 * Returns node id via non-NULL nidp.
372 */
373static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
374{
375 int nid;
376
377 for (nid = 0; nid < nr_node_ids; nid++) {
378 struct node_hstate *nhs = &node_hstates[nid];
379 int i;
380 for (i = 0; i < HUGE_MAX_HSTATE; i++)
381 if (nhs->hstate_kobjs[i] == kobj) {
382 if (nidp)
383 *nidp = nid;
384 return &hstates[i];
385 }
386 }
387
388 BUG();
389 return NULL;
390}
391
392/*
393 * Unregister hstate attributes from a single node device.
394 * No-op if no hstate attributes attached.
395 */
396void hugetlb_unregister_node(struct node *node)
397{
398 struct hstate *h;
399 struct node_hstate *nhs = &node_hstates[node->dev.id];
400
401 if (!nhs->hugepages_kobj)
402 return; /* no hstate attributes */
403
404 for_each_hstate(h) {
405 int idx = hstate_index(h);
406 struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];
407
408 if (!hstate_kobj)
409 continue;
410 if (h->demote_order)
411 sysfs_remove_group(kobj: hstate_kobj, grp: &hstate_demote_attr_group);
412 sysfs_remove_group(kobj: hstate_kobj, grp: &per_node_hstate_attr_group);
413 kobject_put(kobj: hstate_kobj);
414 nhs->hstate_kobjs[idx] = NULL;
415 }
416
417 kobject_put(kobj: nhs->hugepages_kobj);
418 nhs->hugepages_kobj = NULL;
419}
420
421
422/*
423 * Register hstate attributes for a single node device.
424 * No-op if attributes already registered.
425 */
426void hugetlb_register_node(struct node *node)
427{
428 struct hstate *h;
429 struct node_hstate *nhs = &node_hstates[node->dev.id];
430 int err;
431
432 if (!hugetlb_sysfs_initialized)
433 return;
434
435 if (nhs->hugepages_kobj)
436 return; /* already allocated */
437
438 nhs->hugepages_kobj = kobject_create_and_add(name: "hugepages",
439 parent: &node->dev.kobj);
440 if (!nhs->hugepages_kobj)
441 return;
442
443 for_each_hstate(h) {
444 err = hugetlb_sysfs_add_hstate(h, parent: nhs->hugepages_kobj,
445 hstate_kobjs: nhs->hstate_kobjs,
446 hstate_attr_group: &per_node_hstate_attr_group);
447 if (err) {
448 pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
449 h->name, node->dev.id);
450 hugetlb_unregister_node(node);
451 break;
452 }
453 }
454}
455
456/*
457 * hugetlb init time: register hstate attributes for all registered node
458 * devices of nodes that have memory. All on-line nodes should have
459 * registered their associated device by this time.
460 */
461static void __init hugetlb_register_all_nodes(void)
462{
463 int nid;
464
465 for_each_online_node(nid)
466 hugetlb_register_node(node: node_devices[nid]);
467}
468#else /* !CONFIG_NUMA */
469
470static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
471{
472 BUG();
473 if (nidp)
474 *nidp = -1;
475 return NULL;
476}
477
478static void hugetlb_register_all_nodes(void) { }
479
480#endif
481
482void __init hugetlb_sysfs_init(void)
483{
484 struct hstate *h;
485 int err;
486
487 hugepages_kobj = kobject_create_and_add(name: "hugepages", parent: mm_kobj);
488 if (!hugepages_kobj)
489 return;
490
491 for_each_hstate(h) {
492 err = hugetlb_sysfs_add_hstate(h, parent: hugepages_kobj,
493 hstate_kobjs, hstate_attr_group: &hstate_attr_group);
494 if (err)
495 pr_err("HugeTLB: Unable to add hstate %s\n", h->name);
496 }
497
498#ifdef CONFIG_NUMA
499 hugetlb_sysfs_initialized = true;
500#endif
501 hugetlb_register_all_nodes();
502}
503

source code of linux/mm/hugetlb_sysfs.c