1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
4 * <benh@kernel.crashing.org>
5 * Copyright (C) 2012 ARM Limited
6 * Copyright (C) 2015 Regents of the University of California
7 */
8
9#include <linux/elf.h>
10#include <linux/mm.h>
11#include <linux/slab.h>
12#include <linux/binfmts.h>
13#include <linux/err.h>
14#include <asm/page.h>
15#include <asm/vdso.h>
16#include <linux/vdso_datastore.h>
17#include <vdso/datapage.h>
18#include <vdso/vsyscall.h>
19
20#define VVAR_SIZE (VDSO_NR_PAGES << PAGE_SHIFT)
21
22struct __vdso_info {
23 const char *name;
24 const char *vdso_code_start;
25 const char *vdso_code_end;
26 unsigned long vdso_pages;
27 /* Code Mapping */
28 struct vm_special_mapping *cm;
29};
30
31static struct __vdso_info vdso_info;
32#ifdef CONFIG_COMPAT
33static struct __vdso_info compat_vdso_info;
34#endif
35
36static int vdso_mremap(const struct vm_special_mapping *sm,
37 struct vm_area_struct *new_vma)
38{
39 current->mm->context.vdso = (void *)new_vma->vm_start;
40
41 return 0;
42}
43
44static void __init __vdso_init(struct __vdso_info *vdso_info)
45{
46 unsigned int i;
47 struct page **vdso_pagelist;
48 unsigned long pfn;
49
50 if (memcmp(p: vdso_info->vdso_code_start, q: "\177ELF", size: 4))
51 panic(fmt: "vDSO is not a valid ELF object!\n");
52
53 vdso_info->vdso_pages = (
54 vdso_info->vdso_code_end -
55 vdso_info->vdso_code_start) >>
56 PAGE_SHIFT;
57
58 vdso_pagelist = kcalloc(vdso_info->vdso_pages,
59 sizeof(struct page *),
60 GFP_KERNEL);
61 if (vdso_pagelist == NULL)
62 panic(fmt: "vDSO kcalloc failed!\n");
63
64 /* Grab the vDSO code pages. */
65 pfn = sym_to_pfn(vdso_info->vdso_code_start);
66
67 for (i = 0; i < vdso_info->vdso_pages; i++)
68 vdso_pagelist[i] = pfn_to_page(pfn + i);
69
70 vdso_info->cm->pages = vdso_pagelist;
71}
72
73static struct vm_special_mapping rv_vdso_map __ro_after_init = {
74 .name = "[vdso]",
75 .mremap = vdso_mremap,
76};
77
78static struct __vdso_info vdso_info __ro_after_init = {
79 .name = "vdso",
80 .vdso_code_start = vdso_start,
81 .vdso_code_end = vdso_end,
82 .cm = &rv_vdso_map,
83};
84
85#ifdef CONFIG_COMPAT
86static struct vm_special_mapping rv_compat_vdso_map __ro_after_init = {
87 .name = "[vdso]",
88 .mremap = vdso_mremap,
89};
90
91static struct __vdso_info compat_vdso_info __ro_after_init = {
92 .name = "compat_vdso",
93 .vdso_code_start = compat_vdso_start,
94 .vdso_code_end = compat_vdso_end,
95 .cm = &rv_compat_vdso_map,
96};
97#endif
98
99static int __init vdso_init(void)
100{
101 __vdso_init(vdso_info: &vdso_info);
102#ifdef CONFIG_COMPAT
103 __vdso_init(vdso_info: &compat_vdso_info);
104#endif
105
106 return 0;
107}
108arch_initcall(vdso_init);
109
110static int __setup_additional_pages(struct mm_struct *mm,
111 struct linux_binprm *bprm,
112 int uses_interp,
113 struct __vdso_info *vdso_info)
114{
115 unsigned long vdso_base, vdso_text_len, vdso_mapping_len;
116 void *ret;
117
118 BUILD_BUG_ON(VDSO_NR_PAGES != __VDSO_PAGES);
119
120 vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT;
121 /* Be sure to map the data page */
122 vdso_mapping_len = vdso_text_len + VVAR_SIZE;
123
124 vdso_base = get_unmapped_area(NULL, addr: 0, len: vdso_mapping_len, pgoff: 0, flags: 0);
125 if (IS_ERR_VALUE(vdso_base)) {
126 ret = ERR_PTR(error: vdso_base);
127 goto up_fail;
128 }
129
130 ret = vdso_install_vvar_mapping(mm, addr: vdso_base);
131 if (IS_ERR(ptr: ret))
132 goto up_fail;
133
134 vdso_base += VVAR_SIZE;
135 mm->context.vdso = (void *)vdso_base;
136
137 ret =
138 _install_special_mapping(mm, addr: vdso_base, len: vdso_text_len,
139 vm_flags: (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC | VM_SEALED_SYSMAP),
140 spec: vdso_info->cm);
141
142 if (IS_ERR(ptr: ret))
143 goto up_fail;
144
145 return 0;
146
147up_fail:
148 mm->context.vdso = NULL;
149 return PTR_ERR(ptr: ret);
150}
151
152#ifdef CONFIG_COMPAT
153int compat_arch_setup_additional_pages(struct linux_binprm *bprm,
154 int uses_interp)
155{
156 struct mm_struct *mm = current->mm;
157 int ret;
158
159 if (mmap_write_lock_killable(mm))
160 return -EINTR;
161
162 ret = __setup_additional_pages(mm, bprm, uses_interp,
163 vdso_info: &compat_vdso_info);
164 mmap_write_unlock(mm);
165
166 return ret;
167}
168#endif
169
170int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
171{
172 struct mm_struct *mm = current->mm;
173 int ret;
174
175 if (mmap_write_lock_killable(mm))
176 return -EINTR;
177
178 ret = __setup_additional_pages(mm, bprm, uses_interp, vdso_info: &vdso_info);
179 mmap_write_unlock(mm);
180
181 return ret;
182}
183

source code of linux/arch/riscv/kernel/vdso.c