1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kexec_file for riscv, use vmlinux as the dump-capture kernel image.
4 *
5 * Copyright (C) 2021 Huawei Technologies Co, Ltd.
6 *
7 * Author: Liao Chang (liaochang1@huawei.com)
8 */
9#include <linux/kexec.h>
10#include <linux/elf.h>
11#include <linux/slab.h>
12#include <linux/of.h>
13#include <linux/libfdt.h>
14#include <linux/types.h>
15#include <linux/memblock.h>
16#include <linux/vmalloc.h>
17#include <asm/setup.h>
18#include <asm/insn.h>
19
20const struct kexec_file_ops * const kexec_file_loaders[] = {
21 &elf_kexec_ops,
22 &image_kexec_ops,
23 NULL
24};
25
26int arch_kimage_file_post_load_cleanup(struct kimage *image)
27{
28 kvfree(addr: image->arch.fdt);
29 image->arch.fdt = NULL;
30
31 vfree(addr: image->elf_headers);
32 image->elf_headers = NULL;
33 image->elf_headers_sz = 0;
34
35 return kexec_image_post_load_cleanup_default(image);
36}
37
38#ifdef CONFIG_CRASH_DUMP
39static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
40{
41 unsigned int *nr_ranges = arg;
42
43 (*nr_ranges)++;
44 return 0;
45}
46
47static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
48{
49 struct crash_mem *cmem = arg;
50
51 cmem->ranges[cmem->nr_ranges].start = res->start;
52 cmem->ranges[cmem->nr_ranges].end = res->end;
53 cmem->nr_ranges++;
54
55 return 0;
56}
57
58static int prepare_elf_headers(void **addr, unsigned long *sz)
59{
60 struct crash_mem *cmem;
61 unsigned int nr_ranges;
62 int ret;
63
64 nr_ranges = 1; /* For exclusion of crashkernel region */
65 walk_system_ram_res(start: 0, end: -1, arg: &nr_ranges, func: get_nr_ram_ranges_callback);
66
67 cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
68 if (!cmem)
69 return -ENOMEM;
70
71 cmem->max_nr_ranges = nr_ranges;
72 cmem->nr_ranges = 0;
73 ret = walk_system_ram_res(start: 0, end: -1, arg: cmem, func: prepare_elf64_ram_headers_callback);
74 if (ret)
75 goto out;
76
77 /* Exclude crashkernel region */
78 ret = crash_exclude_mem_range(mem: cmem, mstart: crashk_res.start, mend: crashk_res.end);
79 if (!ret)
80 ret = crash_prepare_elf64_headers(mem: cmem, need_kernel_map: true, addr, sz);
81
82out:
83 kfree(objp: cmem);
84 return ret;
85}
86
87static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
88 unsigned long cmdline_len)
89{
90 int elfcorehdr_strlen;
91 char *cmdline_ptr;
92
93 cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
94 if (!cmdline_ptr)
95 return NULL;
96
97 elfcorehdr_strlen = sprintf(buf: cmdline_ptr, fmt: "elfcorehdr=0x%lx ",
98 image->elf_load_addr);
99
100 if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
101 pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
102 kfree(objp: cmdline_ptr);
103 return NULL;
104 }
105
106 memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
107 /* Ensure it's nul terminated */
108 cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
109 return cmdline_ptr;
110}
111#endif
112
113#define RISCV_IMM_BITS 12
114#define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
115#define RISCV_CONST_HIGH_PART(x) \
116 (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
117#define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))
118
119#define ENCODE_ITYPE_IMM(x) \
120 (RV_X(x, 0, 12) << 20)
121#define ENCODE_BTYPE_IMM(x) \
122 ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \
123 (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31))
124#define ENCODE_UTYPE_IMM(x) \
125 (RV_X(x, 12, 20) << 12)
126#define ENCODE_JTYPE_IMM(x) \
127 ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \
128 (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31))
129#define ENCODE_CBTYPE_IMM(x) \
130 ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \
131 (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12))
132#define ENCODE_CJTYPE_IMM(x) \
133 ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \
134 (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \
135 (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12))
136#define ENCODE_UJTYPE_IMM(x) \
137 (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \
138 (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
139#define ENCODE_UITYPE_IMM(x) \
140 (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32))
141
142#define CLEAN_IMM(type, x) \
143 ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))
144
145int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
146 Elf_Shdr *section,
147 const Elf_Shdr *relsec,
148 const Elf_Shdr *symtab)
149{
150 const char *strtab, *name, *shstrtab;
151 const Elf_Shdr *sechdrs;
152 Elf64_Rela *relas;
153 int i, r_type;
154
155 /* String & section header string table */
156 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
157 strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
158 shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
159
160 relas = (void *)pi->ehdr + relsec->sh_offset;
161
162 for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
163 const Elf_Sym *sym; /* symbol to relocate */
164 unsigned long addr; /* final location after relocation */
165 unsigned long val; /* relocated symbol value */
166 unsigned long sec_base; /* relocated symbol value */
167 void *loc; /* tmp location to modify */
168
169 sym = (void *)pi->ehdr + symtab->sh_offset;
170 sym += ELF64_R_SYM(relas[i].r_info);
171
172 if (sym->st_name)
173 name = strtab + sym->st_name;
174 else
175 name = shstrtab + sechdrs[sym->st_shndx].sh_name;
176
177 loc = pi->purgatory_buf;
178 loc += section->sh_offset;
179 loc += relas[i].r_offset;
180
181 if (sym->st_shndx == SHN_ABS)
182 sec_base = 0;
183 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
184 pr_err("Invalid section %d for symbol %s\n",
185 sym->st_shndx, name);
186 return -ENOEXEC;
187 } else
188 sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
189
190 val = sym->st_value;
191 val += sec_base;
192 val += relas[i].r_addend;
193
194 addr = section->sh_addr + relas[i].r_offset;
195
196 r_type = ELF64_R_TYPE(relas[i].r_info);
197
198 switch (r_type) {
199 case R_RISCV_BRANCH:
200 *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) |
201 ENCODE_BTYPE_IMM(val - addr);
202 break;
203 case R_RISCV_JAL:
204 *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) |
205 ENCODE_JTYPE_IMM(val - addr);
206 break;
207 /*
208 * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
209 * sym is expected to be next to R_RISCV_PCREL_HI20
210 * in purgatory relsec. Handle it like R_RISCV_CALL
211 * sym, instead of searching the whole relsec.
212 */
213 case R_RISCV_PCREL_HI20:
214 case R_RISCV_CALL_PLT:
215 case R_RISCV_CALL:
216 *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) |
217 ENCODE_UJTYPE_IMM(val - addr);
218 break;
219 case R_RISCV_RVC_BRANCH:
220 *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) |
221 ENCODE_CBTYPE_IMM(val - addr);
222 break;
223 case R_RISCV_RVC_JUMP:
224 *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) |
225 ENCODE_CJTYPE_IMM(val - addr);
226 break;
227 case R_RISCV_ADD16:
228 *(u16 *)loc += val;
229 break;
230 case R_RISCV_SUB16:
231 *(u16 *)loc -= val;
232 break;
233 case R_RISCV_ADD32:
234 *(u32 *)loc += val;
235 break;
236 case R_RISCV_SUB32:
237 *(u32 *)loc -= val;
238 break;
239 /* It has been applied by R_RISCV_PCREL_HI20 sym */
240 case R_RISCV_PCREL_LO12_I:
241 case R_RISCV_ALIGN:
242 case R_RISCV_RELAX:
243 break;
244 case R_RISCV_64:
245 *(u64 *)loc = val;
246 break;
247 default:
248 pr_err("Unknown rela relocation: %d\n", r_type);
249 return -ENOEXEC;
250 }
251 }
252 return 0;
253}
254
255
256int load_extra_segments(struct kimage *image, unsigned long kernel_start,
257 unsigned long kernel_len, char *initrd,
258 unsigned long initrd_len, char *cmdline,
259 unsigned long cmdline_len)
260{
261 int ret;
262 void *fdt;
263 unsigned long initrd_pbase = 0UL;
264 struct kexec_buf kbuf = {};
265 char *modified_cmdline = NULL;
266
267 kbuf.image = image;
268 kbuf.buf_min = kernel_start + kernel_len;
269 kbuf.buf_max = ULONG_MAX;
270
271#ifdef CONFIG_CRASH_DUMP
272 /* Add elfcorehdr */
273 if (image->type == KEXEC_TYPE_CRASH) {
274 void *headers;
275 unsigned long headers_sz;
276 ret = prepare_elf_headers(addr: &headers, sz: &headers_sz);
277 if (ret) {
278 pr_err("Preparing elf core header failed\n");
279 goto out;
280 }
281
282 kbuf.buffer = headers;
283 kbuf.bufsz = headers_sz;
284 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
285 kbuf.memsz = headers_sz;
286 kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
287 kbuf.top_down = true;
288
289 ret = kexec_add_buffer(kbuf: &kbuf);
290 if (ret) {
291 vfree(addr: headers);
292 goto out;
293 }
294 image->elf_headers = headers;
295 image->elf_load_addr = kbuf.mem;
296 image->elf_headers_sz = headers_sz;
297
298 kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
299 image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
300
301 /* Setup cmdline for kdump kernel case */
302 modified_cmdline = setup_kdump_cmdline(image, cmdline,
303 cmdline_len);
304 if (!modified_cmdline) {
305 pr_err("Setting up cmdline for kdump kernel failed\n");
306 ret = -EINVAL;
307 goto out;
308 }
309 cmdline = modified_cmdline;
310 }
311#endif
312
313#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY
314 /* Add purgatory to the image */
315 kbuf.top_down = true;
316 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
317 ret = kexec_load_purgatory(image, kbuf: &kbuf);
318 if (ret) {
319 pr_err("Error loading purgatory ret=%d\n", ret);
320 goto out;
321 }
322 kexec_dprintk("Loaded purgatory at 0x%lx\n", kbuf.mem);
323
324 ret = kexec_purgatory_get_set_symbol(image, name: "riscv_kernel_entry",
325 buf: &kernel_start,
326 size: sizeof(kernel_start), get_value: 0);
327 if (ret)
328 pr_err("Error update purgatory ret=%d\n", ret);
329#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */
330
331 /* Add the initrd to the image */
332 if (initrd != NULL) {
333 kbuf.buffer = initrd;
334 kbuf.bufsz = kbuf.memsz = initrd_len;
335 kbuf.buf_align = PAGE_SIZE;
336 kbuf.top_down = true;
337 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
338 ret = kexec_add_buffer(kbuf: &kbuf);
339 if (ret)
340 goto out;
341 initrd_pbase = kbuf.mem;
342 kexec_dprintk("Loaded initrd at 0x%lx\n", initrd_pbase);
343 }
344
345 /* Add the DTB to the image */
346 fdt = of_kexec_alloc_and_setup_fdt(image, initrd_load_addr: initrd_pbase,
347 initrd_len, cmdline, extra_fdt_size: 0);
348 if (!fdt) {
349 pr_err("Error setting up the new device tree.\n");
350 ret = -EINVAL;
351 goto out;
352 }
353
354 fdt_pack(fdt);
355 kbuf.buffer = fdt;
356 kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
357 kbuf.buf_align = PAGE_SIZE;
358 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
359 kbuf.top_down = true;
360 ret = kexec_add_buffer(kbuf: &kbuf);
361 if (ret) {
362 pr_err("Error add DTB kbuf ret=%d\n", ret);
363 goto out_free_fdt;
364 }
365 /* Cache the fdt buffer address for memory cleanup */
366 image->arch.fdt = fdt;
367 kexec_dprintk("Loaded device tree at 0x%lx\n", kbuf.mem);
368 goto out;
369
370out_free_fdt:
371 kvfree(addr: fdt);
372out:
373 kfree(objp: modified_cmdline);
374 return ret;
375}
376

source code of linux/arch/riscv/kernel/machine_kexec_file.c