1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kexec: kexec_file_load system call
4 *
5 * Copyright (C) 2014 Red Hat Inc.
6 * Authors:
7 * Vivek Goyal <vgoyal@redhat.com>
8 */
9
10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12#include <linux/capability.h>
13#include <linux/mm.h>
14#include <linux/file.h>
15#include <linux/slab.h>
16#include <linux/kexec.h>
17#include <linux/memblock.h>
18#include <linux/mutex.h>
19#include <linux/list.h>
20#include <linux/fs.h>
21#include <linux/ima.h>
22#include <crypto/sha2.h>
23#include <linux/elf.h>
24#include <linux/elfcore.h>
25#include <linux/kernel.h>
26#include <linux/kernel_read_file.h>
27#include <linux/syscalls.h>
28#include <linux/vmalloc.h>
29#include <linux/dma-map-ops.h>
30#include "kexec_internal.h"
31
32#ifdef CONFIG_KEXEC_SIG
33static bool sig_enforce = IS_ENABLED(CONFIG_KEXEC_SIG_FORCE);
34
35void set_kexec_sig_enforced(void)
36{
37 sig_enforce = true;
38}
39#endif
40
41#ifdef CONFIG_IMA_KEXEC
42static bool check_ima_segment_index(struct kimage *image, int i)
43{
44 if (image->is_ima_segment_index_set && i == image->ima_segment_index)
45 return true;
46 else
47 return false;
48}
49#else
50static bool check_ima_segment_index(struct kimage *image, int i)
51{
52 return false;
53}
54#endif
55
56static int kexec_calculate_store_digests(struct kimage *image);
57
58/* Maximum size in bytes for kernel/initrd files. */
59#define KEXEC_FILE_SIZE_MAX min_t(s64, 4LL << 30, SSIZE_MAX)
60
61/*
62 * Currently this is the only default function that is exported as some
63 * architectures need it to do additional handlings.
64 * In the future, other default functions may be exported too if required.
65 */
66int kexec_image_probe_default(struct kimage *image, void *buf,
67 unsigned long buf_len)
68{
69 const struct kexec_file_ops * const *fops;
70 int ret = -ENOEXEC;
71
72 for (fops = &kexec_file_loaders[0]; *fops && (*fops)->probe; ++fops) {
73 ret = (*fops)->probe(buf, buf_len);
74 if (!ret) {
75 image->fops = *fops;
76 return ret;
77 }
78 }
79
80 return ret;
81}
82
83static void *kexec_image_load_default(struct kimage *image)
84{
85 if (!image->fops || !image->fops->load)
86 return ERR_PTR(error: -ENOEXEC);
87
88 return image->fops->load(image, image->kernel_buf,
89 image->kernel_buf_len, image->initrd_buf,
90 image->initrd_buf_len, image->cmdline_buf,
91 image->cmdline_buf_len);
92}
93
94int kexec_image_post_load_cleanup_default(struct kimage *image)
95{
96 if (!image->fops || !image->fops->cleanup)
97 return 0;
98
99 return image->fops->cleanup(image->image_loader_data);
100}
101
102/*
103 * Free up memory used by kernel, initrd, and command line. This is temporary
104 * memory allocation which is not needed any more after these buffers have
105 * been loaded into separate segments and have been copied elsewhere.
106 */
107void kimage_file_post_load_cleanup(struct kimage *image)
108{
109 struct purgatory_info *pi = &image->purgatory_info;
110
111 vfree(addr: image->kernel_buf);
112 image->kernel_buf = NULL;
113
114 vfree(addr: image->initrd_buf);
115 image->initrd_buf = NULL;
116
117 kfree(objp: image->cmdline_buf);
118 image->cmdline_buf = NULL;
119
120 vfree(addr: pi->purgatory_buf);
121 pi->purgatory_buf = NULL;
122
123 vfree(addr: pi->sechdrs);
124 pi->sechdrs = NULL;
125
126#ifdef CONFIG_IMA_KEXEC
127 vfree(addr: image->ima_buffer);
128 image->ima_buffer = NULL;
129#endif /* CONFIG_IMA_KEXEC */
130
131 /* See if architecture has anything to cleanup post load */
132 arch_kimage_file_post_load_cleanup(image);
133
134 /*
135 * Above call should have called into bootloader to free up
136 * any data stored in kimage->image_loader_data. It should
137 * be ok now to free it up.
138 */
139 kfree(objp: image->image_loader_data);
140 image->image_loader_data = NULL;
141
142 kexec_file_dbg_print = false;
143}
144
145#ifdef CONFIG_KEXEC_SIG
146#ifdef CONFIG_SIGNED_PE_FILE_VERIFICATION
147int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len)
148{
149 int ret;
150
151 ret = verify_pefile_signature(pebuf: kernel, pelen: kernel_len,
152 VERIFY_USE_SECONDARY_KEYRING,
153 usage: VERIFYING_KEXEC_PE_SIGNATURE);
154 if (ret == -ENOKEY && IS_ENABLED(CONFIG_INTEGRITY_PLATFORM_KEYRING)) {
155 ret = verify_pefile_signature(pebuf: kernel, pelen: kernel_len,
156 VERIFY_USE_PLATFORM_KEYRING,
157 usage: VERIFYING_KEXEC_PE_SIGNATURE);
158 }
159 return ret;
160}
161#endif
162
163static int kexec_image_verify_sig(struct kimage *image, void *buf,
164 unsigned long buf_len)
165{
166 if (!image->fops || !image->fops->verify_sig) {
167 pr_debug("kernel loader does not support signature verification.\n");
168 return -EKEYREJECTED;
169 }
170
171 return image->fops->verify_sig(buf, buf_len);
172}
173
174static int
175kimage_validate_signature(struct kimage *image)
176{
177 int ret;
178
179 ret = kexec_image_verify_sig(image, buf: image->kernel_buf,
180 buf_len: image->kernel_buf_len);
181 if (ret) {
182
183 if (sig_enforce) {
184 pr_notice("Enforced kernel signature verification failed (%d).\n", ret);
185 return ret;
186 }
187
188 /*
189 * If IMA is guaranteed to appraise a signature on the kexec
190 * image, permit it even if the kernel is otherwise locked
191 * down.
192 */
193 if (!ima_appraise_signature(func: READING_KEXEC_IMAGE) &&
194 security_locked_down(what: LOCKDOWN_KEXEC))
195 return -EPERM;
196
197 pr_debug("kernel signature verification failed (%d).\n", ret);
198 }
199
200 return 0;
201}
202#endif
203
204static int kexec_post_load(struct kimage *image, unsigned long flags)
205{
206#ifdef CONFIG_IMA_KEXEC
207 if (!(flags & KEXEC_FILE_ON_CRASH))
208 ima_kexec_post_load(image);
209#endif
210 return machine_kexec_post_load(image);
211}
212
213/*
214 * In file mode list of segments is prepared by kernel. Copy relevant
215 * data from user space, do error checking, prepare segment list
216 */
217static int
218kimage_file_prepare_segments(struct kimage *image, int kernel_fd, int initrd_fd,
219 const char __user *cmdline_ptr,
220 unsigned long cmdline_len, unsigned flags)
221{
222 ssize_t ret;
223 void *ldata;
224
225 ret = kernel_read_file_from_fd(fd: kernel_fd, offset: 0, buf: &image->kernel_buf,
226 KEXEC_FILE_SIZE_MAX, NULL,
227 id: READING_KEXEC_IMAGE);
228 if (ret < 0)
229 return ret;
230 image->kernel_buf_len = ret;
231 kexec_dprintk("kernel: %p kernel_size: %#lx\n",
232 image->kernel_buf, image->kernel_buf_len);
233
234 /* Call arch image probe handlers */
235 ret = arch_kexec_kernel_image_probe(image, buf: image->kernel_buf,
236 buf_len: image->kernel_buf_len);
237 if (ret)
238 goto out;
239
240#ifdef CONFIG_KEXEC_SIG
241 ret = kimage_validate_signature(image);
242
243 if (ret)
244 goto out;
245#endif
246 /* It is possible that there no initramfs is being loaded */
247 if (!(flags & KEXEC_FILE_NO_INITRAMFS)) {
248 ret = kernel_read_file_from_fd(fd: initrd_fd, offset: 0, buf: &image->initrd_buf,
249 KEXEC_FILE_SIZE_MAX, NULL,
250 id: READING_KEXEC_INITRAMFS);
251 if (ret < 0)
252 goto out;
253 image->initrd_buf_len = ret;
254 ret = 0;
255 }
256
257 image->no_cma = !!(flags & KEXEC_FILE_NO_CMA);
258 image->force_dtb = flags & KEXEC_FILE_FORCE_DTB;
259
260 if (cmdline_len) {
261 image->cmdline_buf = memdup_user(cmdline_ptr, cmdline_len);
262 if (IS_ERR(ptr: image->cmdline_buf)) {
263 ret = PTR_ERR(ptr: image->cmdline_buf);
264 image->cmdline_buf = NULL;
265 goto out;
266 }
267
268 image->cmdline_buf_len = cmdline_len;
269
270 /* command line should be a string with last byte null */
271 if (image->cmdline_buf[cmdline_len - 1] != '\0') {
272 ret = -EINVAL;
273 goto out;
274 }
275
276 ima_kexec_cmdline(kernel_fd, buf: image->cmdline_buf,
277 size: image->cmdline_buf_len - 1);
278 }
279
280 /* IMA needs to pass the measurement list to the next kernel. */
281 ima_add_kexec_buffer(image);
282
283 /* If KHO is active, add its images to the list */
284 ret = kho_fill_kimage(image);
285 if (ret)
286 goto out;
287
288 /* Call image load handler */
289 ldata = kexec_image_load_default(image);
290
291 if (IS_ERR(ptr: ldata)) {
292 ret = PTR_ERR(ptr: ldata);
293 goto out;
294 }
295
296 image->image_loader_data = ldata;
297out:
298 /* In case of error, free up all allocated memory in this function */
299 if (ret)
300 kimage_file_post_load_cleanup(image);
301 return ret;
302}
303
304static int
305kimage_file_alloc_init(struct kimage **rimage, int kernel_fd,
306 int initrd_fd, const char __user *cmdline_ptr,
307 unsigned long cmdline_len, unsigned long flags)
308{
309 int ret;
310 struct kimage *image;
311 bool kexec_on_panic = flags & KEXEC_FILE_ON_CRASH;
312
313 image = do_kimage_alloc_init();
314 if (!image)
315 return -ENOMEM;
316
317 kexec_file_dbg_print = !!(flags & KEXEC_FILE_DEBUG);
318 image->file_mode = 1;
319
320#ifdef CONFIG_CRASH_DUMP
321 if (kexec_on_panic) {
322 /* Enable special crash kernel control page alloc policy. */
323 image->control_page = crashk_res.start;
324 image->type = KEXEC_TYPE_CRASH;
325 }
326#endif
327
328 ret = kimage_file_prepare_segments(image, kernel_fd, initrd_fd,
329 cmdline_ptr, cmdline_len, flags);
330 if (ret)
331 goto out_free_image;
332
333 ret = sanity_check_segment_list(image);
334 if (ret)
335 goto out_free_post_load_bufs;
336
337 ret = -ENOMEM;
338 image->control_code_page = kimage_alloc_control_pages(image,
339 order: get_order(KEXEC_CONTROL_PAGE_SIZE));
340 if (!image->control_code_page) {
341 pr_err("Could not allocate control_code_buffer\n");
342 goto out_free_post_load_bufs;
343 }
344
345 if (!kexec_on_panic) {
346 image->swap_page = kimage_alloc_control_pages(image, order: 0);
347 if (!image->swap_page) {
348 pr_err("Could not allocate swap buffer\n");
349 goto out_free_control_pages;
350 }
351 }
352
353 *rimage = image;
354 return 0;
355out_free_control_pages:
356 kimage_free_page_list(list: &image->control_pages);
357out_free_post_load_bufs:
358 kimage_file_post_load_cleanup(image);
359out_free_image:
360 kfree(objp: image);
361 return ret;
362}
363
364SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd,
365 unsigned long, cmdline_len, const char __user *, cmdline_ptr,
366 unsigned long, flags)
367{
368 int image_type = (flags & KEXEC_FILE_ON_CRASH) ?
369 KEXEC_TYPE_CRASH : KEXEC_TYPE_DEFAULT;
370 struct kimage **dest_image, *image;
371 int ret = 0, i;
372
373 /* We only trust the superuser with rebooting the system. */
374 if (!kexec_load_permitted(kexec_image_type: image_type))
375 return -EPERM;
376
377 /* Make sure we have a legal set of flags */
378 if (flags != (flags & KEXEC_FILE_FLAGS))
379 return -EINVAL;
380
381 image = NULL;
382
383 if (!kexec_trylock())
384 return -EBUSY;
385
386#ifdef CONFIG_CRASH_DUMP
387 if (image_type == KEXEC_TYPE_CRASH) {
388 dest_image = &kexec_crash_image;
389 if (kexec_crash_image)
390 arch_kexec_unprotect_crashkres();
391 } else
392#endif
393 dest_image = &kexec_image;
394
395 if (flags & KEXEC_FILE_UNLOAD)
396 goto exchange;
397
398 /*
399 * In case of crash, new kernel gets loaded in reserved region. It is
400 * same memory where old crash kernel might be loaded. Free any
401 * current crash dump kernel before we corrupt it.
402 */
403 if (flags & KEXEC_FILE_ON_CRASH)
404 kimage_free(xchg(&kexec_crash_image, NULL));
405
406 ret = kimage_file_alloc_init(rimage: &image, kernel_fd, initrd_fd, cmdline_ptr,
407 cmdline_len, flags);
408 if (ret)
409 goto out;
410
411#ifdef CONFIG_CRASH_HOTPLUG
412 if ((flags & KEXEC_FILE_ON_CRASH) && arch_crash_hotplug_support(image, kexec_flags: flags))
413 image->hotplug_support = 1;
414#endif
415
416 ret = machine_kexec_prepare(image);
417 if (ret)
418 goto out;
419
420 /*
421 * Some architecture(like S390) may touch the crash memory before
422 * machine_kexec_prepare(), we must copy vmcoreinfo data after it.
423 */
424 ret = kimage_crash_copy_vmcoreinfo(image);
425 if (ret)
426 goto out;
427
428 ret = kexec_calculate_store_digests(image);
429 if (ret)
430 goto out;
431
432 kexec_dprintk("nr_segments = %lu\n", image->nr_segments);
433 for (i = 0; i < image->nr_segments; i++) {
434 struct kexec_segment *ksegment;
435
436 ksegment = &image->segment[i];
437 kexec_dprintk("segment[%d]: buf=0x%p bufsz=0x%zx mem=0x%lx memsz=0x%zx\n",
438 i, ksegment->buf, ksegment->bufsz, ksegment->mem,
439 ksegment->memsz);
440
441 ret = kimage_load_segment(image, idx: i);
442 if (ret)
443 goto out;
444 }
445
446 kimage_terminate(image);
447
448 ret = kexec_post_load(image, flags);
449 if (ret)
450 goto out;
451
452 kexec_dprintk("kexec_file_load: type:%u, start:0x%lx head:0x%lx flags:0x%lx\n",
453 image->type, image->start, image->head, flags);
454 /*
455 * Free up any temporary buffers allocated which are not needed
456 * after image has been loaded
457 */
458 kimage_file_post_load_cleanup(image);
459exchange:
460 image = xchg(dest_image, image);
461out:
462#ifdef CONFIG_CRASH_DUMP
463 if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image)
464 arch_kexec_protect_crashkres();
465#endif
466
467 kexec_unlock();
468 kimage_free(image);
469 return ret;
470}
471
472static int locate_mem_hole_top_down(unsigned long start, unsigned long end,
473 struct kexec_buf *kbuf)
474{
475 struct kimage *image = kbuf->image;
476 unsigned long temp_start, temp_end;
477
478 temp_end = min(end, kbuf->buf_max);
479 temp_start = temp_end - kbuf->memsz + 1;
480 kexec_random_range_start(start: temp_start, end: temp_end, kbuf, temp_start: &temp_start);
481
482 do {
483 /* align down start */
484 temp_start = ALIGN_DOWN(temp_start, kbuf->buf_align);
485
486 if (temp_start < start || temp_start < kbuf->buf_min)
487 return 0;
488
489 temp_end = temp_start + kbuf->memsz - 1;
490
491 /*
492 * Make sure this does not conflict with any of existing
493 * segments
494 */
495 if (kimage_is_destination_range(image, start: temp_start, end: temp_end)) {
496 temp_start = temp_start - PAGE_SIZE;
497 continue;
498 }
499
500 /* Make sure this does not conflict with exclude range */
501 if (arch_check_excluded_range(image, start: temp_start, end: temp_end)) {
502 temp_start = temp_start - PAGE_SIZE;
503 continue;
504 }
505
506 /* We found a suitable memory range */
507 break;
508 } while (1);
509
510 /* If we are here, we found a suitable memory range */
511 kbuf->mem = temp_start;
512
513 /* Success, stop navigating through remaining System RAM ranges */
514 return 1;
515}
516
517static int locate_mem_hole_bottom_up(unsigned long start, unsigned long end,
518 struct kexec_buf *kbuf)
519{
520 struct kimage *image = kbuf->image;
521 unsigned long temp_start, temp_end;
522
523 temp_start = max(start, kbuf->buf_min);
524
525 kexec_random_range_start(start: temp_start, end, kbuf, temp_start: &temp_start);
526
527 do {
528 temp_start = ALIGN(temp_start, kbuf->buf_align);
529 temp_end = temp_start + kbuf->memsz - 1;
530
531 if (temp_end > end || temp_end > kbuf->buf_max)
532 return 0;
533 /*
534 * Make sure this does not conflict with any of existing
535 * segments
536 */
537 if (kimage_is_destination_range(image, start: temp_start, end: temp_end)) {
538 temp_start = temp_start + PAGE_SIZE;
539 continue;
540 }
541
542 /* Make sure this does not conflict with exclude range */
543 if (arch_check_excluded_range(image, start: temp_start, end: temp_end)) {
544 temp_start = temp_start + PAGE_SIZE;
545 continue;
546 }
547
548 /* We found a suitable memory range */
549 break;
550 } while (1);
551
552 /* If we are here, we found a suitable memory range */
553 kbuf->mem = temp_start;
554
555 /* Success, stop navigating through remaining System RAM ranges */
556 return 1;
557}
558
559static int locate_mem_hole_callback(struct resource *res, void *arg)
560{
561 struct kexec_buf *kbuf = (struct kexec_buf *)arg;
562 u64 start = res->start, end = res->end;
563 unsigned long sz = end - start + 1;
564
565 /* Returning 0 will take to next memory range */
566
567 /* Don't use memory that will be detected and handled by a driver. */
568 if (res->flags & IORESOURCE_SYSRAM_DRIVER_MANAGED)
569 return 0;
570
571 if (sz < kbuf->memsz)
572 return 0;
573
574 if (end < kbuf->buf_min || start > kbuf->buf_max)
575 return 0;
576
577 /*
578 * Allocate memory top down with-in ram range. Otherwise bottom up
579 * allocation.
580 */
581 if (kbuf->top_down)
582 return locate_mem_hole_top_down(start, end, kbuf);
583 return locate_mem_hole_bottom_up(start, end, kbuf);
584}
585
586#ifdef CONFIG_ARCH_KEEP_MEMBLOCK
587static int kexec_walk_memblock(struct kexec_buf *kbuf,
588 int (*func)(struct resource *, void *))
589{
590 int ret = 0;
591 u64 i;
592 phys_addr_t mstart, mend;
593 struct resource res = { };
594
595#ifdef CONFIG_CRASH_DUMP
596 if (kbuf->image->type == KEXEC_TYPE_CRASH)
597 return func(&crashk_res, kbuf);
598#endif
599
600 /*
601 * Using MEMBLOCK_NONE will properly skip MEMBLOCK_DRIVER_MANAGED. See
602 * IORESOURCE_SYSRAM_DRIVER_MANAGED handling in
603 * locate_mem_hole_callback().
604 */
605 if (kbuf->top_down) {
606 for_each_free_mem_range_reverse(i, NUMA_NO_NODE, MEMBLOCK_NONE,
607 &mstart, &mend, NULL) {
608 /*
609 * In memblock, end points to the first byte after the
610 * range while in kexec, end points to the last byte
611 * in the range.
612 */
613 res.start = mstart;
614 res.end = mend - 1;
615 ret = func(&res, kbuf);
616 if (ret)
617 break;
618 }
619 } else {
620 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE,
621 &mstart, &mend, NULL) {
622 /*
623 * In memblock, end points to the first byte after the
624 * range while in kexec, end points to the last byte
625 * in the range.
626 */
627 res.start = mstart;
628 res.end = mend - 1;
629 ret = func(&res, kbuf);
630 if (ret)
631 break;
632 }
633 }
634
635 return ret;
636}
637#else
638static int kexec_walk_memblock(struct kexec_buf *kbuf,
639 int (*func)(struct resource *, void *))
640{
641 return 0;
642}
643#endif
644
645/**
646 * kexec_walk_resources - call func(data) on free memory regions
647 * @kbuf: Context info for the search. Also passed to @func.
648 * @func: Function to call for each memory region.
649 *
650 * Return: The memory walk will stop when func returns a non-zero value
651 * and that value will be returned. If all free regions are visited without
652 * func returning non-zero, then zero will be returned.
653 */
654static int kexec_walk_resources(struct kexec_buf *kbuf,
655 int (*func)(struct resource *, void *))
656{
657#ifdef CONFIG_CRASH_DUMP
658 if (kbuf->image->type == KEXEC_TYPE_CRASH)
659 return walk_iomem_res_desc(desc: crashk_res.desc,
660 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
661 start: crashk_res.start, end: crashk_res.end,
662 arg: kbuf, func);
663#endif
664 if (kbuf->top_down)
665 return walk_system_ram_res_rev(start: 0, ULONG_MAX, arg: kbuf, func);
666 else
667 return walk_system_ram_res(start: 0, ULONG_MAX, arg: kbuf, func);
668}
669
670static int kexec_alloc_contig(struct kexec_buf *kbuf)
671{
672 size_t nr_pages = kbuf->memsz >> PAGE_SHIFT;
673 unsigned long mem;
674 struct page *p;
675
676 /* User space disabled CMA allocations, bail out. */
677 if (kbuf->image->no_cma)
678 return -EPERM;
679
680 /* Skip CMA logic for crash kernel */
681 if (kbuf->image->type == KEXEC_TYPE_CRASH)
682 return -EPERM;
683
684 p = dma_alloc_from_contiguous(NULL, count: nr_pages, order: get_order(size: kbuf->buf_align), no_warn: true);
685 if (!p)
686 return -ENOMEM;
687
688 pr_debug("allocated %zu DMA pages at 0x%lx", nr_pages, page_to_boot_pfn(p));
689
690 mem = page_to_boot_pfn(page: p) << PAGE_SHIFT;
691
692 if (kimage_is_destination_range(image: kbuf->image, start: mem, end: mem + kbuf->memsz)) {
693 /* Our region is already in use by a statically defined one. Bail out. */
694 pr_debug("CMA overlaps existing mem: 0x%lx+0x%lx\n", mem, kbuf->memsz);
695 dma_release_from_contiguous(NULL, pages: p, count: nr_pages);
696 return -EBUSY;
697 }
698
699 kbuf->mem = page_to_boot_pfn(page: p) << PAGE_SHIFT;
700 kbuf->cma = p;
701
702 arch_kexec_post_alloc_pages(page_address(p), pages: (int)nr_pages, gfp: 0);
703
704 return 0;
705}
706
707/**
708 * kexec_locate_mem_hole - find free memory for the purgatory or the next kernel
709 * @kbuf: Parameters for the memory search.
710 *
711 * On success, kbuf->mem will have the start address of the memory region found.
712 *
713 * Return: 0 on success, negative errno on error.
714 */
715int kexec_locate_mem_hole(struct kexec_buf *kbuf)
716{
717 int ret;
718
719 /* Arch knows where to place */
720 if (kbuf->mem != KEXEC_BUF_MEM_UNKNOWN)
721 return 0;
722
723 /*
724 * If KHO is active, only use KHO scratch memory. All other memory
725 * could potentially be handed over.
726 */
727 ret = kho_locate_mem_hole(kbuf, func: locate_mem_hole_callback);
728 if (ret <= 0)
729 return ret;
730
731 /*
732 * Try to find a free physically contiguous block of memory first. With that, we
733 * can avoid any copying at kexec time.
734 */
735 if (!kexec_alloc_contig(kbuf))
736 return 0;
737
738 if (!IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
739 ret = kexec_walk_resources(kbuf, func: locate_mem_hole_callback);
740 else
741 ret = kexec_walk_memblock(kbuf, func: locate_mem_hole_callback);
742
743 return ret == 1 ? 0 : -EADDRNOTAVAIL;
744}
745
746/**
747 * kexec_add_buffer - place a buffer in a kexec segment
748 * @kbuf: Buffer contents and memory parameters.
749 *
750 * This function assumes that kexec_lock is held.
751 * On successful return, @kbuf->mem will have the physical address of
752 * the buffer in memory.
753 *
754 * Return: 0 on success, negative errno on error.
755 */
756int kexec_add_buffer(struct kexec_buf *kbuf)
757{
758 struct kexec_segment *ksegment;
759 int ret;
760
761 /* Currently adding segment this way is allowed only in file mode */
762 if (!kbuf->image->file_mode)
763 return -EINVAL;
764
765 if (kbuf->image->nr_segments >= KEXEC_SEGMENT_MAX)
766 return -EINVAL;
767
768 /*
769 * Make sure we are not trying to add buffer after allocating
770 * control pages. All segments need to be placed first before
771 * any control pages are allocated. As control page allocation
772 * logic goes through list of segments to make sure there are
773 * no destination overlaps.
774 */
775 if (!list_empty(head: &kbuf->image->control_pages)) {
776 WARN_ON(1);
777 return -EINVAL;
778 }
779
780 /* Ensure minimum alignment needed for segments. */
781 kbuf->memsz = ALIGN(kbuf->memsz, PAGE_SIZE);
782 kbuf->buf_align = max(kbuf->buf_align, PAGE_SIZE);
783 kbuf->cma = NULL;
784
785 /* Walk the RAM ranges and allocate a suitable range for the buffer */
786 ret = arch_kexec_locate_mem_hole(kbuf);
787 if (ret)
788 return ret;
789
790 /* Found a suitable memory range */
791 ksegment = &kbuf->image->segment[kbuf->image->nr_segments];
792 ksegment->kbuf = kbuf->buffer;
793 ksegment->bufsz = kbuf->bufsz;
794 ksegment->mem = kbuf->mem;
795 ksegment->memsz = kbuf->memsz;
796 kbuf->image->segment_cma[kbuf->image->nr_segments] = kbuf->cma;
797 kbuf->image->nr_segments++;
798 return 0;
799}
800
801/* Calculate and store the digest of segments */
802static int kexec_calculate_store_digests(struct kimage *image)
803{
804 struct sha256_ctx sctx;
805 int ret = 0, i, j, zero_buf_sz, sha_region_sz;
806 size_t nullsz;
807 u8 digest[SHA256_DIGEST_SIZE];
808 void *zero_buf;
809 struct kexec_sha_region *sha_regions;
810 struct purgatory_info *pi = &image->purgatory_info;
811
812 if (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY))
813 return 0;
814
815 zero_buf = __va(page_to_pfn(ZERO_PAGE(0)) << PAGE_SHIFT);
816 zero_buf_sz = PAGE_SIZE;
817
818 sha_region_sz = KEXEC_SEGMENT_MAX * sizeof(struct kexec_sha_region);
819 sha_regions = vzalloc(sha_region_sz);
820 if (!sha_regions)
821 return -ENOMEM;
822
823 sha256_init(ctx: &sctx);
824
825 for (j = i = 0; i < image->nr_segments; i++) {
826 struct kexec_segment *ksegment;
827
828#ifdef CONFIG_CRASH_HOTPLUG
829 /* Exclude elfcorehdr segment to allow future changes via hotplug */
830 if (i == image->elfcorehdr_index)
831 continue;
832#endif
833
834 ksegment = &image->segment[i];
835 /*
836 * Skip purgatory as it will be modified once we put digest
837 * info in purgatory.
838 */
839 if (ksegment->kbuf == pi->purgatory_buf)
840 continue;
841
842 /*
843 * Skip the segment if ima_segment_index is set and matches
844 * the current index
845 */
846 if (check_ima_segment_index(image, i))
847 continue;
848
849 sha256_update(ctx: &sctx, data: ksegment->kbuf, len: ksegment->bufsz);
850
851 /*
852 * Assume rest of the buffer is filled with zero and
853 * update digest accordingly.
854 */
855 nullsz = ksegment->memsz - ksegment->bufsz;
856 while (nullsz) {
857 unsigned long bytes = nullsz;
858
859 if (bytes > zero_buf_sz)
860 bytes = zero_buf_sz;
861 sha256_update(ctx: &sctx, data: zero_buf, len: bytes);
862 nullsz -= bytes;
863 }
864
865 sha_regions[j].start = ksegment->mem;
866 sha_regions[j].len = ksegment->memsz;
867 j++;
868 }
869
870 sha256_final(ctx: &sctx, out: digest);
871
872 ret = kexec_purgatory_get_set_symbol(image, name: "purgatory_sha_regions",
873 buf: sha_regions, size: sha_region_sz, get_value: 0);
874 if (ret)
875 goto out_free_sha_regions;
876
877 ret = kexec_purgatory_get_set_symbol(image, name: "purgatory_sha256_digest",
878 buf: digest, SHA256_DIGEST_SIZE, get_value: 0);
879out_free_sha_regions:
880 vfree(addr: sha_regions);
881 return ret;
882}
883
884#ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY
885/*
886 * kexec_purgatory_setup_kbuf - prepare buffer to load purgatory.
887 * @pi: Purgatory to be loaded.
888 * @kbuf: Buffer to setup.
889 *
890 * Allocates the memory needed for the buffer. Caller is responsible to free
891 * the memory after use.
892 *
893 * Return: 0 on success, negative errno on error.
894 */
895static int kexec_purgatory_setup_kbuf(struct purgatory_info *pi,
896 struct kexec_buf *kbuf)
897{
898 const Elf_Shdr *sechdrs;
899 unsigned long bss_align;
900 unsigned long bss_sz;
901 unsigned long align;
902 int i, ret;
903
904 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
905 kbuf->buf_align = bss_align = 1;
906 kbuf->bufsz = bss_sz = 0;
907
908 for (i = 0; i < pi->ehdr->e_shnum; i++) {
909 if (!(sechdrs[i].sh_flags & SHF_ALLOC))
910 continue;
911
912 align = sechdrs[i].sh_addralign;
913 if (sechdrs[i].sh_type != SHT_NOBITS) {
914 if (kbuf->buf_align < align)
915 kbuf->buf_align = align;
916 kbuf->bufsz = ALIGN(kbuf->bufsz, align);
917 kbuf->bufsz += sechdrs[i].sh_size;
918 } else {
919 if (bss_align < align)
920 bss_align = align;
921 bss_sz = ALIGN(bss_sz, align);
922 bss_sz += sechdrs[i].sh_size;
923 }
924 }
925 kbuf->bufsz = ALIGN(kbuf->bufsz, bss_align);
926 kbuf->memsz = kbuf->bufsz + bss_sz;
927 if (kbuf->buf_align < bss_align)
928 kbuf->buf_align = bss_align;
929
930 kbuf->buffer = vzalloc(kbuf->bufsz);
931 if (!kbuf->buffer)
932 return -ENOMEM;
933 pi->purgatory_buf = kbuf->buffer;
934
935 ret = kexec_add_buffer(kbuf);
936 if (ret)
937 goto out;
938
939 return 0;
940out:
941 vfree(addr: pi->purgatory_buf);
942 pi->purgatory_buf = NULL;
943 return ret;
944}
945
946/*
947 * kexec_purgatory_setup_sechdrs - prepares the pi->sechdrs buffer.
948 * @pi: Purgatory to be loaded.
949 * @kbuf: Buffer prepared to store purgatory.
950 *
951 * Allocates the memory needed for the buffer. Caller is responsible to free
952 * the memory after use.
953 *
954 * Return: 0 on success, negative errno on error.
955 */
956static int kexec_purgatory_setup_sechdrs(struct purgatory_info *pi,
957 struct kexec_buf *kbuf)
958{
959 unsigned long bss_addr;
960 unsigned long offset;
961 size_t sechdrs_size;
962 Elf_Shdr *sechdrs;
963 int i;
964
965 /*
966 * The section headers in kexec_purgatory are read-only. In order to
967 * have them modifiable make a temporary copy.
968 */
969 sechdrs_size = array_size(sizeof(Elf_Shdr), pi->ehdr->e_shnum);
970 sechdrs = vzalloc(sechdrs_size);
971 if (!sechdrs)
972 return -ENOMEM;
973 memcpy(sechdrs, (void *)pi->ehdr + pi->ehdr->e_shoff, sechdrs_size);
974 pi->sechdrs = sechdrs;
975
976 offset = 0;
977 bss_addr = kbuf->mem + kbuf->bufsz;
978 kbuf->image->start = pi->ehdr->e_entry;
979
980 for (i = 0; i < pi->ehdr->e_shnum; i++) {
981 unsigned long align;
982 void *src, *dst;
983
984 if (!(sechdrs[i].sh_flags & SHF_ALLOC))
985 continue;
986
987 align = sechdrs[i].sh_addralign;
988 if (sechdrs[i].sh_type == SHT_NOBITS) {
989 bss_addr = ALIGN(bss_addr, align);
990 sechdrs[i].sh_addr = bss_addr;
991 bss_addr += sechdrs[i].sh_size;
992 continue;
993 }
994
995 offset = ALIGN(offset, align);
996
997 /*
998 * Check if the segment contains the entry point, if so,
999 * calculate the value of image->start based on it.
1000 * If the compiler has produced more than one .text section
1001 * (Eg: .text.hot), they are generally after the main .text
1002 * section, and they shall not be used to calculate
1003 * image->start. So do not re-calculate image->start if it
1004 * is not set to the initial value, and warn the user so they
1005 * have a chance to fix their purgatory's linker script.
1006 */
1007 if (sechdrs[i].sh_flags & SHF_EXECINSTR &&
1008 pi->ehdr->e_entry >= sechdrs[i].sh_addr &&
1009 pi->ehdr->e_entry < (sechdrs[i].sh_addr
1010 + sechdrs[i].sh_size) &&
1011 !WARN_ON(kbuf->image->start != pi->ehdr->e_entry)) {
1012 kbuf->image->start -= sechdrs[i].sh_addr;
1013 kbuf->image->start += kbuf->mem + offset;
1014 }
1015
1016 src = (void *)pi->ehdr + sechdrs[i].sh_offset;
1017 dst = pi->purgatory_buf + offset;
1018 memcpy(dst, src, sechdrs[i].sh_size);
1019
1020 sechdrs[i].sh_addr = kbuf->mem + offset;
1021 sechdrs[i].sh_offset = offset;
1022 offset += sechdrs[i].sh_size;
1023 }
1024
1025 return 0;
1026}
1027
1028static int kexec_apply_relocations(struct kimage *image)
1029{
1030 int i, ret;
1031 struct purgatory_info *pi = &image->purgatory_info;
1032 const Elf_Shdr *sechdrs;
1033
1034 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
1035
1036 for (i = 0; i < pi->ehdr->e_shnum; i++) {
1037 const Elf_Shdr *relsec;
1038 const Elf_Shdr *symtab;
1039 Elf_Shdr *section;
1040
1041 relsec = sechdrs + i;
1042
1043 if (relsec->sh_type != SHT_RELA &&
1044 relsec->sh_type != SHT_REL)
1045 continue;
1046
1047 /*
1048 * For section of type SHT_RELA/SHT_REL,
1049 * ->sh_link contains section header index of associated
1050 * symbol table. And ->sh_info contains section header
1051 * index of section to which relocations apply.
1052 */
1053 if (relsec->sh_info >= pi->ehdr->e_shnum ||
1054 relsec->sh_link >= pi->ehdr->e_shnum)
1055 return -ENOEXEC;
1056
1057 section = pi->sechdrs + relsec->sh_info;
1058 symtab = sechdrs + relsec->sh_link;
1059
1060 if (!(section->sh_flags & SHF_ALLOC))
1061 continue;
1062
1063 /*
1064 * symtab->sh_link contain section header index of associated
1065 * string table.
1066 */
1067 if (symtab->sh_link >= pi->ehdr->e_shnum)
1068 /* Invalid section number? */
1069 continue;
1070
1071 /*
1072 * Respective architecture needs to provide support for applying
1073 * relocations of type SHT_RELA/SHT_REL.
1074 */
1075 if (relsec->sh_type == SHT_RELA)
1076 ret = arch_kexec_apply_relocations_add(pi, section,
1077 relsec, symtab);
1078 else if (relsec->sh_type == SHT_REL)
1079 ret = arch_kexec_apply_relocations(pi, section,
1080 relsec, symtab);
1081 if (ret)
1082 return ret;
1083 }
1084
1085 return 0;
1086}
1087
1088/*
1089 * kexec_load_purgatory - Load and relocate the purgatory object.
1090 * @image: Image to add the purgatory to.
1091 * @kbuf: Memory parameters to use.
1092 *
1093 * Allocates the memory needed for image->purgatory_info.sechdrs and
1094 * image->purgatory_info.purgatory_buf/kbuf->buffer. Caller is responsible
1095 * to free the memory after use.
1096 *
1097 * Return: 0 on success, negative errno on error.
1098 */
1099int kexec_load_purgatory(struct kimage *image, struct kexec_buf *kbuf)
1100{
1101 struct purgatory_info *pi = &image->purgatory_info;
1102 int ret;
1103
1104 if (kexec_purgatory_size <= 0)
1105 return -EINVAL;
1106
1107 pi->ehdr = (const Elf_Ehdr *)kexec_purgatory;
1108
1109 ret = kexec_purgatory_setup_kbuf(pi, kbuf);
1110 if (ret)
1111 return ret;
1112
1113 ret = kexec_purgatory_setup_sechdrs(pi, kbuf);
1114 if (ret)
1115 goto out_free_kbuf;
1116
1117 ret = kexec_apply_relocations(image);
1118 if (ret)
1119 goto out;
1120
1121 return 0;
1122out:
1123 vfree(addr: pi->sechdrs);
1124 pi->sechdrs = NULL;
1125out_free_kbuf:
1126 vfree(addr: pi->purgatory_buf);
1127 pi->purgatory_buf = NULL;
1128 return ret;
1129}
1130
1131/*
1132 * kexec_purgatory_find_symbol - find a symbol in the purgatory
1133 * @pi: Purgatory to search in.
1134 * @name: Name of the symbol.
1135 *
1136 * Return: pointer to symbol in read-only symtab on success, NULL on error.
1137 */
1138static const Elf_Sym *kexec_purgatory_find_symbol(struct purgatory_info *pi,
1139 const char *name)
1140{
1141 const Elf_Shdr *sechdrs;
1142 const Elf_Ehdr *ehdr;
1143 const Elf_Sym *syms;
1144 const char *strtab;
1145 int i, k;
1146
1147 if (!pi->ehdr)
1148 return NULL;
1149
1150 ehdr = pi->ehdr;
1151 sechdrs = (void *)ehdr + ehdr->e_shoff;
1152
1153 for (i = 0; i < ehdr->e_shnum; i++) {
1154 if (sechdrs[i].sh_type != SHT_SYMTAB)
1155 continue;
1156
1157 if (sechdrs[i].sh_link >= ehdr->e_shnum)
1158 /* Invalid strtab section number */
1159 continue;
1160 strtab = (void *)ehdr + sechdrs[sechdrs[i].sh_link].sh_offset;
1161 syms = (void *)ehdr + sechdrs[i].sh_offset;
1162
1163 /* Go through symbols for a match */
1164 for (k = 0; k < sechdrs[i].sh_size/sizeof(Elf_Sym); k++) {
1165 if (ELF_ST_BIND(syms[k].st_info) != STB_GLOBAL)
1166 continue;
1167
1168 if (strcmp(strtab + syms[k].st_name, name) != 0)
1169 continue;
1170
1171 if (syms[k].st_shndx == SHN_UNDEF ||
1172 syms[k].st_shndx >= ehdr->e_shnum) {
1173 pr_debug("Symbol: %s has bad section index %d.\n",
1174 name, syms[k].st_shndx);
1175 return NULL;
1176 }
1177
1178 /* Found the symbol we are looking for */
1179 return &syms[k];
1180 }
1181 }
1182
1183 return NULL;
1184}
1185
1186void *kexec_purgatory_get_symbol_addr(struct kimage *image, const char *name)
1187{
1188 struct purgatory_info *pi = &image->purgatory_info;
1189 const Elf_Sym *sym;
1190 Elf_Shdr *sechdr;
1191
1192 sym = kexec_purgatory_find_symbol(pi, name);
1193 if (!sym)
1194 return ERR_PTR(error: -EINVAL);
1195
1196 sechdr = &pi->sechdrs[sym->st_shndx];
1197
1198 /*
1199 * Returns the address where symbol will finally be loaded after
1200 * kexec_load_segment()
1201 */
1202 return (void *)(sechdr->sh_addr + sym->st_value);
1203}
1204
1205/*
1206 * Get or set value of a symbol. If "get_value" is true, symbol value is
1207 * returned in buf otherwise symbol value is set based on value in buf.
1208 */
1209int kexec_purgatory_get_set_symbol(struct kimage *image, const char *name,
1210 void *buf, unsigned int size, bool get_value)
1211{
1212 struct purgatory_info *pi = &image->purgatory_info;
1213 const Elf_Sym *sym;
1214 Elf_Shdr *sec;
1215 char *sym_buf;
1216
1217 sym = kexec_purgatory_find_symbol(pi, name);
1218 if (!sym)
1219 return -EINVAL;
1220
1221 if (sym->st_size != size) {
1222 pr_err("symbol %s size mismatch: expected %lu actual %u\n",
1223 name, (unsigned long)sym->st_size, size);
1224 return -EINVAL;
1225 }
1226
1227 sec = pi->sechdrs + sym->st_shndx;
1228
1229 if (sec->sh_type == SHT_NOBITS) {
1230 pr_err("symbol %s is in a bss section. Cannot %s\n", name,
1231 get_value ? "get" : "set");
1232 return -EINVAL;
1233 }
1234
1235 sym_buf = (char *)pi->purgatory_buf + sec->sh_offset + sym->st_value;
1236
1237 if (get_value)
1238 memcpy((void *)buf, sym_buf, size);
1239 else
1240 memcpy((void *)sym_buf, buf, size);
1241
1242 return 0;
1243}
1244#endif /* CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY */
1245

source code of linux/kernel/kexec_file.c