1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/arch/arm/mm/fault.c
4 *
5 * Copyright (C) 1995 Linus Torvalds
6 * Modifications for ARM processor (c) 1995-2004 Russell King
7 */
8#include <linux/extable.h>
9#include <linux/signal.h>
10#include <linux/mm.h>
11#include <linux/hardirq.h>
12#include <linux/init.h>
13#include <linux/kprobes.h>
14#include <linux/uaccess.h>
15#include <linux/page-flags.h>
16#include <linux/sched/signal.h>
17#include <linux/sched/debug.h>
18#include <linux/highmem.h>
19#include <linux/perf_event.h>
20#include <linux/kfence.h>
21
22#include <asm/system_misc.h>
23#include <asm/system_info.h>
24#include <asm/tlbflush.h>
25
26#include "fault.h"
27
28#ifdef CONFIG_MMU
29
30bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size)
31{
32 unsigned long addr = (unsigned long)unsafe_src;
33
34 return addr >= TASK_SIZE && ULONG_MAX - addr >= size;
35}
36
37/*
38 * This is useful to dump out the page tables associated with
39 * 'addr' in mm 'mm'.
40 */
41void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
42{
43 pgd_t *pgd;
44
45 if (!mm)
46 mm = &init_mm;
47
48 pgd = pgd_offset(mm, addr);
49 printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));
50
51 do {
52 p4d_t *p4d;
53 pud_t *pud;
54 pmd_t *pmd;
55 pte_t *pte;
56
57 p4d = p4d_offset(pgd, address: addr);
58 if (p4d_none(p4d: *p4d))
59 break;
60
61 if (p4d_bad(p4d: *p4d)) {
62 pr_cont("(bad)");
63 break;
64 }
65
66 pud = pud_offset(p4d, address: addr);
67 if (PTRS_PER_PUD != 1)
68 pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
69
70 if (pud_none(pud: *pud))
71 break;
72
73 if (pud_bad(pud: *pud)) {
74 pr_cont("(bad)");
75 break;
76 }
77
78 pmd = pmd_offset(pud, address: addr);
79 if (PTRS_PER_PMD != 1)
80 pr_cont(", *pmd=%08llx", (long long)pmd_val(*pmd));
81
82 if (pmd_none(pmd: *pmd))
83 break;
84
85 if (pmd_bad(pmd: *pmd)) {
86 pr_cont("(bad)");
87 break;
88 }
89
90 /* We must not map this if we have highmem enabled */
91 if (PageHighMem(pfn_to_page(pmd_val(*pmd) >> PAGE_SHIFT)))
92 break;
93
94 pte = pte_offset_map(pmd, addr);
95 if (!pte)
96 break;
97
98 pr_cont(", *pte=%08llx", (long long)pte_val(*pte));
99#ifndef CONFIG_ARM_LPAE
100 pr_cont(", *ppte=%08llx",
101 (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
102#endif
103 pte_unmap(pte);
104 } while(0);
105
106 pr_cont("\n");
107}
108#else /* CONFIG_MMU */
109void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
110{ }
111#endif /* CONFIG_MMU */
112
113static inline bool is_write_fault(unsigned int fsr)
114{
115 return (fsr & FSR_WRITE) && !(fsr & FSR_CM);
116}
117
118static inline bool is_translation_fault(unsigned int fsr)
119{
120 int fs = fsr_fs(fsr);
121#ifdef CONFIG_ARM_LPAE
122 if ((fs & FS_MMU_NOLL_MASK) == FS_TRANS_NOLL)
123 return true;
124#else
125 if (fs == FS_L1_TRANS || fs == FS_L2_TRANS)
126 return true;
127#endif
128 return false;
129}
130
131static inline bool is_permission_fault(unsigned int fsr)
132{
133 int fs = fsr_fs(fsr);
134#ifdef CONFIG_ARM_LPAE
135 if ((fs & FS_MMU_NOLL_MASK) == FS_PERM_NOLL)
136 return true;
137#else
138 if (fs == FS_L1_PERM || fs == FS_L2_PERM)
139 return true;
140#endif
141 return false;
142}
143
144static void die_kernel_fault(const char *msg, struct mm_struct *mm,
145 unsigned long addr, unsigned int fsr,
146 struct pt_regs *regs)
147{
148 bust_spinlocks(yes: 1);
149 pr_alert("8<--- cut here ---\n");
150 pr_alert("Unable to handle kernel %s at virtual address %08lx when %s\n",
151 msg, addr, fsr & FSR_LNX_PF ? "execute" : str_write_read(fsr & FSR_WRITE));
152
153 show_pte(KERN_ALERT, mm, addr);
154 die("Oops", regs, fsr);
155 bust_spinlocks(yes: 0);
156 make_task_dead(SIGKILL);
157}
158
159/*
160 * Oops. The kernel tried to access some page that wasn't present.
161 */
162static void
163__do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
164 struct pt_regs *regs)
165{
166 const char *msg;
167 /*
168 * Are we prepared to handle this kernel fault?
169 */
170 if (fixup_exception(regs))
171 return;
172
173 /*
174 * No handler, we'll have to terminate things with extreme prejudice.
175 */
176 if (addr < PAGE_SIZE) {
177 msg = "NULL pointer dereference";
178 } else if (is_permission_fault(fsr) && fsr & FSR_LNX_PF) {
179 msg = "execution of memory";
180 } else {
181 if (is_translation_fault(fsr) &&
182 kfence_handle_page_fault(addr, is_write: is_write_fault(fsr), regs))
183 return;
184
185 msg = "paging request";
186 }
187
188 die_kernel_fault(msg, mm, addr, fsr, regs);
189}
190
191/*
192 * Something tried to access memory that isn't in our memory map..
193 * User mode accesses just cause a SIGSEGV
194 */
195static void
196__do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig,
197 int code, struct pt_regs *regs)
198{
199 struct task_struct *tsk = current;
200
201#ifdef CONFIG_DEBUG_USER
202 if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) ||
203 ((user_debug & UDBG_BUS) && (sig == SIGBUS))) {
204 pr_err("8<--- cut here ---\n");
205 pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n",
206 tsk->comm, sig, addr, fsr);
207 show_pte(KERN_ERR, tsk->mm, addr);
208 show_regs(regs);
209 }
210#endif
211#ifndef CONFIG_KUSER_HELPERS
212 if ((sig == SIGSEGV) && ((addr & PAGE_MASK) == 0xffff0000))
213 printk_ratelimited(KERN_DEBUG
214 "%s: CONFIG_KUSER_HELPERS disabled at 0x%08lx\n",
215 tsk->comm, addr);
216#endif
217
218 tsk->thread.address = addr;
219 tsk->thread.error_code = fsr;
220 tsk->thread.trap_no = 14;
221 force_sig_fault(sig, code, addr: (void __user *)addr);
222}
223
224void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
225{
226 struct task_struct *tsk = current;
227 struct mm_struct *mm = tsk->active_mm;
228
229 /*
230 * If we are in kernel mode at this point, we
231 * have no context to handle this fault with.
232 */
233 if (user_mode(regs))
234 __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
235 else
236 __do_kernel_fault(mm, addr, fsr, regs);
237}
238
239#ifdef CONFIG_MMU
240#ifdef CONFIG_CPU_TTBR0_PAN
241static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs)
242{
243 struct svc_pt_regs *svcregs;
244
245 /* If we are in user mode: permission granted */
246 if (user_mode(regs))
247 return true;
248
249 /* uaccess state saved above pt_regs on SVC exception entry */
250 svcregs = to_svc_pt_regs(regs);
251
252 return !(svcregs->ttbcr & TTBCR_EPD0);
253}
254#else
255static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs)
256{
257 return true;
258}
259#endif
260
261static int __kprobes
262do_kernel_address_page_fault(struct mm_struct *mm, unsigned long addr,
263 unsigned int fsr, struct pt_regs *regs)
264{
265 if (user_mode(regs)) {
266 /*
267 * Fault from user mode for a kernel space address. User mode
268 * should not be faulting in kernel space, which includes the
269 * vector/khelper page. Handle the branch predictor hardening
270 * while interrupts are still disabled, then send a SIGSEGV.
271 */
272 harden_branch_predictor();
273 __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs);
274 } else {
275 /*
276 * Fault from kernel mode. Enable interrupts if they were
277 * enabled in the parent context. Section (upper page table)
278 * translation faults are handled via do_translation_fault(),
279 * so we will only get here for a non-present kernel space
280 * PTE or PTE permission fault. This may happen in exceptional
281 * circumstances and need the fixup tables to be walked.
282 */
283 if (interrupts_enabled(regs))
284 local_irq_enable();
285
286 __do_kernel_fault(mm, addr, fsr, regs);
287 }
288
289 return 0;
290}
291
292static int __kprobes
293do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
294{
295 struct mm_struct *mm = current->mm;
296 struct vm_area_struct *vma;
297 int sig, code;
298 vm_fault_t fault;
299 unsigned int flags = FAULT_FLAG_DEFAULT;
300 vm_flags_t vm_flags = VM_ACCESS_FLAGS;
301
302 if (kprobe_page_fault(regs, trap: fsr))
303 return 0;
304
305 /*
306 * Handle kernel addresses faults separately, which avoids touching
307 * the mmap lock from contexts that are not able to sleep.
308 */
309 if (addr >= TASK_SIZE)
310 return do_kernel_address_page_fault(mm, addr, fsr, regs);
311
312 /* Enable interrupts if they were enabled in the parent context. */
313 if (interrupts_enabled(regs))
314 local_irq_enable();
315
316 /*
317 * If we're in an interrupt or have no user
318 * context, we must not take the fault..
319 */
320 if (faulthandler_disabled() || !mm)
321 goto no_context;
322
323 if (user_mode(regs))
324 flags |= FAULT_FLAG_USER;
325
326 if (is_write_fault(fsr)) {
327 flags |= FAULT_FLAG_WRITE;
328 vm_flags = VM_WRITE;
329 }
330
331 if (fsr & FSR_LNX_PF) {
332 vm_flags = VM_EXEC;
333
334 if (is_permission_fault(fsr) && !user_mode(regs))
335 die_kernel_fault(msg: "execution of memory",
336 mm, addr, fsr, regs);
337 }
338
339 perf_sw_event(event_id: PERF_COUNT_SW_PAGE_FAULTS, nr: 1, regs, addr);
340
341 /*
342 * Privileged access aborts with CONFIG_CPU_TTBR0_PAN enabled are
343 * routed via the translation fault mechanism. Check whether uaccess
344 * is disabled while in kernel mode.
345 */
346 if (!ttbr0_usermode_access_allowed(regs))
347 goto no_context;
348
349 if (!(flags & FAULT_FLAG_USER))
350 goto lock_mmap;
351
352 vma = lock_vma_under_rcu(mm, address: addr);
353 if (!vma)
354 goto lock_mmap;
355
356 if (!(vma->vm_flags & vm_flags)) {
357 vma_end_read(vma);
358 count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
359 fault = 0;
360 code = SEGV_ACCERR;
361 goto bad_area;
362 }
363 fault = handle_mm_fault(vma, address: addr, flags: flags | FAULT_FLAG_VMA_LOCK, regs);
364 if (!(fault & (VM_FAULT_RETRY | VM_FAULT_COMPLETED)))
365 vma_end_read(vma);
366
367 if (!(fault & VM_FAULT_RETRY)) {
368 count_vm_vma_lock_event(VMA_LOCK_SUCCESS);
369 goto done;
370 }
371 count_vm_vma_lock_event(VMA_LOCK_RETRY);
372 if (fault & VM_FAULT_MAJOR)
373 flags |= FAULT_FLAG_TRIED;
374
375 /* Quick path to respond to signals */
376 if (fault_signal_pending(fault_flags: fault, regs)) {
377 if (!user_mode(regs))
378 goto no_context;
379 return 0;
380 }
381lock_mmap:
382
383retry:
384 vma = lock_mm_and_find_vma(mm, address: addr, regs);
385 if (unlikely(!vma)) {
386 fault = 0;
387 code = SEGV_MAPERR;
388 goto bad_area;
389 }
390
391 /*
392 * ok, we have a good vm_area for this memory access, check the
393 * permissions on the VMA allow for the fault which occurred.
394 */
395 if (!(vma->vm_flags & vm_flags)) {
396 mmap_read_unlock(mm);
397 fault = 0;
398 code = SEGV_ACCERR;
399 goto bad_area;
400 }
401
402 fault = handle_mm_fault(vma, address: addr & PAGE_MASK, flags, regs);
403
404 /* If we need to retry but a fatal signal is pending, handle the
405 * signal first. We do not need to release the mmap_lock because
406 * it would already be released in __lock_page_or_retry in
407 * mm/filemap.c. */
408 if (fault_signal_pending(fault_flags: fault, regs)) {
409 if (!user_mode(regs))
410 goto no_context;
411 return 0;
412 }
413
414 /* The fault is fully completed (including releasing mmap lock) */
415 if (fault & VM_FAULT_COMPLETED)
416 return 0;
417
418 if (!(fault & VM_FAULT_ERROR)) {
419 if (fault & VM_FAULT_RETRY) {
420 flags |= FAULT_FLAG_TRIED;
421 goto retry;
422 }
423 }
424
425 mmap_read_unlock(mm);
426done:
427
428 /* Handle the "normal" case first */
429 if (likely(!(fault & VM_FAULT_ERROR)))
430 return 0;
431
432 code = SEGV_MAPERR;
433bad_area:
434 /*
435 * If we are in kernel mode at this point, we
436 * have no context to handle this fault with.
437 */
438 if (!user_mode(regs))
439 goto no_context;
440
441 if (fault & VM_FAULT_OOM) {
442 /*
443 * We ran out of memory, call the OOM killer, and return to
444 * userspace (which will retry the fault, or kill us if we
445 * got oom-killed)
446 */
447 pagefault_out_of_memory();
448 return 0;
449 }
450
451 if (fault & VM_FAULT_SIGBUS) {
452 /*
453 * We had some memory, but were unable to
454 * successfully fix up this page fault.
455 */
456 sig = SIGBUS;
457 code = BUS_ADRERR;
458 } else {
459 /*
460 * Something tried to access memory that
461 * isn't in our memory map..
462 */
463 sig = SIGSEGV;
464 }
465
466 __do_user_fault(addr, fsr, sig, code, regs);
467 return 0;
468
469no_context:
470 __do_kernel_fault(mm, addr, fsr, regs);
471 return 0;
472}
473#else /* CONFIG_MMU */
474static int
475do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
476{
477 return 0;
478}
479#endif /* CONFIG_MMU */
480
481/*
482 * First Level Translation Fault Handler
483 *
484 * We enter here because the first level page table doesn't contain
485 * a valid entry for the address.
486 *
487 * If this is a user address (addr < TASK_SIZE), we handle this as a
488 * normal page fault. This leaves the remainder of the function to handle
489 * kernel address translation faults.
490 *
491 * Since user mode is not permitted to access kernel addresses, pass these
492 * directly to do_kernel_address_page_fault() to handle.
493 *
494 * Otherwise, we're probably faulting in the vmalloc() area, so try to fix
495 * that up. Note that we must not take any locks or enable interrupts in
496 * this case.
497 *
498 * If vmalloc() fixup fails, that means the non-leaf page tables did not
499 * contain an entry for this address, so handle this via
500 * do_kernel_address_page_fault().
501 */
502#ifdef CONFIG_MMU
503static int __kprobes
504do_translation_fault(unsigned long addr, unsigned int fsr,
505 struct pt_regs *regs)
506{
507 unsigned int index;
508 pgd_t *pgd, *pgd_k;
509 p4d_t *p4d, *p4d_k;
510 pud_t *pud, *pud_k;
511 pmd_t *pmd, *pmd_k;
512
513 if (addr < TASK_SIZE)
514 return do_page_fault(addr, fsr, regs);
515
516 if (user_mode(regs))
517 goto bad_area;
518
519 index = pgd_index(addr);
520
521 pgd = cpu_get_pgd() + index;
522 pgd_k = init_mm.pgd + index;
523
524 p4d = p4d_offset(pgd, address: addr);
525 p4d_k = p4d_offset(pgd: pgd_k, address: addr);
526
527 if (p4d_none(p4d: *p4d_k))
528 goto bad_area;
529 if (!p4d_present(p4d: *p4d))
530 set_p4d(p4dp: p4d, p4d: *p4d_k);
531
532 pud = pud_offset(p4d, address: addr);
533 pud_k = pud_offset(p4d: p4d_k, address: addr);
534
535 if (pud_none(pud: *pud_k))
536 goto bad_area;
537 if (!pud_present(pud: *pud))
538 set_pud(pudp: pud, pud: *pud_k);
539
540 pmd = pmd_offset(pud, address: addr);
541 pmd_k = pmd_offset(pud: pud_k, address: addr);
542
543#ifdef CONFIG_ARM_LPAE
544 /*
545 * Only one hardware entry per PMD with LPAE.
546 */
547 index = 0;
548#else
549 /*
550 * On ARM one Linux PGD entry contains two hardware entries (see page
551 * tables layout in pgtable.h). We normally guarantee that we always
552 * fill both L1 entries. But create_mapping() doesn't follow the rule.
553 * It can create inidividual L1 entries, so here we have to call
554 * pmd_none() check for the entry really corresponded to address, not
555 * for the first of pair.
556 */
557 index = (addr >> SECTION_SHIFT) & 1;
558#endif
559 if (pmd_none(pmd: pmd_k[index]))
560 goto bad_area;
561
562 copy_pmd(pmd, pmd_k);
563 return 0;
564
565bad_area:
566 do_kernel_address_page_fault(current->mm, addr, fsr, regs);
567
568 return 0;
569}
570#else /* CONFIG_MMU */
571static int
572do_translation_fault(unsigned long addr, unsigned int fsr,
573 struct pt_regs *regs)
574{
575 return 0;
576}
577#endif /* CONFIG_MMU */
578
579/*
580 * Some section permission faults need to be handled gracefully.
581 * They can happen due to a __{get,put}_user during an oops.
582 */
583#ifndef CONFIG_ARM_LPAE
584static int
585do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
586{
587 /*
588 * If this is a kernel address, but from user mode, then userspace
589 * is trying bad stuff. Invoke the branch predictor handling.
590 * Interrupts are disabled here.
591 */
592 if (addr >= TASK_SIZE && user_mode(regs))
593 harden_branch_predictor();
594
595 do_bad_area(addr, fsr, regs);
596
597 return 0;
598}
599#endif /* CONFIG_ARM_LPAE */
600
601/*
602 * This abort handler always returns "fault".
603 */
604static int
605do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
606{
607 return 1;
608}
609
610struct fsr_info {
611 int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
612 int sig;
613 int code;
614 const char *name;
615};
616
617/* FSR definition */
618#ifdef CONFIG_ARM_LPAE
619#include "fsr-3level.c"
620#else
621#include "fsr-2level.c"
622#endif
623
624void __init
625hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
626 int sig, int code, const char *name)
627{
628 if (nr < 0 || nr >= ARRAY_SIZE(fsr_info))
629 BUG();
630
631 fsr_info[nr].fn = fn;
632 fsr_info[nr].sig = sig;
633 fsr_info[nr].code = code;
634 fsr_info[nr].name = name;
635}
636
637/*
638 * Dispatch a data abort to the relevant handler.
639 */
640asmlinkage void
641do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
642{
643 const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
644
645 if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs))
646 return;
647
648 pr_alert("8<--- cut here ---\n");
649 pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n",
650 inf->name, fsr, addr);
651 show_pte(KERN_ALERT, current->mm, addr);
652
653 arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
654 fsr, 0);
655}
656
657void __init
658hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
659 int sig, int code, const char *name)
660{
661 if (nr < 0 || nr >= ARRAY_SIZE(ifsr_info))
662 BUG();
663
664 ifsr_info[nr].fn = fn;
665 ifsr_info[nr].sig = sig;
666 ifsr_info[nr].code = code;
667 ifsr_info[nr].name = name;
668}
669
670asmlinkage void
671do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
672{
673 const struct fsr_info *inf = ifsr_info + fsr_fs(fsr: ifsr);
674
675 if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs))
676 return;
677
678 pr_alert("8<--- cut here ---\n");
679 pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n",
680 inf->name, ifsr, addr);
681
682 arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr,
683 ifsr, 0);
684}
685
686/*
687 * Abort handler to be used only during first unmasking of asynchronous aborts
688 * on the boot CPU. This makes sure that the machine will not die if the
689 * firmware/bootloader left an imprecise abort pending for us to trip over.
690 */
691static int __init early_abort_handler(unsigned long addr, unsigned int fsr,
692 struct pt_regs *regs)
693{
694 pr_warn("Hit pending asynchronous external abort (FSR=0x%08x) during "
695 "first unmask, this is most likely caused by a "
696 "firmware/bootloader bug.\n", fsr);
697
698 return 0;
699}
700
701void __init early_abt_enable(void)
702{
703 fsr_info[FSR_FS_AEA].fn = early_abort_handler;
704 local_abt_enable();
705 fsr_info[FSR_FS_AEA].fn = do_bad;
706}
707
708#ifndef CONFIG_ARM_LPAE
709static int __init exceptions_init(void)
710{
711 if (cpu_architecture() >= CPU_ARCH_ARMv6) {
712 hook_fault_code(nr: 4, fn: do_translation_fault, SIGSEGV, SEGV_MAPERR,
713 name: "I-cache maintenance fault");
714 }
715
716 if (cpu_architecture() >= CPU_ARCH_ARMv7) {
717 /*
718 * TODO: Access flag faults introduced in ARMv6K.
719 * Runtime check for 'K' extension is needed
720 */
721 hook_fault_code(nr: 3, fn: do_bad, SIGSEGV, SEGV_MAPERR,
722 name: "section access flag fault");
723 hook_fault_code(nr: 6, fn: do_bad, SIGSEGV, SEGV_MAPERR,
724 name: "section access flag fault");
725 }
726
727 return 0;
728}
729
730arch_initcall(exceptions_init);
731#endif
732

source code of linux/arch/arm/mm/fault.c