1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 1994-2002 Russell King
4 * Copyright (c) 2003, 2020 ARM Limited
5 * All Rights Reserved
6 */
7
8#include <linux/init.h>
9#include <linux/linkage.h>
10#include <asm/assembler.h>
11#include <asm/page.h>
12
13#ifdef __ARMEB__
14#define LOW_OFFSET 0x4
15#define HIGH_OFFSET 0x0
16#else
17#define LOW_OFFSET 0x0
18#define HIGH_OFFSET 0x4
19#endif
20
21/*
22 * __fixup_pv_table - patch the stub instructions with the delta between
23 * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be
24 * 2 MiB aligned.
25 *
26 * Called from head.S, which expects the following registers to be preserved:
27 * r1 = machine no, r2 = atags or dtb,
28 * r8 = phys_offset, r9 = cpuid, r10 = procinfo
29 */
30 __HEAD
31ENTRY(__fixup_pv_table)
32 mov r0, r8, lsr #PAGE_SHIFT @ convert to PFN
33 str_l r0, __pv_phys_pfn_offset, r3
34
35 adr_l r0, __pv_offset
36 subs r3, r8, #PAGE_OFFSET @ PHYS_OFFSET - PAGE_OFFSET
37 mvn ip, #0
38 strcc ip, [r0, #HIGH_OFFSET] @ save to __pv_offset high bits
39 str r3, [r0, #LOW_OFFSET] @ save to __pv_offset low bits
40
41 mov r0, r3, lsr #21 @ constant for add/sub instructions
42 teq r3, r0, lsl #21 @ must be 2 MiB aligned
43 bne 0f
44
45 adr_l r4, __pv_table_begin
46 adr_l r5, __pv_table_end
47 b __fixup_a_pv_table
48
490: mov r0, r0 @ deadloop on error
50 b 0b
51ENDPROC(__fixup_pv_table)
52
53 .text
54__fixup_a_pv_table:
55 adr_l r6, __pv_offset
56 ldr r0, [r6, #HIGH_OFFSET] @ pv_offset high word
57 ldr r6, [r6, #LOW_OFFSET] @ pv_offset low word
58 cmn r0, #1
59#ifdef CONFIG_THUMB2_KERNEL
60 @
61 @ The Thumb-2 versions of the patchable sequences are
62 @
63 @ phys-to-virt: movw <reg>, #offset<31:21>
64 @ lsl <reg>, #21
65 @ sub <VA>, <PA>, <reg>
66 @
67 @ virt-to-phys (non-LPAE): movw <reg>, #offset<31:21>
68 @ lsl <reg>, #21
69 @ add <PA>, <VA>, <reg>
70 @
71 @ virt-to-phys (LPAE): movw <reg>, #offset<31:21>
72 @ lsl <reg>, #21
73 @ adds <PAlo>, <VA>, <reg>
74 @ mov <PAhi>, #offset<39:32>
75 @ adc <PAhi>, <PAhi>, #0
76 @
77 @ In the non-LPAE case, all patchable instructions are MOVW
78 @ instructions, where we need to patch in the offset into the
79 @ second halfword of the opcode (the 16-bit immediate is encoded
80 @ as imm4:i:imm3:imm8)
81 @
82 @ 15 11 10 9 4 3 0 15 14 12 11 8 7 0
83 @ +-----------+---+-------------+------++---+------+----+------+
84 @ MOVW | 1 1 1 1 0 | i | 1 0 0 1 0 0 | imm4 || 0 | imm3 | Rd | imm8 |
85 @ +-----------+---+-------------+------++---+------+----+------+
86 @
87 @ In the LPAE case, we also need to patch in the high word of the
88 @ offset into the immediate field of the MOV instruction, or patch it
89 @ to a MVN instruction if the offset is negative. In this case, we
90 @ need to inspect the first halfword of the opcode, to check whether
91 @ it is MOVW or MOV/MVN, and to perform the MOV to MVN patching if
92 @ needed. The encoding of the immediate is rather complex for values
93 @ of i:imm3 != 0b0000, but fortunately, we never need more than 8 lower
94 @ order bits, which can be patched into imm8 directly (and i:imm3
95 @ cleared)
96 @
97 @ 15 11 10 9 5 0 15 14 12 11 8 7 0
98 @ +-----------+---+---------------------++---+------+----+------+
99 @ MOV | 1 1 1 1 0 | i | 0 0 0 1 0 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
100 @ MVN | 1 1 1 1 0 | i | 0 0 0 1 1 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
101 @ +-----------+---+---------------------++---+------+----+------+
102 @
103 moveq r0, #0x200000 @ set bit 21, mov to mvn instruction
104 lsrs r3, r6, #29 @ isolate top 3 bits of displacement
105 ubfx r6, r6, #21, #8 @ put bits 28:21 into the MOVW imm8 field
106 bfi r6, r3, #12, #3 @ put bits 31:29 into the MOVW imm3 field
107 b .Lnext
108.Lloop: add r7, r4
109 adds r4, #4 @ clears Z flag
110#ifdef CONFIG_ARM_LPAE
111 ldrh ip, [r7]
112ARM_BE8(rev16 ip, ip)
113 tst ip, #0x200 @ MOVW has bit 9 set, MVN has it clear
114 bne 0f @ skip to MOVW handling (Z flag is clear)
115 bic ip, #0x20 @ clear bit 5 (MVN -> MOV)
116 orr ip, ip, r0, lsr #16 @ MOV -> MVN if offset < 0
117ARM_BE8(rev16 ip, ip)
118 strh ip, [r7]
119 @ Z flag is set
1200:
121#endif
122 ldrh ip, [r7, #2]
123ARM_BE8(rev16 ip, ip)
124 and ip, #0xf00 @ clear everything except Rd field
125 orreq ip, r0 @ Z flag set -> MOV/MVN -> patch in high bits
126 orrne ip, r6 @ Z flag clear -> MOVW -> patch in low bits
127ARM_BE8(rev16 ip, ip)
128 strh ip, [r7, #2]
129#else
130#ifdef CONFIG_CPU_ENDIAN_BE8
131@ in BE8, we load data in BE, but instructions still in LE
132#define PV_BIT24 0x00000001
133#define PV_IMM8_MASK 0xff000000
134#define PV_IMMR_MSB 0x00080000
135#else
136#define PV_BIT24 0x01000000
137#define PV_IMM8_MASK 0x000000ff
138#define PV_IMMR_MSB 0x00000800
139#endif
140
141 @
142 @ The ARM versions of the patchable sequences are
143 @
144 @ phys-to-virt: sub <VA>, <PA>, #offset<31:24>, lsl #24
145 @ sub <VA>, <PA>, #offset<23:16>, lsl #16
146 @
147 @ virt-to-phys (non-LPAE): add <PA>, <VA>, #offset<31:24>, lsl #24
148 @ add <PA>, <VA>, #offset<23:16>, lsl #16
149 @
150 @ virt-to-phys (LPAE): movw <reg>, #offset<31:20>
151 @ adds <PAlo>, <VA>, <reg>, lsl #20
152 @ mov <PAhi>, #offset<39:32>
153 @ adc <PAhi>, <PAhi>, #0
154 @
155 @ In the non-LPAE case, all patchable instructions are ADD or SUB
156 @ instructions, where we need to patch in the offset into the
157 @ immediate field of the opcode, which is emitted with the correct
158 @ rotation value. (The effective value of the immediate is imm12<7:0>
159 @ rotated right by [2 * imm12<11:8>] bits)
160 @
161 @ 31 28 27 23 22 20 19 16 15 12 11 0
162 @ +------+-----------------+------+------+-------+
163 @ ADD | cond | 0 0 1 0 1 0 0 0 | Rn | Rd | imm12 |
164 @ SUB | cond | 0 0 1 0 0 1 0 0 | Rn | Rd | imm12 |
165 @ MOV | cond | 0 0 1 1 1 0 1 0 | Rn | Rd | imm12 |
166 @ MVN | cond | 0 0 1 1 1 1 1 0 | Rn | Rd | imm12 |
167 @ +------+-----------------+------+------+-------+
168 @
169 @ In the LPAE case, we use a MOVW instruction to carry the low offset
170 @ word, and patch in the high word of the offset into the immediate
171 @ field of the subsequent MOV instruction, or patch it to a MVN
172 @ instruction if the offset is negative. We can distinguish MOVW
173 @ instructions based on bits 23:22 of the opcode, and ADD/SUB can be
174 @ distinguished from MOV/MVN (all using the encodings above) using
175 @ bit 24.
176 @
177 @ 31 28 27 23 22 20 19 16 15 12 11 0
178 @ +------+-----------------+------+------+-------+
179 @ MOVW | cond | 0 0 1 1 0 0 0 0 | imm4 | Rd | imm12 |
180 @ +------+-----------------+------+------+-------+
181 @
182 moveq r0, #0x400000 @ set bit 22, mov to mvn instruction
183 mov r3, r6, lsr #16 @ put offset bits 31-16 into r3
184 mov r6, r6, lsr #24 @ put offset bits 31-24 into r6
185 and r3, r3, #0xf0 @ only keep offset bits 23-20 in r3
186 b .Lnext
187.Lloop: ldr ip, [r7, r4]
188#ifdef CONFIG_ARM_LPAE
189 tst ip, #PV_BIT24 @ ADD/SUB have bit 24 clear
190 beq 1f
191ARM_BE8(rev ip, ip)
192 tst ip, #0xc00000 @ MOVW has bits 23:22 clear
193 bic ip, ip, #0x400000 @ clear bit 22
194 bfc ip, #0, #12 @ clear imm12 field of MOV[W] instruction
195 orreq ip, ip, r6, lsl #4 @ MOVW -> mask in offset bits 31-24
196 orreq ip, ip, r3, lsr #4 @ MOVW -> mask in offset bits 23-20
197 orrne ip, ip, r0 @ MOV -> mask in offset bits 7-0 (or bit 22)
198ARM_BE8(rev ip, ip)
199 b 2f
2001:
201#endif
202 tst ip, #PV_IMMR_MSB @ rotation value >= 16 ?
203 bic ip, ip, #PV_IMM8_MASK
204 orreq ip, ip, r6 ARM_BE8(, lsl #24) @ mask in offset bits 31-24
205 orrne ip, ip, r3 ARM_BE8(, lsl #24) @ mask in offset bits 23-20
2062:
207 str ip, [r7, r4]
208 add r4, r4, #4
209#endif
210
211.Lnext:
212 cmp r4, r5
213 ldrcc r7, [r4] @ use branch for delay slot
214 bcc .Lloop
215 ret lr
216ENDPROC(__fixup_a_pv_table)
217
218ENTRY(fixup_pv_table)
219 stmfd sp!, {r4 - r7, lr}
220 mov r4, r0 @ r0 = table start
221 add r5, r0, r1 @ r1 = table size
222 bl __fixup_a_pv_table
223 ldmfd sp!, {r4 - r7, pc}
224ENDPROC(fixup_pv_table)
225
226 .data
227 .align 2
228 .globl __pv_phys_pfn_offset
229 .type __pv_phys_pfn_offset, %object
230__pv_phys_pfn_offset:
231 .word 0
232 .size __pv_phys_pfn_offset, . -__pv_phys_pfn_offset
233
234 .globl __pv_offset
235 .type __pv_offset, %object
236__pv_offset:
237 .quad 0
238 .size __pv_offset, . -__pv_offset
239

source code of linux/arch/arm/kernel/phys2virt.S