1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2023 SiFive
4 * Author: Andy Chiu <andy.chiu@sifive.com>
5 */
6#include <linux/export.h>
7#include <linux/sched/signal.h>
8#include <linux/types.h>
9#include <linux/slab.h>
10#include <linux/sched.h>
11#include <linux/uaccess.h>
12#include <linux/prctl.h>
13
14#include <asm/thread_info.h>
15#include <asm/processor.h>
16#include <asm/insn.h>
17#include <asm/vector.h>
18#include <asm/csr.h>
19#include <asm/elf.h>
20#include <asm/ptrace.h>
21#include <asm/bug.h>
22
23static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
24static struct kmem_cache *riscv_v_user_cachep;
25#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
26static struct kmem_cache *riscv_v_kernel_cachep;
27#endif
28
29unsigned long riscv_v_vsize __read_mostly;
30EXPORT_SYMBOL_GPL(riscv_v_vsize);
31
32int riscv_v_setup_vsize(void)
33{
34 unsigned long this_vsize;
35
36 /*
37 * There are 32 vector registers with vlenb length.
38 *
39 * If the thead,vlenb property was provided by the firmware, use that
40 * instead of probing the CSRs.
41 */
42 if (thead_vlenb_of) {
43 riscv_v_vsize = thead_vlenb_of * 32;
44 return 0;
45 }
46
47 riscv_v_enable();
48 this_vsize = csr_read(CSR_VLENB) * 32;
49 riscv_v_disable();
50
51 if (!riscv_v_vsize) {
52 riscv_v_vsize = this_vsize;
53 return 0;
54 }
55
56 if (riscv_v_vsize != this_vsize) {
57 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
58 return -EOPNOTSUPP;
59 }
60
61 return 0;
62}
63
64void __init riscv_v_setup_ctx_cache(void)
65{
66 if (!(has_vector() || has_xtheadvector()))
67 return;
68
69 update_regset_vector_info(riscv_v_vsize);
70
71 riscv_v_user_cachep = kmem_cache_create_usercopy(name: "riscv_vector_ctx",
72 size: riscv_v_vsize, align: 16, SLAB_PANIC,
73 useroffset: 0, usersize: riscv_v_vsize, NULL);
74#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
75 riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx",
76 riscv_v_vsize, 16,
77 SLAB_PANIC, NULL);
78#endif
79}
80
81bool insn_is_vector(u32 insn_buf)
82{
83 u32 opcode = insn_buf & __INSN_OPCODE_MASK;
84 u32 width, csr;
85
86 /*
87 * All V-related instructions, including CSR operations are 4-Byte. So,
88 * do not handle if the instruction length is not 4-Byte.
89 */
90 if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
91 return false;
92
93 switch (opcode) {
94 case RVV_OPCODE_VECTOR:
95 return true;
96 case RVV_OPCODE_VL:
97 case RVV_OPCODE_VS:
98 width = RVV_EXTRACT_VL_VS_WIDTH(insn_buf);
99 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
100 width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
101 return true;
102
103 break;
104 case RVG_OPCODE_SYSTEM:
105 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
106 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
107 (csr >= CSR_VL && csr <= CSR_VLENB))
108 return true;
109 }
110
111 return false;
112}
113
114static int riscv_v_thread_zalloc(struct kmem_cache *cache,
115 struct __riscv_v_ext_state *ctx)
116{
117 void *datap;
118
119 datap = kmem_cache_zalloc(cache, GFP_KERNEL);
120 if (!datap)
121 return -ENOMEM;
122
123 ctx->datap = datap;
124 memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap));
125 return 0;
126}
127
128void riscv_v_thread_alloc(struct task_struct *tsk)
129{
130#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
131 riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate);
132#endif
133}
134
135void riscv_v_thread_free(struct task_struct *tsk)
136{
137 if (tsk->thread.vstate.datap)
138 kmem_cache_free(s: riscv_v_user_cachep, objp: tsk->thread.vstate.datap);
139#ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
140 if (tsk->thread.kernel_vstate.datap)
141 kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap);
142#endif
143}
144
145#define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
146#define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
147#define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
148#define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
149static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
150{
151 return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
152}
153
154static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
155{
156 return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
157}
158
159static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
160{
161 return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
162}
163
164static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
165 bool inherit)
166{
167 unsigned long ctrl;
168
169 ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
170 ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
171 if (inherit)
172 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
173 tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK;
174 tsk->thread.vstate_ctrl |= ctrl;
175}
176
177bool riscv_v_vstate_ctrl_user_allowed(void)
178{
179 return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
180}
181EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
182
183bool riscv_v_first_use_handler(struct pt_regs *regs)
184{
185 u32 __user *epc = (u32 __user *)regs->epc;
186 u32 insn = (u32)regs->badaddr;
187
188 if (!(has_vector() || has_xtheadvector()))
189 return false;
190
191 /* Do not handle if V is not supported, or disabled */
192 if (!riscv_v_vstate_ctrl_user_allowed())
193 return false;
194
195 /* If V has been enabled then it is not the first-use trap */
196 if (riscv_v_vstate_query(regs))
197 return false;
198
199 /* Get the instruction */
200 if (!insn) {
201 if (__get_user(insn, epc))
202 return false;
203 }
204
205 /* Filter out non-V instructions */
206 if (!insn_is_vector(insn_buf: insn))
207 return false;
208
209 /* Sanity check. datap should be null by the time of the first-use trap */
210 WARN_ON(current->thread.vstate.datap);
211
212 /*
213 * Now we sure that this is a V instruction. And it executes in the
214 * context where VS has been off. So, try to allocate the user's V
215 * context and resume execution.
216 */
217 if (riscv_v_thread_zalloc(cache: riscv_v_user_cachep, ctx: &current->thread.vstate)) {
218 force_sig(SIGBUS);
219 return true;
220 }
221 riscv_v_vstate_on(regs);
222 riscv_v_vstate_set_restore(current, regs);
223 return true;
224}
225
226void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
227{
228 bool inherit;
229 int cur, next;
230
231 if (!(has_vector() || has_xtheadvector()))
232 return;
233
234 next = riscv_v_ctrl_get_next(tsk);
235 if (!next) {
236 if (READ_ONCE(riscv_v_implicit_uacc))
237 cur = PR_RISCV_V_VSTATE_CTRL_ON;
238 else
239 cur = PR_RISCV_V_VSTATE_CTRL_OFF;
240 } else {
241 cur = next;
242 }
243 /* Clear next mask if inherit-bit is not set */
244 inherit = riscv_v_ctrl_test_inherit(tsk);
245 if (!inherit)
246 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
247
248 riscv_v_ctrl_set(tsk, cur, nxt: next, inherit);
249}
250
251long riscv_v_vstate_ctrl_get_current(void)
252{
253 if (!(has_vector() || has_xtheadvector()))
254 return -EINVAL;
255
256 return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
257}
258
259long riscv_v_vstate_ctrl_set_current(unsigned long arg)
260{
261 bool inherit;
262 int cur, next;
263
264 if (!(has_vector() || has_xtheadvector()))
265 return -EINVAL;
266
267 if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
268 return -EINVAL;
269
270 cur = VSTATE_CTRL_GET_CUR(arg);
271 switch (cur) {
272 case PR_RISCV_V_VSTATE_CTRL_OFF:
273 /* Do not allow user to turn off V if current is not off */
274 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
275 return -EPERM;
276
277 break;
278 case PR_RISCV_V_VSTATE_CTRL_ON:
279 break;
280 case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
281 cur = riscv_v_ctrl_get_cur(current);
282 break;
283 default:
284 return -EINVAL;
285 }
286
287 next = VSTATE_CTRL_GET_NEXT(arg);
288 inherit = VSTATE_CTRL_GET_INHERIT(arg);
289 switch (next) {
290 case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
291 case PR_RISCV_V_VSTATE_CTRL_OFF:
292 case PR_RISCV_V_VSTATE_CTRL_ON:
293 riscv_v_ctrl_set(current, cur, nxt: next, inherit);
294 return 0;
295 }
296
297 return -EINVAL;
298}
299
300#ifdef CONFIG_SYSCTL
301
302static const struct ctl_table riscv_v_default_vstate_table[] = {
303 {
304 .procname = "riscv_v_default_allow",
305 .data = &riscv_v_implicit_uacc,
306 .maxlen = sizeof(riscv_v_implicit_uacc),
307 .mode = 0644,
308 .proc_handler = proc_dobool,
309 },
310};
311
312static int __init riscv_v_sysctl_init(void)
313{
314 if (has_vector() || has_xtheadvector())
315 if (!register_sysctl("abi", riscv_v_default_vstate_table))
316 return -EINVAL;
317 return 0;
318}
319
320#else /* ! CONFIG_SYSCTL */
321static int __init riscv_v_sysctl_init(void) { return 0; }
322#endif /* ! CONFIG_SYSCTL */
323
324static int __init riscv_v_init(void)
325{
326 return riscv_v_sysctl_init();
327}
328core_initcall(riscv_v_init);
329

source code of linux/arch/riscv/kernel/vector.c