1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * FPU data structures:
4 */
5#ifndef _ASM_X86_FPU_TYPES_H
6#define _ASM_X86_FPU_TYPES_H
7
8#include <asm/page_types.h>
9
10/*
11 * The legacy x87 FPU state format, as saved by FSAVE and
12 * restored by the FRSTOR instructions:
13 */
14struct fregs_state {
15 u32 cwd; /* FPU Control Word */
16 u32 swd; /* FPU Status Word */
17 u32 twd; /* FPU Tag Word */
18 u32 fip; /* FPU IP Offset */
19 u32 fcs; /* FPU IP Selector */
20 u32 foo; /* FPU Operand Pointer Offset */
21 u32 fos; /* FPU Operand Pointer Selector */
22
23 /* 8*10 bytes for each FP-reg = 80 bytes: */
24 u32 st_space[20];
25
26 /* Software status information [not touched by FSAVE]: */
27 u32 status;
28};
29
30/*
31 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and
32 * restored by the FXRSTOR instructions. It's similar to the FSAVE
33 * format, but differs in some areas, plus has extensions at
34 * the end for the XMM registers.
35 */
36struct fxregs_state {
37 u16 cwd; /* Control Word */
38 u16 swd; /* Status Word */
39 u16 twd; /* Tag Word */
40 u16 fop; /* Last Instruction Opcode */
41 union {
42 struct {
43 u64 rip; /* Instruction Pointer */
44 u64 rdp; /* Data Pointer */
45 };
46 struct {
47 u32 fip; /* FPU IP Offset */
48 u32 fcs; /* FPU IP Selector */
49 u32 foo; /* FPU Operand Offset */
50 u32 fos; /* FPU Operand Selector */
51 };
52 };
53 u32 mxcsr; /* MXCSR Register State */
54 u32 mxcsr_mask; /* MXCSR Mask */
55
56 /* 8*16 bytes for each FP-reg = 128 bytes: */
57 u32 st_space[32];
58
59 /* 16*16 bytes for each XMM-reg = 256 bytes: */
60 u32 xmm_space[64];
61
62 u32 padding[12];
63
64 union {
65 u32 padding1[12];
66 u32 sw_reserved[12];
67 };
68
69} __attribute__((aligned(16)));
70
71/* Default value for fxregs_state.mxcsr: */
72#define MXCSR_DEFAULT 0x1f80
73
74/* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */
75#define MXCSR_AND_FLAGS_SIZE sizeof(u64)
76
77/*
78 * Software based FPU emulation state. This is arbitrary really,
79 * it matches the x87 format to make it easier to understand:
80 */
81struct swregs_state {
82 u32 cwd;
83 u32 swd;
84 u32 twd;
85 u32 fip;
86 u32 fcs;
87 u32 foo;
88 u32 fos;
89 /* 8*10 bytes for each FP-reg = 80 bytes: */
90 u32 st_space[20];
91 u8 ftop;
92 u8 changed;
93 u8 lookahead;
94 u8 no_update;
95 u8 rm;
96 u8 alimit;
97 struct math_emu_info *info;
98 u32 entry_eip;
99};
100
101/*
102 * List of XSAVE features Linux knows about:
103 */
104enum xfeature {
105 XFEATURE_FP,
106 XFEATURE_SSE,
107 /*
108 * Values above here are "legacy states".
109 * Those below are "extended states".
110 */
111 XFEATURE_YMM,
112 XFEATURE_BNDREGS,
113 XFEATURE_BNDCSR,
114 XFEATURE_OPMASK,
115 XFEATURE_ZMM_Hi256,
116 XFEATURE_Hi16_ZMM,
117 XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
118 XFEATURE_PKRU,
119 XFEATURE_PASID,
120 XFEATURE_CET_USER,
121 XFEATURE_CET_KERNEL,
122 XFEATURE_RSRVD_COMP_13,
123 XFEATURE_RSRVD_COMP_14,
124 XFEATURE_LBR,
125 XFEATURE_RSRVD_COMP_16,
126 XFEATURE_XTILE_CFG,
127 XFEATURE_XTILE_DATA,
128 XFEATURE_APX,
129
130 XFEATURE_MAX,
131};
132
133#define XFEATURE_MASK_FP (1 << XFEATURE_FP)
134#define XFEATURE_MASK_SSE (1 << XFEATURE_SSE)
135#define XFEATURE_MASK_YMM (1 << XFEATURE_YMM)
136#define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS)
137#define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR)
138#define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK)
139#define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256)
140#define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM)
141#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
142#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU)
143#define XFEATURE_MASK_PASID (1 << XFEATURE_PASID)
144#define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER)
145#define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL)
146#define XFEATURE_MASK_LBR (1 << XFEATURE_LBR)
147#define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG)
148#define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA)
149#define XFEATURE_MASK_APX (1 << XFEATURE_APX)
150
151#define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE)
152#define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \
153 | XFEATURE_MASK_ZMM_Hi256 \
154 | XFEATURE_MASK_Hi16_ZMM)
155
156#ifdef CONFIG_X86_64
157# define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \
158 | XFEATURE_MASK_XTILE_CFG)
159#else
160# define XFEATURE_MASK_XTILE (0)
161#endif
162
163#define FIRST_EXTENDED_XFEATURE XFEATURE_YMM
164
165struct reg_128_bit {
166 u8 regbytes[128/8];
167};
168struct reg_256_bit {
169 u8 regbytes[256/8];
170};
171struct reg_512_bit {
172 u8 regbytes[512/8];
173};
174struct reg_1024_byte {
175 u8 regbytes[1024];
176};
177
178/*
179 * State component 2:
180 *
181 * There are 16x 256-bit AVX registers named YMM0-YMM15.
182 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15)
183 * and are stored in 'struct fxregs_state::xmm_space[]' in the
184 * "legacy" area.
185 *
186 * The high 128 bits are stored here.
187 */
188struct ymmh_struct {
189 struct reg_128_bit hi_ymm[16];
190} __packed;
191
192/* Intel MPX support: */
193
194struct mpx_bndreg {
195 u64 lower_bound;
196 u64 upper_bound;
197} __packed;
198/*
199 * State component 3 is used for the 4 128-bit bounds registers
200 */
201struct mpx_bndreg_state {
202 struct mpx_bndreg bndreg[4];
203} __packed;
204
205/*
206 * State component 4 is used for the 64-bit user-mode MPX
207 * configuration register BNDCFGU and the 64-bit MPX status
208 * register BNDSTATUS. We call the pair "BNDCSR".
209 */
210struct mpx_bndcsr {
211 u64 bndcfgu;
212 u64 bndstatus;
213} __packed;
214
215/*
216 * The BNDCSR state is padded out to be 64-bytes in size.
217 */
218struct mpx_bndcsr_state {
219 union {
220 struct mpx_bndcsr bndcsr;
221 u8 pad_to_64_bytes[64];
222 };
223} __packed;
224
225/* AVX-512 Components: */
226
227/*
228 * State component 5 is used for the 8 64-bit opmask registers
229 * k0-k7 (opmask state).
230 */
231struct avx_512_opmask_state {
232 u64 opmask_reg[8];
233} __packed;
234
235/*
236 * State component 6 is used for the upper 256 bits of the
237 * registers ZMM0-ZMM15. These 16 256-bit values are denoted
238 * ZMM0_H-ZMM15_H (ZMM_Hi256 state).
239 */
240struct avx_512_zmm_uppers_state {
241 struct reg_256_bit zmm_upper[16];
242} __packed;
243
244/*
245 * State component 7 is used for the 16 512-bit registers
246 * ZMM16-ZMM31 (Hi16_ZMM state).
247 */
248struct avx_512_hi16_state {
249 struct reg_512_bit hi16_zmm[16];
250} __packed;
251
252/*
253 * State component 9: 32-bit PKRU register. The state is
254 * 8 bytes long but only 4 bytes is used currently.
255 */
256struct pkru_state {
257 u32 pkru;
258 u32 pad;
259} __packed;
260
261/*
262 * State component 11 is Control-flow Enforcement user states
263 */
264struct cet_user_state {
265 /* user control-flow settings */
266 u64 user_cet;
267 /* user shadow stack pointer */
268 u64 user_ssp;
269};
270
271/*
272 * State component 12 is Control-flow Enforcement supervisor states.
273 * This state includes SSP pointers for privilege levels 0 through 2.
274 */
275struct cet_supervisor_state {
276 u64 pl0_ssp;
277 u64 pl1_ssp;
278 u64 pl2_ssp;
279} __packed;
280
281/*
282 * State component 15: Architectural LBR configuration state.
283 * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
284 */
285
286struct lbr_entry {
287 u64 from;
288 u64 to;
289 u64 info;
290};
291
292struct arch_lbr_state {
293 u64 lbr_ctl;
294 u64 lbr_depth;
295 u64 ler_from;
296 u64 ler_to;
297 u64 ler_info;
298 struct lbr_entry entries[];
299};
300
301/*
302 * State component 17: 64-byte tile configuration register.
303 */
304struct xtile_cfg {
305 u64 tcfg[8];
306} __packed;
307
308/*
309 * State component 18: 1KB tile data register.
310 * Each register represents 16 64-byte rows of the matrix
311 * data. But the number of registers depends on the actual
312 * implementation.
313 */
314struct xtile_data {
315 struct reg_1024_byte tmm;
316} __packed;
317
318/*
319 * State component 19: 8B extended general purpose register.
320 */
321struct apx_state {
322 u64 egpr[16];
323} __packed;
324
325/*
326 * State component 10 is supervisor state used for context-switching the
327 * PASID state.
328 */
329struct ia32_pasid_state {
330 u64 pasid;
331} __packed;
332
333struct xstate_header {
334 u64 xfeatures;
335 u64 xcomp_bv;
336 u64 reserved[6];
337} __attribute__((packed));
338
339/*
340 * xstate_header.xcomp_bv[63] indicates that the extended_state_area
341 * is in compacted format.
342 */
343#define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63)
344
345/*
346 * This is our most modern FPU state format, as saved by the XSAVE
347 * and restored by the XRSTOR instructions.
348 *
349 * It consists of a legacy fxregs portion, an xstate header and
350 * subsequent areas as defined by the xstate header. Not all CPUs
351 * support all the extensions, so the size of the extended area
352 * can vary quite a bit between CPUs.
353 */
354struct xregs_state {
355 struct fxregs_state i387;
356 struct xstate_header header;
357 u8 extended_state_area[];
358} __attribute__ ((packed, aligned (64)));
359
360/*
361 * This is a union of all the possible FPU state formats
362 * put together, so that we can pick the right one runtime.
363 *
364 * The size of the structure is determined by the largest
365 * member - which is the xsave area. The padding is there
366 * to ensure that statically-allocated task_structs (just
367 * the init_task today) have enough space.
368 */
369union fpregs_state {
370 struct fregs_state fsave;
371 struct fxregs_state fxsave;
372 struct swregs_state soft;
373 struct xregs_state xsave;
374 u8 __padding[PAGE_SIZE];
375};
376
377struct fpstate {
378 /* @kernel_size: The size of the kernel register image */
379 unsigned int size;
380
381 /* @user_size: The size in non-compacted UABI format */
382 unsigned int user_size;
383
384 /* @xfeatures: xfeatures for which the storage is sized */
385 u64 xfeatures;
386
387 /* @user_xfeatures: xfeatures valid in UABI buffers */
388 u64 user_xfeatures;
389
390 /* @xfd: xfeatures disabled to trap userspace use. */
391 u64 xfd;
392
393 /* @is_valloc: Indicator for dynamically allocated state */
394 unsigned int is_valloc : 1;
395
396 /* @is_guest: Indicator for guest state (KVM) */
397 unsigned int is_guest : 1;
398
399 /*
400 * @is_confidential: Indicator for KVM confidential mode.
401 * The FPU registers are restored by the
402 * vmentry firmware from encrypted guest
403 * memory. On vmexit the FPU registers are
404 * saved by firmware to encrypted guest memory
405 * and the registers are scrubbed before
406 * returning to the host. So there is no
407 * content which is worth saving and restoring.
408 * The fpstate has to be there so that
409 * preemption and softirq FPU usage works
410 * without special casing.
411 */
412 unsigned int is_confidential : 1;
413
414 /* @in_use: State is in use */
415 unsigned int in_use : 1;
416
417 /* @regs: The register state union for all supported formats */
418 union fpregs_state regs;
419
420 /* @regs is dynamically sized! Don't add anything after @regs! */
421} __aligned(64);
422
423#define FPU_GUEST_PERM_LOCKED BIT_ULL(63)
424
425struct fpu_state_perm {
426 /*
427 * @__state_perm:
428 *
429 * This bitmap indicates the permission for state components
430 * available to a thread group, including both user and supervisor
431 * components and software-defined bits like FPU_GUEST_PERM_LOCKED.
432 * The permission prctl() sets the enabled state bits in
433 * thread_group_leader()->thread.fpu.
434 *
435 * All run time operations use the per thread information in the
436 * currently active fpu.fpstate which contains the xfeature masks
437 * and sizes for kernel and user space.
438 *
439 * This master permission field is only to be used when
440 * task.fpu.fpstate based checks fail to validate whether the task
441 * is allowed to expand its xfeatures set which requires to
442 * allocate a larger sized fpstate buffer.
443 *
444 * Do not access this field directly. Use the provided helper
445 * function. Unlocked access is possible for quick checks.
446 */
447 u64 __state_perm;
448
449 /*
450 * @__state_size:
451 *
452 * The size required for @__state_perm. Only valid to access
453 * with sighand locked.
454 */
455 unsigned int __state_size;
456
457 /*
458 * @__user_state_size:
459 *
460 * The size required for @__state_perm user part. Only valid to
461 * access with sighand locked.
462 */
463 unsigned int __user_state_size;
464};
465
466/*
467 * Highest level per task FPU state data structure that
468 * contains the FPU register state plus various FPU
469 * state fields:
470 */
471struct fpu {
472 /*
473 * @last_cpu:
474 *
475 * Records the last CPU on which this context was loaded into
476 * FPU registers. (In the lazy-restore case we might be
477 * able to reuse FPU registers across multiple context switches
478 * this way, if no intermediate task used the FPU.)
479 *
480 * A value of -1 is used to indicate that the FPU state in context
481 * memory is newer than the FPU state in registers, and that the
482 * FPU state should be reloaded next time the task is run.
483 */
484 unsigned int last_cpu;
485
486 /*
487 * @avx512_timestamp:
488 *
489 * Records the timestamp of AVX512 use during last context switch.
490 */
491 unsigned long avx512_timestamp;
492
493 /*
494 * @fpstate:
495 *
496 * Pointer to the active struct fpstate. Initialized to
497 * point at @__fpstate below.
498 */
499 struct fpstate *fpstate;
500
501 /*
502 * @__task_fpstate:
503 *
504 * Pointer to an inactive struct fpstate. Initialized to NULL. Is
505 * used only for KVM support to swap out the regular task fpstate.
506 */
507 struct fpstate *__task_fpstate;
508
509 /*
510 * @perm:
511 *
512 * Permission related information
513 */
514 struct fpu_state_perm perm;
515
516 /*
517 * @guest_perm:
518 *
519 * Permission related information for guest pseudo FPUs
520 */
521 struct fpu_state_perm guest_perm;
522
523 /*
524 * @__fpstate:
525 *
526 * Initial in-memory storage for FPU registers which are saved in
527 * context switch and when the kernel uses the FPU. The registers
528 * are restored from this storage on return to user space if they
529 * are not longer containing the tasks FPU register state.
530 */
531 struct fpstate __fpstate;
532 /*
533 * WARNING: '__fpstate' is dynamically-sized. Do not put
534 * anything after it here.
535 */
536};
537
538/*
539 * Guest pseudo FPU container
540 */
541struct fpu_guest {
542 /*
543 * @xfeatures: xfeature bitmap of features which are
544 * currently enabled for the guest vCPU.
545 */
546 u64 xfeatures;
547
548 /*
549 * @xfd_err: Save the guest value.
550 */
551 u64 xfd_err;
552
553 /*
554 * @uabi_size: Size required for save/restore
555 */
556 unsigned int uabi_size;
557
558 /*
559 * @fpstate: Pointer to the allocated guest fpstate
560 */
561 struct fpstate *fpstate;
562};
563
564/*
565 * FPU state configuration data for fpu_guest.
566 * Initialized at boot time. Read only after init.
567 */
568struct vcpu_fpu_config {
569 /*
570 * @size:
571 *
572 * The default size of the register state buffer in guest FPUs.
573 * Includes all supported features except independent managed
574 * features and features which have to be requested by user space
575 * before usage.
576 */
577 unsigned int size;
578
579 /*
580 * @features:
581 *
582 * The default supported features bitmap in guest FPUs. Does not
583 * include independent managed features and features which have to
584 * be requested by user space before usage.
585 */
586 u64 features;
587};
588
589/*
590 * FPU state configuration data. Initialized at boot time. Read only after init.
591 */
592struct fpu_state_config {
593 /*
594 * @max_size:
595 *
596 * The maximum size of the register state buffer. Includes all
597 * supported features except independent managed features.
598 */
599 unsigned int max_size;
600
601 /*
602 * @default_size:
603 *
604 * The default size of the register state buffer. Includes all
605 * supported features except independent managed features,
606 * guest-only features and features which have to be requested by
607 * user space before usage.
608 */
609 unsigned int default_size;
610
611 /*
612 * @max_features:
613 *
614 * The maximum supported features bitmap. Does not include
615 * independent managed features.
616 */
617 u64 max_features;
618
619 /*
620 * @default_features:
621 *
622 * The default supported features bitmap. Does not include
623 * independent managed features, guest-only features and features
624 * which have to be requested by user space before usage.
625 */
626 u64 default_features;
627 /*
628 * @legacy_features:
629 *
630 * Features which can be reported back to user space
631 * even without XSAVE support, i.e. legacy features FP + SSE
632 */
633 u64 legacy_features;
634 /*
635 * @independent_features:
636 *
637 * Features that are supported by XSAVES, but not managed as part of
638 * the FPU core, such as LBR
639 */
640 u64 independent_features;
641};
642
643/* FPU state configuration information */
644extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg;
645extern struct vcpu_fpu_config guest_default_cfg;
646
647#endif /* _ASM_X86_FPU_TYPES_H */
648

source code of linux/arch/x86/include/asm/fpu/types.h