1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 * Leo Duran <leo.duran@amd.com>
6 */
7
8#ifndef _ASM_X86_AMD_IOMMU_TYPES_H
9#define _ASM_X86_AMD_IOMMU_TYPES_H
10
11#include <linux/bitfield.h>
12#include <linux/iommu.h>
13#include <linux/types.h>
14#include <linux/mmu_notifier.h>
15#include <linux/mutex.h>
16#include <linux/msi.h>
17#include <linux/list.h>
18#include <linux/spinlock.h>
19#include <linux/pci.h>
20#include <linux/irqreturn.h>
21#include <linux/generic_pt/iommu.h>
22
23/*
24 * Maximum number of IOMMUs supported
25 */
26#define MAX_IOMMUS 32
27
28/*
29 * some size calculation constants
30 */
31#define DEV_TABLE_ENTRY_SIZE 32
32
33/* Capability offsets used by the driver */
34#define MMIO_CAP_HDR_OFFSET 0x00
35#define MMIO_RANGE_OFFSET 0x0c
36#define MMIO_MISC_OFFSET 0x10
37
38/* Masks, shifts and macros to parse the device range capability */
39#define MMIO_RANGE_LD_MASK 0xff000000
40#define MMIO_RANGE_FD_MASK 0x00ff0000
41#define MMIO_RANGE_BUS_MASK 0x0000ff00
42#define MMIO_RANGE_LD_SHIFT 24
43#define MMIO_RANGE_FD_SHIFT 16
44#define MMIO_RANGE_BUS_SHIFT 8
45#define MMIO_GET_LD(x) (((x) & MMIO_RANGE_LD_MASK) >> MMIO_RANGE_LD_SHIFT)
46#define MMIO_GET_FD(x) (((x) & MMIO_RANGE_FD_MASK) >> MMIO_RANGE_FD_SHIFT)
47#define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
48#define MMIO_MSI_NUM(x) ((x) & 0x1f)
49
50/* Flag masks for the AMD IOMMU exclusion range */
51#define MMIO_EXCL_ENABLE_MASK 0x01ULL
52#define MMIO_EXCL_ALLOW_MASK 0x02ULL
53
54/* Used offsets into the MMIO space */
55#define MMIO_DEV_TABLE_OFFSET 0x0000
56#define MMIO_CMD_BUF_OFFSET 0x0008
57#define MMIO_EVT_BUF_OFFSET 0x0010
58#define MMIO_CONTROL_OFFSET 0x0018
59#define MMIO_EXCL_BASE_OFFSET 0x0020
60#define MMIO_EXCL_LIMIT_OFFSET 0x0028
61#define MMIO_EXT_FEATURES 0x0030
62#define MMIO_PPR_LOG_OFFSET 0x0038
63#define MMIO_GA_LOG_BASE_OFFSET 0x00e0
64#define MMIO_GA_LOG_TAIL_OFFSET 0x00e8
65#define MMIO_MSI_ADDR_LO_OFFSET 0x015C
66#define MMIO_MSI_ADDR_HI_OFFSET 0x0160
67#define MMIO_MSI_DATA_OFFSET 0x0164
68#define MMIO_INTCAPXT_EVT_OFFSET 0x0170
69#define MMIO_INTCAPXT_PPR_OFFSET 0x0178
70#define MMIO_INTCAPXT_GALOG_OFFSET 0x0180
71#define MMIO_EXT_FEATURES2 0x01A0
72#define MMIO_CMD_HEAD_OFFSET 0x2000
73#define MMIO_CMD_TAIL_OFFSET 0x2008
74#define MMIO_EVT_HEAD_OFFSET 0x2010
75#define MMIO_EVT_TAIL_OFFSET 0x2018
76#define MMIO_STATUS_OFFSET 0x2020
77#define MMIO_PPR_HEAD_OFFSET 0x2030
78#define MMIO_PPR_TAIL_OFFSET 0x2038
79#define MMIO_GA_HEAD_OFFSET 0x2040
80#define MMIO_GA_TAIL_OFFSET 0x2048
81#define MMIO_CNTR_CONF_OFFSET 0x4000
82#define MMIO_CNTR_REG_OFFSET 0x40000
83#define MMIO_REG_END_OFFSET 0x80000
84
85
86
87/* Extended Feature Bits */
88#define FEATURE_PREFETCH BIT_ULL(0)
89#define FEATURE_PPR BIT_ULL(1)
90#define FEATURE_X2APIC BIT_ULL(2)
91#define FEATURE_NX BIT_ULL(3)
92#define FEATURE_GT BIT_ULL(4)
93#define FEATURE_IA BIT_ULL(6)
94#define FEATURE_GA BIT_ULL(7)
95#define FEATURE_HE BIT_ULL(8)
96#define FEATURE_PC BIT_ULL(9)
97#define FEATURE_HATS GENMASK_ULL(11, 10)
98#define FEATURE_GATS GENMASK_ULL(13, 12)
99#define FEATURE_GLX GENMASK_ULL(15, 14)
100#define FEATURE_GAM_VAPIC BIT_ULL(21)
101#define FEATURE_PASMAX GENMASK_ULL(36, 32)
102#define FEATURE_GIOSUP BIT_ULL(48)
103#define FEATURE_HASUP BIT_ULL(49)
104#define FEATURE_EPHSUP BIT_ULL(50)
105#define FEATURE_HDSUP BIT_ULL(52)
106#define FEATURE_SNP BIT_ULL(63)
107
108
109/* Extended Feature 2 Bits */
110#define FEATURE_SEVSNPIO_SUP BIT_ULL(1)
111#define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5)
112#define FEATURE_SNPAVICSUP_GAM(x) \
113 (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1)
114#define FEATURE_HT_RANGE_IGNORE BIT_ULL(11)
115
116#define FEATURE_NUM_INT_REMAP_SUP GENMASK_ULL(9, 8)
117#define FEATURE_NUM_INT_REMAP_SUP_2K(x) \
118 (FIELD_GET(FEATURE_NUM_INT_REMAP_SUP, x) == 0x1)
119
120/* Note:
121 * The current driver only support 16-bit PASID.
122 * Currently, hardware only implement upto 16-bit PASID
123 * even though the spec says it could have upto 20 bits.
124 */
125#define PASID_MASK 0x0000ffff
126
127/* MMIO status bits */
128#define MMIO_STATUS_EVT_OVERFLOW_MASK BIT(0)
129#define MMIO_STATUS_EVT_INT_MASK BIT(1)
130#define MMIO_STATUS_COM_WAIT_INT_MASK BIT(2)
131#define MMIO_STATUS_EVT_RUN_MASK BIT(3)
132#define MMIO_STATUS_PPR_OVERFLOW_MASK BIT(5)
133#define MMIO_STATUS_PPR_INT_MASK BIT(6)
134#define MMIO_STATUS_PPR_RUN_MASK BIT(7)
135#define MMIO_STATUS_GALOG_RUN_MASK BIT(8)
136#define MMIO_STATUS_GALOG_OVERFLOW_MASK BIT(9)
137#define MMIO_STATUS_GALOG_INT_MASK BIT(10)
138
139/* event logging constants */
140#define EVENT_ENTRY_SIZE 0x10
141#define EVENT_TYPE_SHIFT 28
142#define EVENT_TYPE_MASK 0xf
143#define EVENT_TYPE_ILL_DEV 0x1
144#define EVENT_TYPE_IO_FAULT 0x2
145#define EVENT_TYPE_DEV_TAB_ERR 0x3
146#define EVENT_TYPE_PAGE_TAB_ERR 0x4
147#define EVENT_TYPE_ILL_CMD 0x5
148#define EVENT_TYPE_CMD_HARD_ERR 0x6
149#define EVENT_TYPE_IOTLB_INV_TO 0x7
150#define EVENT_TYPE_INV_DEV_REQ 0x8
151#define EVENT_TYPE_INV_PPR_REQ 0x9
152#define EVENT_TYPE_RMP_FAULT 0xd
153#define EVENT_TYPE_RMP_HW_ERR 0xe
154#define EVENT_DEVID_MASK 0xffff
155#define EVENT_DEVID_SHIFT 0
156#define EVENT_DOMID_MASK_LO 0xffff
157#define EVENT_DOMID_MASK_HI 0xf0000
158#define EVENT_FLAGS_MASK 0xfff
159#define EVENT_FLAGS_SHIFT 0x10
160#define EVENT_FLAG_RW 0x020
161#define EVENT_FLAG_I 0x008
162
163/* feature control bits */
164#define CONTROL_IOMMU_EN 0
165#define CONTROL_HT_TUN_EN 1
166#define CONTROL_EVT_LOG_EN 2
167#define CONTROL_EVT_INT_EN 3
168#define CONTROL_COMWAIT_EN 4
169#define CONTROL_INV_TIMEOUT 5
170#define CONTROL_PASSPW_EN 8
171#define CONTROL_RESPASSPW_EN 9
172#define CONTROL_COHERENT_EN 10
173#define CONTROL_ISOC_EN 11
174#define CONTROL_CMDBUF_EN 12
175#define CONTROL_PPRLOG_EN 13
176#define CONTROL_PPRINT_EN 14
177#define CONTROL_PPR_EN 15
178#define CONTROL_GT_EN 16
179#define CONTROL_GA_EN 17
180#define CONTROL_GAM_EN 25
181#define CONTROL_GALOG_EN 28
182#define CONTROL_GAINT_EN 29
183#define CONTROL_NUM_INT_REMAP_MODE 43
184#define CONTROL_NUM_INT_REMAP_MODE_MASK 0x03
185#define CONTROL_NUM_INT_REMAP_MODE_2K 0x01
186#define CONTROL_EPH_EN 45
187#define CONTROL_XT_EN 50
188#define CONTROL_INTCAPXT_EN 51
189#define CONTROL_IRTCACHEDIS 59
190#define CONTROL_SNPAVIC_EN 61
191
192#define CTRL_INV_TO_MASK 7
193#define CTRL_INV_TO_NONE 0
194#define CTRL_INV_TO_1MS 1
195#define CTRL_INV_TO_10MS 2
196#define CTRL_INV_TO_100MS 3
197#define CTRL_INV_TO_1S 4
198#define CTRL_INV_TO_10S 5
199#define CTRL_INV_TO_100S 6
200
201/* command specific defines */
202#define CMD_COMPL_WAIT 0x01
203#define CMD_INV_DEV_ENTRY 0x02
204#define CMD_INV_IOMMU_PAGES 0x03
205#define CMD_INV_IOTLB_PAGES 0x04
206#define CMD_INV_IRT 0x05
207#define CMD_COMPLETE_PPR 0x07
208#define CMD_INV_ALL 0x08
209
210#define CMD_COMPL_WAIT_STORE_MASK 0x01
211#define CMD_COMPL_WAIT_INT_MASK 0x02
212#define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01
213#define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02
214#define CMD_INV_IOMMU_PAGES_GN_MASK 0x04
215
216#define PPR_STATUS_MASK 0xf
217#define PPR_STATUS_SHIFT 12
218
219#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL
220
221/* macros and definitions for device table entries */
222#define DEV_ENTRY_VALID 0x00
223#define DEV_ENTRY_TRANSLATION 0x01
224#define DEV_ENTRY_HAD 0x07
225#define DEV_ENTRY_PPR 0x34
226#define DEV_ENTRY_IR 0x3d
227#define DEV_ENTRY_IW 0x3e
228#define DEV_ENTRY_NO_PAGE_FAULT 0x62
229#define DEV_ENTRY_EX 0x67
230#define DEV_ENTRY_SYSMGT1 0x68
231#define DEV_ENTRY_SYSMGT2 0x69
232#define DTE_DATA1_SYSMGT_MASK GENMASK_ULL(41, 40)
233
234#define DEV_ENTRY_IRQ_TBL_EN 0x80
235#define DEV_ENTRY_INIT_PASS 0xb8
236#define DEV_ENTRY_EINT_PASS 0xb9
237#define DEV_ENTRY_NMI_PASS 0xba
238#define DEV_ENTRY_LINT0_PASS 0xbe
239#define DEV_ENTRY_LINT1_PASS 0xbf
240#define DEV_ENTRY_MODE_MASK 0x07
241#define DEV_ENTRY_MODE_SHIFT 0x09
242
243#define MAX_DEV_TABLE_ENTRIES 0xffff
244
245/* constants to configure the command buffer */
246#define CMD_BUFFER_SIZE 8192
247#define CMD_BUFFER_UNINITIALIZED 1
248#define CMD_BUFFER_ENTRIES 512
249#define MMIO_CMD_SIZE_SHIFT 56
250#define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT)
251#define MMIO_CMD_HEAD_MASK GENMASK_ULL(18, 4) /* Command buffer head ptr field [18:4] */
252#define MMIO_CMD_BUFFER_HEAD(x) FIELD_GET(MMIO_CMD_HEAD_MASK, (x))
253#define MMIO_CMD_TAIL_MASK GENMASK_ULL(18, 4) /* Command buffer tail ptr field [18:4] */
254#define MMIO_CMD_BUFFER_TAIL(x) FIELD_GET(MMIO_CMD_TAIL_MASK, (x))
255
256/* constants for event buffer handling */
257#define EVT_BUFFER_SIZE 8192 /* 512 entries */
258#define EVT_LEN_MASK (0x9ULL << 56)
259
260/* Constants for PPR Log handling */
261#define PPR_LOG_ENTRIES 512
262#define PPR_LOG_SIZE_SHIFT 56
263#define PPR_LOG_SIZE_512 (0x9ULL << PPR_LOG_SIZE_SHIFT)
264#define PPR_ENTRY_SIZE 16
265#define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES)
266
267/* PAGE_SERVICE_REQUEST PPR Log Buffer Entry flags */
268#define PPR_FLAG_EXEC 0x002 /* Execute permission requested */
269#define PPR_FLAG_READ 0x004 /* Read permission requested */
270#define PPR_FLAG_WRITE 0x020 /* Write permission requested */
271#define PPR_FLAG_US 0x040 /* 1: User, 0: Supervisor */
272#define PPR_FLAG_RVSD 0x080 /* Reserved bit not zero */
273#define PPR_FLAG_GN 0x100 /* GVA and PASID is valid */
274
275#define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
276#define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
277#define PPR_DEVID(x) ((x) & 0xffffULL)
278#define PPR_TAG(x) (((x) >> 32) & 0x3ffULL)
279#define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
280#define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
281#define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
282
283#define PPR_REQ_FAULT 0x01
284
285/* Constants for GA Log handling */
286#define GA_LOG_ENTRIES 512
287#define GA_LOG_SIZE_SHIFT 56
288#define GA_LOG_SIZE_512 (0x8ULL << GA_LOG_SIZE_SHIFT)
289#define GA_ENTRY_SIZE 8
290#define GA_LOG_SIZE (GA_ENTRY_SIZE * GA_LOG_ENTRIES)
291
292#define GA_TAG(x) (u32)(x & 0xffffffffULL)
293#define GA_DEVID(x) (u16)(((x) >> 32) & 0xffffULL)
294#define GA_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
295
296#define GA_GUEST_NR 0x1
297
298#define IOMMU_IN_ADDR_BIT_SIZE 52
299#define IOMMU_OUT_ADDR_BIT_SIZE 52
300
301/*
302 * This bitmap is used to advertise the page sizes our hardware support
303 * to the IOMMU core, which will then use this information to split
304 * physically contiguous memory regions it is mapping into page sizes
305 * that we support.
306 *
307 * 512GB Pages are not supported due to a hardware bug
308 * Page sizes >= the 52 bit max physical address of the CPU are not supported.
309 */
310#define AMD_IOMMU_PGSIZES (GENMASK_ULL(51, 12) ^ SZ_512G)
311
312/* Special mode where page-sizes are limited to 4 KiB */
313#define AMD_IOMMU_PGSIZES_4K (PAGE_SIZE)
314
315/* 4K, 2MB, 1G page sizes are supported */
316#define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 21) | (1ULL << 30))
317
318/* Bit value definition for dte irq remapping fields*/
319#define DTE_IRQ_PHYS_ADDR_MASK GENMASK_ULL(51, 6)
320#define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60)
321#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
322#define DTE_IRQ_REMAP_ENABLE 1ULL
323
324#define DTE_INTTAB_ALIGNMENT 128
325#define DTE_INTTABLEN_MASK (0xfULL << 1)
326#define DTE_INTTABLEN_VALUE_512 9ULL
327#define DTE_INTTABLEN_512 (DTE_INTTABLEN_VALUE_512 << 1)
328#define MAX_IRQS_PER_TABLE_512 BIT(DTE_INTTABLEN_VALUE_512)
329#define DTE_INTTABLEN_VALUE_2K 11ULL
330#define DTE_INTTABLEN_2K (DTE_INTTABLEN_VALUE_2K << 1)
331#define MAX_IRQS_PER_TABLE_2K BIT(DTE_INTTABLEN_VALUE_2K)
332
333#define PAGE_MODE_NONE 0x00
334#define PAGE_MODE_1_LEVEL 0x01
335#define PAGE_MODE_2_LEVEL 0x02
336#define PAGE_MODE_3_LEVEL 0x03
337#define PAGE_MODE_4_LEVEL 0x04
338#define PAGE_MODE_5_LEVEL 0x05
339#define PAGE_MODE_6_LEVEL 0x06
340#define PAGE_MODE_7_LEVEL 0x07
341
342#define GUEST_PGTABLE_4_LEVEL 0x00
343#define GUEST_PGTABLE_5_LEVEL 0x01
344
345#define PM_ADDR_MASK 0x000ffffffffff000ULL
346
347/*
348 * Bit value definition for DTE fields
349 */
350#define DTE_FLAG_V BIT_ULL(0)
351#define DTE_FLAG_TV BIT_ULL(1)
352#define DTE_FLAG_HAD (3ULL << 7)
353#define DTE_FLAG_GIOV BIT_ULL(54)
354#define DTE_FLAG_GV BIT_ULL(55)
355#define DTE_GLX GENMASK_ULL(57, 56)
356#define DTE_FLAG_IR BIT_ULL(61)
357#define DTE_FLAG_IW BIT_ULL(62)
358
359#define DTE_FLAG_IOTLB BIT_ULL(32)
360#define DTE_FLAG_MASK (0x3ffULL << 32)
361#define DEV_DOMID_MASK 0xffffULL
362
363#define DTE_GCR3_14_12 GENMASK_ULL(60, 58)
364#define DTE_GCR3_30_15 GENMASK_ULL(31, 16)
365#define DTE_GCR3_51_31 GENMASK_ULL(63, 43)
366
367#define DTE_GPT_LEVEL_SHIFT 54
368#define DTE_GPT_LEVEL_MASK GENMASK_ULL(55, 54)
369
370#define GCR3_VALID 0x01ULL
371
372/* DTE[128:179] | DTE[184:191] */
373#define DTE_DATA2_INTR_MASK ~GENMASK_ULL(55, 52)
374
375#define IOMMU_PROT_MASK 0x03
376#define IOMMU_PROT_IR 0x01
377#define IOMMU_PROT_IW 0x02
378
379#define IOMMU_UNITY_MAP_FLAG_EXCL_RANGE (1 << 2)
380
381/* IOMMU capabilities */
382#define IOMMU_CAP_IOTLB 24
383#define IOMMU_CAP_NPCACHE 26
384#define IOMMU_CAP_EFR 27
385
386/* IOMMU IVINFO */
387#define IOMMU_IVINFO_OFFSET 36
388#define IOMMU_IVINFO_EFRSUP BIT(0)
389#define IOMMU_IVINFO_DMA_REMAP BIT(1)
390
391/* IOMMU Feature Reporting Field (for IVHD type 10h */
392#define IOMMU_FEAT_GASUP_SHIFT 6
393
394/* IOMMU HATDIS for IVHD type 11h and 40h */
395#define IOMMU_IVHD_ATTR_HATDIS_SHIFT 0
396
397/* IOMMU Extended Feature Register (EFR) */
398#define IOMMU_EFR_XTSUP_SHIFT 2
399#define IOMMU_EFR_GASUP_SHIFT 7
400#define IOMMU_EFR_MSICAPMMIOSUP_SHIFT 46
401
402#define MAX_DOMAIN_ID 65536
403
404/* Timeout stuff */
405#define LOOP_TIMEOUT 100000
406#define MMIO_STATUS_TIMEOUT 2000000
407
408extern bool amd_iommu_dump;
409#define DUMP_printk(format, arg...) \
410 do { \
411 if (amd_iommu_dump) \
412 pr_info(format, ## arg); \
413 } while(0);
414
415/* global flag if IOMMUs cache non-present entries */
416extern bool amd_iommu_np_cache;
417/* Only true if all IOMMUs support device IOTLBs */
418extern bool amd_iommu_iotlb_sup;
419
420struct irq_remap_table {
421 raw_spinlock_t lock;
422 unsigned min_index;
423 u32 *table;
424};
425
426/* Interrupt remapping feature used? */
427extern bool amd_iommu_irq_remap;
428
429extern const struct iommu_ops amd_iommu_ops;
430
431/* IVRS indicates that pre-boot remapping was enabled */
432extern bool amdr_ivrs_remap_support;
433
434#define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff)
435#define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff)
436#define PCI_SEG_DEVID_TO_SBDF(seg, devid) ((((u32)(seg) & 0xffff) << 16) | \
437 ((devid) & 0xffff))
438
439/* Make iterating over all pci segment easier */
440#define for_each_pci_segment(pci_seg) \
441 list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list)
442#define for_each_pci_segment_safe(pci_seg, next) \
443 list_for_each_entry_safe((pci_seg), (next), &amd_iommu_pci_seg_list, list)
444/*
445 * Make iterating over all IOMMUs easier
446 */
447#define for_each_iommu(iommu) \
448 list_for_each_entry((iommu), &amd_iommu_list, list)
449#define for_each_iommu_safe(iommu, next) \
450 list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
451/* Making iterating over protection_domain->dev_data_list easier */
452#define for_each_pdom_dev_data(pdom_dev_data, pdom) \
453 list_for_each_entry(pdom_dev_data, &pdom->dev_data_list, list)
454#define for_each_pdom_dev_data_safe(pdom_dev_data, next, pdom) \
455 list_for_each_entry_safe((pdom_dev_data), (next), &pdom->dev_data_list, list)
456
457#define for_each_ivhd_dte_flags(entry) \
458 list_for_each_entry((entry), &amd_ivhd_dev_flags_list, list)
459
460struct amd_iommu;
461struct iommu_domain;
462struct irq_domain;
463struct amd_irte_ops;
464
465#define AMD_IOMMU_FLAG_TRANS_PRE_ENABLED (1 << 0)
466
467struct gcr3_tbl_info {
468 u64 *gcr3_tbl; /* Guest CR3 table */
469 int glx; /* Number of levels for GCR3 table */
470 u32 pasid_cnt; /* Track attached PASIDs */
471 u16 domid; /* Per device domain ID */
472};
473
474enum protection_domain_mode {
475 PD_MODE_NONE,
476 PD_MODE_V1,
477 PD_MODE_V2,
478};
479
480/* Track dev_data/PASID list for the protection domain */
481struct pdom_dev_data {
482 /* Points to attached device data */
483 struct iommu_dev_data *dev_data;
484 /* PASID attached to the protection domain */
485 ioasid_t pasid;
486 /* For protection_domain->dev_data_list */
487 struct list_head list;
488};
489
490/* Keeps track of the IOMMUs attached to protection domain */
491struct pdom_iommu_info {
492 struct amd_iommu *iommu; /* IOMMUs attach to protection domain */
493 u32 refcnt; /* Count of attached dev/pasid per domain/IOMMU */
494};
495
496/*
497 * This structure contains generic data for IOMMU protection domains
498 * independent of their use.
499 */
500struct protection_domain {
501 union {
502 struct iommu_domain domain;
503 struct pt_iommu iommu;
504 struct pt_iommu_amdv1 amdv1;
505 struct pt_iommu_x86_64 amdv2;
506 };
507 struct list_head dev_list; /* List of all devices in this domain */
508 spinlock_t lock; /* mostly used to lock the page table*/
509 u16 id; /* the domain id written to the device table */
510 enum protection_domain_mode pd_mode; /* Track page table type */
511 bool dirty_tracking; /* dirty tracking is enabled in the domain */
512 struct xarray iommu_array; /* per-IOMMU reference count */
513
514 struct mmu_notifier mn; /* mmu notifier for the SVA domain */
515 struct list_head dev_data_list; /* List of pdom_dev_data */
516};
517PT_IOMMU_CHECK_DOMAIN(struct protection_domain, iommu, domain);
518PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv1.iommu, domain);
519PT_IOMMU_CHECK_DOMAIN(struct protection_domain, amdv2.iommu, domain);
520
521/*
522 * This structure contains information about one PCI segment in the system.
523 */
524struct amd_iommu_pci_seg {
525 /* List with all PCI segments in the system */
526 struct list_head list;
527
528 /* List of all available dev_data structures */
529 struct llist_head dev_data_list;
530
531 /* PCI segment number */
532 u16 id;
533
534 /* Largest PCI device id we expect translation requests for */
535 u16 last_bdf;
536
537 /* Size of the device table */
538 u32 dev_table_size;
539
540 /*
541 * device table virtual address
542 *
543 * Pointer to the per PCI segment device table.
544 * It is indexed by the PCI device id or the HT unit id and contains
545 * information about the domain the device belongs to as well as the
546 * page table root pointer.
547 */
548 struct dev_table_entry *dev_table;
549
550 /*
551 * The rlookup iommu table is used to find the IOMMU which is
552 * responsible for a specific device. It is indexed by the PCI
553 * device id.
554 */
555 struct amd_iommu **rlookup_table;
556
557 /*
558 * This table is used to find the irq remapping table for a given
559 * device id quickly.
560 */
561 struct irq_remap_table **irq_lookup_table;
562
563 /*
564 * Pointer to a device table which the content of old device table
565 * will be copied to. It's only be used in kdump kernel.
566 */
567 struct dev_table_entry *old_dev_tbl_cpy;
568
569 /*
570 * The alias table is a driver specific data structure which contains the
571 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
572 * More than one device can share the same requestor id.
573 */
574 u16 *alias_table;
575
576 /*
577 * A list of required unity mappings we find in ACPI. It is not locked
578 * because as runtime it is only read. It is created at ACPI table
579 * parsing time.
580 */
581 struct list_head unity_map;
582};
583
584/*
585 * Structure where we save information about one hardware AMD IOMMU in the
586 * system.
587 */
588struct amd_iommu {
589 struct list_head list;
590
591 /* Index within the IOMMU array */
592 int index;
593
594 /* locks the accesses to the hardware */
595 raw_spinlock_t lock;
596
597 /* Pointer to PCI device of this IOMMU */
598 struct pci_dev *dev;
599
600 /* Cache pdev to root device for resume quirks */
601 struct pci_dev *root_pdev;
602
603 /* physical address of MMIO space */
604 u64 mmio_phys;
605
606 /* physical end address of MMIO space */
607 u64 mmio_phys_end;
608
609 /* virtual address of MMIO space */
610 u8 __iomem *mmio_base;
611
612 /* capabilities of that IOMMU read from ACPI */
613 u32 cap;
614
615 /* flags read from acpi table */
616 u8 acpi_flags;
617
618 /* Extended features */
619 u64 features;
620
621 /* Extended features 2 */
622 u64 features2;
623
624 /* PCI device id of the IOMMU device */
625 u16 devid;
626
627 /*
628 * Capability pointer. There could be more than one IOMMU per PCI
629 * device function if there are more than one AMD IOMMU capability
630 * pointers.
631 */
632 u16 cap_ptr;
633
634 /* pci domain of this IOMMU */
635 struct amd_iommu_pci_seg *pci_seg;
636
637 /* start of exclusion range of that IOMMU */
638 u64 exclusion_start;
639 /* length of exclusion range of that IOMMU */
640 u64 exclusion_length;
641
642 /* command buffer virtual address */
643 u8 *cmd_buf;
644 u32 cmd_buf_head;
645 u32 cmd_buf_tail;
646
647 /* event buffer virtual address */
648 u8 *evt_buf;
649
650 /* Name for event log interrupt */
651 unsigned char evt_irq_name[16];
652
653 /* Base of the PPR log, if present */
654 u8 *ppr_log;
655
656 /* Name for PPR log interrupt */
657 unsigned char ppr_irq_name[16];
658
659 /* Base of the GA log, if present */
660 u8 *ga_log;
661
662 /* Name for GA log interrupt */
663 unsigned char ga_irq_name[16];
664
665 /* Tail of the GA log, if present */
666 u8 *ga_log_tail;
667
668 /* true if interrupts for this IOMMU are already enabled */
669 bool int_enabled;
670
671 /* if one, we need to send a completion wait command */
672 bool need_sync;
673
674 /* true if disable irte caching */
675 bool irtcachedis_enabled;
676
677 /* Handle for IOMMU core code */
678 struct iommu_device iommu;
679
680 /*
681 * We can't rely on the BIOS to restore all values on reinit, so we
682 * need to stash them
683 */
684
685 /* The iommu BAR */
686 u32 stored_addr_lo;
687 u32 stored_addr_hi;
688
689 /*
690 * Each iommu has 6 l1s, each of which is documented as having 0x12
691 * registers
692 */
693 u32 stored_l1[6][0x12];
694
695 /* The l2 indirect registers */
696 u32 stored_l2[0x83];
697
698 /* The maximum PC banks and counters/bank (PCSup=1) */
699 u8 max_banks;
700 u8 max_counters;
701#ifdef CONFIG_IRQ_REMAP
702 struct irq_domain *ir_domain;
703
704 struct amd_irte_ops *irte_ops;
705#endif
706
707 u32 flags;
708 volatile u64 *cmd_sem;
709 atomic64_t cmd_sem_val;
710 /*
711 * Track physical address to directly use it in build_completion_wait()
712 * and avoid adding any special checks and handling for kdump.
713 */
714 u64 cmd_sem_paddr;
715
716#ifdef CONFIG_AMD_IOMMU_DEBUGFS
717 /* DebugFS Info */
718 struct dentry *debugfs;
719 int dbg_mmio_offset;
720 int dbg_cap_offset;
721#endif
722
723 /* IOPF support */
724 struct iopf_queue *iopf_queue;
725 unsigned char iopfq_name[32];
726};
727
728static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev)
729{
730 struct iommu_device *iommu = dev_to_iommu_device(dev);
731
732 return container_of(iommu, struct amd_iommu, iommu);
733}
734
735#define ACPIHID_UID_LEN 256
736#define ACPIHID_HID_LEN 9
737
738struct acpihid_map_entry {
739 struct list_head list;
740 u8 uid[ACPIHID_UID_LEN];
741 u8 hid[ACPIHID_HID_LEN];
742 u32 devid;
743 u32 root_devid;
744 bool cmd_line;
745 struct iommu_group *group;
746};
747
748struct devid_map {
749 struct list_head list;
750 u8 id;
751 u32 devid;
752 bool cmd_line;
753};
754
755#define AMD_IOMMU_DEVICE_FLAG_ATS_SUP 0x1 /* ATS feature supported */
756#define AMD_IOMMU_DEVICE_FLAG_PRI_SUP 0x2 /* PRI feature supported */
757#define AMD_IOMMU_DEVICE_FLAG_PASID_SUP 0x4 /* PASID context supported */
758/* Device may request execution on memory pages */
759#define AMD_IOMMU_DEVICE_FLAG_EXEC_SUP 0x8
760/* Device may request super-user privileges */
761#define AMD_IOMMU_DEVICE_FLAG_PRIV_SUP 0x10
762
763/*
764 * This struct contains device specific data for the IOMMU
765 */
766struct iommu_dev_data {
767 /*Protect against attach/detach races */
768 struct mutex mutex;
769 spinlock_t dte_lock; /* DTE lock for 256-bit access */
770
771 struct list_head list; /* For domain->dev_list */
772 struct llist_node dev_data_list; /* For global dev_data_list */
773 struct protection_domain *domain; /* Domain the device is bound to */
774 struct gcr3_tbl_info gcr3_info; /* Per-device GCR3 table */
775 struct device *dev;
776 u16 devid; /* PCI Device ID */
777
778 unsigned int max_irqs; /* Maximum IRQs supported by device */
779 u32 max_pasids; /* Max supported PASIDs */
780 u32 flags; /* Holds AMD_IOMMU_DEVICE_FLAG_<*> */
781 int ats_qdep;
782 u8 ats_enabled :1; /* ATS state */
783 u8 pri_enabled :1; /* PRI state */
784 u8 pasid_enabled:1; /* PASID state */
785 u8 pri_tlp :1; /* PASID TLB required for
786 PPR completions */
787 u8 ppr :1; /* Enable device PPR support */
788 bool use_vapic; /* Enable device to use vapic mode */
789 bool defer_attach;
790
791 struct ratelimit_state rs; /* Ratelimit IOPF messages */
792};
793
794/* Map HPET and IOAPIC ids to the devid used by the IOMMU */
795extern struct list_head ioapic_map;
796extern struct list_head hpet_map;
797extern struct list_head acpihid_map;
798
799/*
800 * List with all PCI segments in the system. This list is not locked because
801 * it is only written at driver initialization time
802 */
803extern struct list_head amd_iommu_pci_seg_list;
804
805/*
806 * List with all IOMMUs in the system. This list is not locked because it is
807 * only written and read at driver initialization or suspend time
808 */
809extern struct list_head amd_iommu_list;
810
811/*
812 * Structure defining one entry in the device table
813 */
814struct dev_table_entry {
815 union {
816 u64 data[4];
817 u128 data128[2];
818 };
819};
820
821/*
822 * Structure defining one entry in the command buffer
823 */
824struct iommu_cmd {
825 u32 data[4];
826};
827
828/*
829 * Structure to sture persistent DTE flags from IVHD
830 */
831struct ivhd_dte_flags {
832 struct list_head list;
833 u16 segid;
834 u16 devid_first;
835 u16 devid_last;
836 struct dev_table_entry dte;
837};
838
839/*
840 * One entry for unity mappings parsed out of the ACPI table.
841 */
842struct unity_map_entry {
843 struct list_head list;
844
845 /* starting device id this entry is used for (including) */
846 u16 devid_start;
847 /* end device id this entry is used for (including) */
848 u16 devid_end;
849
850 /* start address to unity map (including) */
851 u64 address_start;
852 /* end address to unity map (including) */
853 u64 address_end;
854
855 /* required protection */
856 int prot;
857};
858
859/*
860 * Data structures for device handling
861 */
862
863extern bool amd_iommu_force_isolation;
864
865/* Max levels of glxval supported */
866extern int amd_iommu_max_glx_val;
867
868/* IDA to track protection domain IDs */
869extern struct ida pdom_ids;
870
871/* Global EFR and EFR2 registers */
872extern u64 amd_iommu_efr;
873extern u64 amd_iommu_efr2;
874
875static inline int get_ioapic_devid(int id)
876{
877 struct devid_map *entry;
878
879 list_for_each_entry(entry, &ioapic_map, list) {
880 if (entry->id == id)
881 return entry->devid;
882 }
883
884 return -EINVAL;
885}
886
887static inline int get_hpet_devid(int id)
888{
889 struct devid_map *entry;
890
891 list_for_each_entry(entry, &hpet_map, list) {
892 if (entry->id == id)
893 return entry->devid;
894 }
895
896 return -EINVAL;
897}
898
899enum amd_iommu_intr_mode_type {
900 AMD_IOMMU_GUEST_IR_LEGACY,
901
902 /* This mode is not visible to users. It is used when
903 * we cannot fully enable vAPIC and fallback to only support
904 * legacy interrupt remapping via 128-bit IRTE.
905 */
906 AMD_IOMMU_GUEST_IR_LEGACY_GA,
907 AMD_IOMMU_GUEST_IR_VAPIC,
908};
909
910#define AMD_IOMMU_GUEST_IR_GA(x) (x == AMD_IOMMU_GUEST_IR_VAPIC || \
911 x == AMD_IOMMU_GUEST_IR_LEGACY_GA)
912
913#define AMD_IOMMU_GUEST_IR_VAPIC(x) (x == AMD_IOMMU_GUEST_IR_VAPIC)
914
915union irte {
916 u32 val;
917 struct {
918 u32 valid : 1,
919 no_fault : 1,
920 int_type : 3,
921 rq_eoi : 1,
922 dm : 1,
923 rsvd_1 : 1,
924 destination : 8,
925 vector : 8,
926 rsvd_2 : 8;
927 } fields;
928};
929
930#define APICID_TO_IRTE_DEST_LO(x) (x & 0xffffff)
931#define APICID_TO_IRTE_DEST_HI(x) ((x >> 24) & 0xff)
932
933union irte_ga_lo {
934 u64 val;
935
936 /* For int remapping */
937 struct {
938 u64 valid : 1,
939 no_fault : 1,
940 /* ------ */
941 int_type : 3,
942 rq_eoi : 1,
943 dm : 1,
944 /* ------ */
945 guest_mode : 1,
946 destination : 24,
947 ga_tag : 32;
948 } fields_remap;
949
950 /* For guest vAPIC */
951 struct {
952 u64 valid : 1,
953 no_fault : 1,
954 /* ------ */
955 ga_log_intr : 1,
956 rsvd1 : 3,
957 is_run : 1,
958 /* ------ */
959 guest_mode : 1,
960 destination : 24,
961 ga_tag : 32;
962 } fields_vapic;
963};
964
965union irte_ga_hi {
966 u64 val;
967 struct {
968 u64 vector : 8,
969 rsvd_1 : 4,
970 ga_root_ptr : 40,
971 rsvd_2 : 4,
972 destination : 8;
973 } fields;
974};
975
976struct irte_ga {
977 union {
978 struct {
979 union irte_ga_lo lo;
980 union irte_ga_hi hi;
981 };
982 u128 irte;
983 };
984};
985
986struct irq_2_irte {
987 u16 devid; /* Device ID for IRTE table */
988 u16 index; /* Index into IRTE table*/
989};
990
991struct amd_ir_data {
992 struct amd_iommu *iommu;
993 struct irq_2_irte irq_2_irte;
994 struct msi_msg msi_entry;
995 void *entry; /* Pointer to union irte or struct irte_ga */
996
997 /**
998 * Store information for activate/de-activate
999 * Guest virtual APIC mode during runtime.
1000 */
1001 struct irq_cfg *cfg;
1002 int ga_vector;
1003 u64 ga_root_ptr;
1004 u32 ga_tag;
1005};
1006
1007struct amd_irte_ops {
1008 void (*prepare)(void *, u32, bool, u8, u32, int);
1009 void (*activate)(struct amd_iommu *iommu, void *, u16, u16);
1010 void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);
1011 void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);
1012 void *(*get)(struct irq_remap_table *, int);
1013 void (*set_allocated)(struct irq_remap_table *, int);
1014 bool (*is_allocated)(struct irq_remap_table *, int);
1015 void (*clear_allocated)(struct irq_remap_table *, int);
1016};
1017
1018#ifdef CONFIG_IRQ_REMAP
1019extern struct amd_irte_ops irte_32_ops;
1020extern struct amd_irte_ops irte_128_ops;
1021#endif
1022
1023#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
1024

source code of linux/drivers/iommu/amd/amd_iommu_types.h