blob: 9652848e315580afeee72e3780e397ebe7d75ec6 [file] [log] [blame]
Joerg Roedel8d283c32008-06-26 21:27:38 +02001/*
Joerg Roedel5d0d7152010-10-13 11:13:21 +02002 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
Joerg Roedel63ce3ae2015-02-04 16:12:55 +01003 * Author: Joerg Roedel <jroedel@suse.de>
Joerg Roedel8d283c32008-06-26 21:27:38 +02004 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
H. Peter Anvin1965aae2008-10-22 22:26:29 -070020#ifndef _ASM_X86_AMD_IOMMU_TYPES_H
21#define _ASM_X86_AMD_IOMMU_TYPES_H
Joerg Roedel8d283c32008-06-26 21:27:38 +020022
23#include <linux/types.h>
Joerg Roedel5d214fe2010-02-08 14:44:49 +010024#include <linux/mutex.h>
Joerg Roedel8d283c32008-06-26 21:27:38 +020025#include <linux/list.h>
26#include <linux/spinlock.h>
Shuah Khanc5081cd2013-02-27 17:07:19 -070027#include <linux/pci.h>
Bjorn Helgaas4b180d92014-02-14 14:08:51 -070028#include <linux/irqreturn.h>
Joerg Roedel8d283c32008-06-26 21:27:38 +020029
30/*
Joerg Roedelbb527772009-11-20 14:31:51 +010031 * Maximum number of IOMMUs supported
32 */
33#define MAX_IOMMUS 32
34
35/*
Joerg Roedel8d283c32008-06-26 21:27:38 +020036 * some size calculation constants
37 */
Joerg Roedel83f5aac2008-07-11 17:14:34 +020038#define DEV_TABLE_ENTRY_SIZE 32
Joerg Roedel8d283c32008-06-26 21:27:38 +020039#define ALIAS_TABLE_ENTRY_SIZE 2
40#define RLOOKUP_TABLE_ENTRY_SIZE (sizeof(void *))
41
Joerg Roedel8d283c32008-06-26 21:27:38 +020042/* Capability offsets used by the driver */
43#define MMIO_CAP_HDR_OFFSET 0x00
44#define MMIO_RANGE_OFFSET 0x0c
Joerg Roedela80dc3e2008-09-11 16:51:41 +020045#define MMIO_MISC_OFFSET 0x10
Joerg Roedel8d283c32008-06-26 21:27:38 +020046
47/* Masks, shifts and macros to parse the device range capability */
48#define MMIO_RANGE_LD_MASK 0xff000000
49#define MMIO_RANGE_FD_MASK 0x00ff0000
50#define MMIO_RANGE_BUS_MASK 0x0000ff00
51#define MMIO_RANGE_LD_SHIFT 24
52#define MMIO_RANGE_FD_SHIFT 16
53#define MMIO_RANGE_BUS_SHIFT 8
54#define MMIO_GET_LD(x) (((x) & MMIO_RANGE_LD_MASK) >> MMIO_RANGE_LD_SHIFT)
55#define MMIO_GET_FD(x) (((x) & MMIO_RANGE_FD_MASK) >> MMIO_RANGE_FD_SHIFT)
56#define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
Joerg Roedela80dc3e2008-09-11 16:51:41 +020057#define MMIO_MSI_NUM(x) ((x) & 0x1f)
Joerg Roedel8d283c32008-06-26 21:27:38 +020058
59/* Flag masks for the AMD IOMMU exclusion range */
60#define MMIO_EXCL_ENABLE_MASK 0x01ULL
61#define MMIO_EXCL_ALLOW_MASK 0x02ULL
62
63/* Used offsets into the MMIO space */
64#define MMIO_DEV_TABLE_OFFSET 0x0000
65#define MMIO_CMD_BUF_OFFSET 0x0008
66#define MMIO_EVT_BUF_OFFSET 0x0010
67#define MMIO_CONTROL_OFFSET 0x0018
68#define MMIO_EXCL_BASE_OFFSET 0x0020
69#define MMIO_EXCL_LIMIT_OFFSET 0x0028
Joerg Roedeld99ddec2011-04-11 11:03:18 +020070#define MMIO_EXT_FEATURES 0x0030
Joerg Roedel1a29ac02011-11-10 15:41:40 +010071#define MMIO_PPR_LOG_OFFSET 0x0038
Joerg Roedel8d283c32008-06-26 21:27:38 +020072#define MMIO_CMD_HEAD_OFFSET 0x2000
73#define MMIO_CMD_TAIL_OFFSET 0x2008
74#define MMIO_EVT_HEAD_OFFSET 0x2010
75#define MMIO_EVT_TAIL_OFFSET 0x2018
76#define MMIO_STATUS_OFFSET 0x2020
Joerg Roedel1a29ac02011-11-10 15:41:40 +010077#define MMIO_PPR_HEAD_OFFSET 0x2030
78#define MMIO_PPR_TAIL_OFFSET 0x2038
Steven L Kinney30861dd2013-06-05 16:11:48 -050079#define MMIO_CNTR_CONF_OFFSET 0x4000
80#define MMIO_CNTR_REG_OFFSET 0x40000
81#define MMIO_REG_END_OFFSET 0x80000
82
Joerg Roedel8d283c32008-06-26 21:27:38 +020083
Joerg Roedeld99ddec2011-04-11 11:03:18 +020084
85/* Extended Feature Bits */
86#define FEATURE_PREFETCH (1ULL<<0)
87#define FEATURE_PPR (1ULL<<1)
88#define FEATURE_X2APIC (1ULL<<2)
89#define FEATURE_NX (1ULL<<3)
90#define FEATURE_GT (1ULL<<4)
91#define FEATURE_IA (1ULL<<6)
92#define FEATURE_GA (1ULL<<7)
93#define FEATURE_HE (1ULL<<8)
94#define FEATURE_PC (1ULL<<9)
95
Joerg Roedel62f71ab2011-11-10 14:41:57 +010096#define FEATURE_PASID_SHIFT 32
97#define FEATURE_PASID_MASK (0x1fULL << FEATURE_PASID_SHIFT)
98
Joerg Roedel52815b72011-11-17 17:24:28 +010099#define FEATURE_GLXVAL_SHIFT 14
100#define FEATURE_GLXVAL_MASK (0x03ULL << FEATURE_GLXVAL_SHIFT)
101
Suravee Suthikulpanita919a012014-03-05 18:54:18 -0600102/* Note:
103 * The current driver only support 16-bit PASID.
104 * Currently, hardware only implement upto 16-bit PASID
105 * even though the spec says it could have upto 20 bits.
106 */
107#define PASID_MASK 0x0000ffff
Joerg Roedel52815b72011-11-17 17:24:28 +0100108
Joerg Roedel519c31b2008-08-14 19:55:15 +0200109/* MMIO status bits */
Suravee Suthikulpanit925fe082013-03-27 18:51:52 -0500110#define MMIO_STATUS_EVT_INT_MASK (1 << 1)
Joerg Roedel72e1dcc2011-11-10 19:13:51 +0100111#define MMIO_STATUS_COM_WAIT_INT_MASK (1 << 2)
112#define MMIO_STATUS_PPR_INT_MASK (1 << 6)
Joerg Roedel519c31b2008-08-14 19:55:15 +0200113
Joerg Roedel90008ee2008-09-09 16:41:05 +0200114/* event logging constants */
115#define EVENT_ENTRY_SIZE 0x10
116#define EVENT_TYPE_SHIFT 28
117#define EVENT_TYPE_MASK 0xf
118#define EVENT_TYPE_ILL_DEV 0x1
119#define EVENT_TYPE_IO_FAULT 0x2
120#define EVENT_TYPE_DEV_TAB_ERR 0x3
121#define EVENT_TYPE_PAGE_TAB_ERR 0x4
122#define EVENT_TYPE_ILL_CMD 0x5
123#define EVENT_TYPE_CMD_HARD_ERR 0x6
124#define EVENT_TYPE_IOTLB_INV_TO 0x7
125#define EVENT_TYPE_INV_DEV_REQ 0x8
126#define EVENT_DEVID_MASK 0xffff
127#define EVENT_DEVID_SHIFT 0
128#define EVENT_DOMID_MASK 0xffff
129#define EVENT_DOMID_SHIFT 0
130#define EVENT_FLAGS_MASK 0xfff
131#define EVENT_FLAGS_SHIFT 0x10
132
Joerg Roedel8d283c32008-06-26 21:27:38 +0200133/* feature control bits */
134#define CONTROL_IOMMU_EN 0x00ULL
135#define CONTROL_HT_TUN_EN 0x01ULL
136#define CONTROL_EVT_LOG_EN 0x02ULL
137#define CONTROL_EVT_INT_EN 0x03ULL
138#define CONTROL_COMWAIT_EN 0x04ULL
Joerg Roedel1456e9d2011-12-22 14:51:53 +0100139#define CONTROL_INV_TIMEOUT 0x05ULL
Joerg Roedel8d283c32008-06-26 21:27:38 +0200140#define CONTROL_PASSPW_EN 0x08ULL
141#define CONTROL_RESPASSPW_EN 0x09ULL
142#define CONTROL_COHERENT_EN 0x0aULL
143#define CONTROL_ISOC_EN 0x0bULL
144#define CONTROL_CMDBUF_EN 0x0cULL
145#define CONTROL_PPFLOG_EN 0x0dULL
146#define CONTROL_PPFINT_EN 0x0eULL
Joerg Roedel1a29ac02011-11-10 15:41:40 +0100147#define CONTROL_PPR_EN 0x0fULL
Joerg Roedelcbc33a92011-11-25 11:41:31 +0100148#define CONTROL_GT_EN 0x10ULL
Joerg Roedel8d283c32008-06-26 21:27:38 +0200149
Joerg Roedel1456e9d2011-12-22 14:51:53 +0100150#define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT)
151#define CTRL_INV_TO_NONE 0
152#define CTRL_INV_TO_1MS 1
153#define CTRL_INV_TO_10MS 2
154#define CTRL_INV_TO_100MS 3
155#define CTRL_INV_TO_1S 4
156#define CTRL_INV_TO_10S 5
157#define CTRL_INV_TO_100S 6
158
Joerg Roedel8d283c32008-06-26 21:27:38 +0200159/* command specific defines */
160#define CMD_COMPL_WAIT 0x01
161#define CMD_INV_DEV_ENTRY 0x02
Joerg Roedelcb41ed82011-04-05 11:00:53 +0200162#define CMD_INV_IOMMU_PAGES 0x03
163#define CMD_INV_IOTLB_PAGES 0x04
Joerg Roedel7ef27982012-06-21 16:46:04 +0200164#define CMD_INV_IRT 0x05
Joerg Roedelc99afa22011-11-21 18:19:25 +0100165#define CMD_COMPLETE_PPR 0x07
Joerg Roedel58fc7f12011-04-11 11:13:24 +0200166#define CMD_INV_ALL 0x08
Joerg Roedel8d283c32008-06-26 21:27:38 +0200167
168#define CMD_COMPL_WAIT_STORE_MASK 0x01
Joerg Roedel519c31b2008-08-14 19:55:15 +0200169#define CMD_COMPL_WAIT_INT_MASK 0x02
Joerg Roedel8d283c32008-06-26 21:27:38 +0200170#define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01
171#define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02
Joerg Roedel22e266c2011-11-21 15:59:08 +0100172#define CMD_INV_IOMMU_PAGES_GN_MASK 0x04
Joerg Roedel8d283c32008-06-26 21:27:38 +0200173
Joerg Roedelc99afa22011-11-21 18:19:25 +0100174#define PPR_STATUS_MASK 0xf
175#define PPR_STATUS_SHIFT 12
176
Joerg Roedel999ba412008-07-03 19:35:08 +0200177#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL
178
Joerg Roedel8d283c32008-06-26 21:27:38 +0200179/* macros and definitions for device table entries */
180#define DEV_ENTRY_VALID 0x00
181#define DEV_ENTRY_TRANSLATION 0x01
182#define DEV_ENTRY_IR 0x3d
183#define DEV_ENTRY_IW 0x3e
Joerg Roedel9f5f5fb2008-08-14 19:55:16 +0200184#define DEV_ENTRY_NO_PAGE_FAULT 0x62
Joerg Roedel8d283c32008-06-26 21:27:38 +0200185#define DEV_ENTRY_EX 0x67
186#define DEV_ENTRY_SYSMGT1 0x68
187#define DEV_ENTRY_SYSMGT2 0x69
Joerg Roedel0ea2c422012-06-15 18:05:20 +0200188#define DEV_ENTRY_IRQ_TBL_EN 0x80
Joerg Roedel8d283c32008-06-26 21:27:38 +0200189#define DEV_ENTRY_INIT_PASS 0xb8
190#define DEV_ENTRY_EINT_PASS 0xb9
191#define DEV_ENTRY_NMI_PASS 0xba
192#define DEV_ENTRY_LINT0_PASS 0xbe
193#define DEV_ENTRY_LINT1_PASS 0xbf
Joerg Roedel38ddf412008-09-11 10:38:32 +0200194#define DEV_ENTRY_MODE_MASK 0x07
195#define DEV_ENTRY_MODE_SHIFT 0x09
Joerg Roedel8d283c32008-06-26 21:27:38 +0200196
Joerg Roedel7ef27982012-06-21 16:46:04 +0200197#define MAX_DEV_TABLE_ENTRIES 0xffff
198
Joerg Roedel8d283c32008-06-26 21:27:38 +0200199/* constants to configure the command buffer */
200#define CMD_BUFFER_SIZE 8192
Chris Wright549c90dc2010-04-02 18:27:53 -0700201#define CMD_BUFFER_UNINITIALIZED 1
Joerg Roedel8d283c32008-06-26 21:27:38 +0200202#define CMD_BUFFER_ENTRIES 512
203#define MMIO_CMD_SIZE_SHIFT 56
204#define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT)
205
Joerg Roedel335503e2008-09-05 14:29:07 +0200206/* constants for event buffer handling */
207#define EVT_BUFFER_SIZE 8192 /* 512 entries */
208#define EVT_LEN_MASK (0x9ULL << 56)
209
Joerg Roedel1a29ac02011-11-10 15:41:40 +0100210/* Constants for PPR Log handling */
211#define PPR_LOG_ENTRIES 512
212#define PPR_LOG_SIZE_SHIFT 56
213#define PPR_LOG_SIZE_512 (0x9ULL << PPR_LOG_SIZE_SHIFT)
214#define PPR_ENTRY_SIZE 16
215#define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES)
216
Joerg Roedel72e1dcc2011-11-10 19:13:51 +0100217#define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
218#define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
219#define PPR_DEVID(x) ((x) & 0xffffULL)
220#define PPR_TAG(x) (((x) >> 32) & 0x3ffULL)
221#define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
222#define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
223#define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
224
225#define PPR_REQ_FAULT 0x01
226
Joerg Roedel0feae532009-08-26 15:26:30 +0200227#define PAGE_MODE_NONE 0x00
Joerg Roedel8d283c32008-06-26 21:27:38 +0200228#define PAGE_MODE_1_LEVEL 0x01
229#define PAGE_MODE_2_LEVEL 0x02
230#define PAGE_MODE_3_LEVEL 0x03
Joerg Roedel9355a082009-09-02 14:24:08 +0200231#define PAGE_MODE_4_LEVEL 0x04
232#define PAGE_MODE_5_LEVEL 0x05
233#define PAGE_MODE_6_LEVEL 0x06
Joerg Roedel8d283c32008-06-26 21:27:38 +0200234
Joerg Roedel9355a082009-09-02 14:24:08 +0200235#define PM_LEVEL_SHIFT(x) (12 + ((x) * 9))
236#define PM_LEVEL_SIZE(x) (((x) < 6) ? \
237 ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \
238 (0xffffffffffffffffULL))
239#define PM_LEVEL_INDEX(x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL)
Joerg Roedel50020fb2009-09-02 15:38:40 +0200240#define PM_LEVEL_ENC(x) (((x) << 9) & 0xe00ULL)
241#define PM_LEVEL_PDE(x, a) ((a) | PM_LEVEL_ENC((x)) | \
242 IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)
Joerg Roedela6b256b2009-09-03 12:21:31 +0200243#define PM_PTE_LEVEL(pte) (((pte) >> 9) & 0x7ULL)
Joerg Roedel8d283c32008-06-26 21:27:38 +0200244
Joerg Roedelabdc5eb2009-09-03 11:33:51 +0200245#define PM_MAP_4k 0
246#define PM_ADDR_MASK 0x000ffffffffff000ULL
247#define PM_MAP_MASK(lvl) (PM_ADDR_MASK & \
248 (~((1ULL << (12 + ((lvl) * 9))) - 1)))
249#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr))
Joerg Roedel8d283c32008-06-26 21:27:38 +0200250
Joerg Roedelcbb9d722010-01-15 14:41:15 +0100251/*
252 * Returns the page table level to use for a given page size
253 * Pagesize is expected to be a power-of-two
254 */
255#define PAGE_SIZE_LEVEL(pagesize) \
256 ((__ffs(pagesize) - 12) / 9)
257/*
258 * Returns the number of ptes to use for a given page size
259 * Pagesize is expected to be a power-of-two
260 */
261#define PAGE_SIZE_PTE_COUNT(pagesize) \
262 (1ULL << ((__ffs(pagesize) - 12) % 9))
263
264/*
265 * Aligns a given io-virtual address to a given page size
266 * Pagesize is expected to be a power-of-two
267 */
268#define PAGE_SIZE_ALIGN(address, pagesize) \
269 ((address) & ~((pagesize) - 1))
270/*
Frank Arnolddf805ab2012-08-27 19:21:04 +0200271 * Creates an IOMMU PTE for an address and a given pagesize
Joerg Roedelcbb9d722010-01-15 14:41:15 +0100272 * The PTE has no permission bits set
273 * Pagesize is expected to be a power-of-two larger than 4096
274 */
275#define PAGE_SIZE_PTE(address, pagesize) \
276 (((address) | ((pagesize) - 1)) & \
277 (~(pagesize >> 1)) & PM_ADDR_MASK)
278
Joerg Roedel24cd7722010-01-19 17:27:39 +0100279/*
280 * Takes a PTE value with mode=0x07 and returns the page size it maps
281 */
282#define PTE_PAGE_SIZE(pte) \
283 (1ULL << (1 + ffz(((pte) | 0xfffULL))))
284
Joerg Roedel3039ca12015-04-01 14:58:48 +0200285/*
286 * Takes a page-table level and returns the default page-size for this level
287 */
288#define PTE_LEVEL_PAGE_SIZE(level) \
289 (1ULL << (12 + (9 * (level))))
290
Joerg Roedel8d283c32008-06-26 21:27:38 +0200291#define IOMMU_PTE_P (1ULL << 0)
Joerg Roedel38ddf412008-09-11 10:38:32 +0200292#define IOMMU_PTE_TV (1ULL << 1)
Joerg Roedel8d283c32008-06-26 21:27:38 +0200293#define IOMMU_PTE_U (1ULL << 59)
294#define IOMMU_PTE_FC (1ULL << 60)
295#define IOMMU_PTE_IR (1ULL << 61)
296#define IOMMU_PTE_IW (1ULL << 62)
297
Joerg Roedelca9cab32015-10-20 17:33:40 +0200298#define DTE_FLAG_IOTLB (1ULL << 32)
299#define DTE_FLAG_GV (1ULL << 55)
Joerg Roedelcbf3ccd2015-10-20 14:59:36 +0200300#define DTE_FLAG_MASK (0x3ffULL << 32)
Joerg Roedel52815b72011-11-17 17:24:28 +0100301#define DTE_GLX_SHIFT (56)
302#define DTE_GLX_MASK (3)
303
304#define DTE_GCR3_VAL_A(x) (((x) >> 12) & 0x00007ULL)
305#define DTE_GCR3_VAL_B(x) (((x) >> 15) & 0x0ffffULL)
306#define DTE_GCR3_VAL_C(x) (((x) >> 31) & 0xfffffULL)
307
308#define DTE_GCR3_INDEX_A 0
309#define DTE_GCR3_INDEX_B 1
310#define DTE_GCR3_INDEX_C 1
311
312#define DTE_GCR3_SHIFT_A 58
313#define DTE_GCR3_SHIFT_B 16
314#define DTE_GCR3_SHIFT_C 43
315
Joerg Roedelb16137b2011-11-21 16:50:23 +0100316#define GCR3_VALID 0x01ULL
Joerg Roedelfd7b5532011-04-05 15:31:08 +0200317
Joerg Roedel8d283c32008-06-26 21:27:38 +0200318#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
319#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
320#define IOMMU_PTE_PAGE(pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK))
321#define IOMMU_PTE_MODE(pte) (((pte) >> 9) & 0x07)
322
323#define IOMMU_PROT_MASK 0x03
324#define IOMMU_PROT_IR 0x01
325#define IOMMU_PROT_IW 0x02
326
327/* IOMMU capabilities */
328#define IOMMU_CAP_IOTLB 24
329#define IOMMU_CAP_NPCACHE 26
Joerg Roedeld99ddec2011-04-11 11:03:18 +0200330#define IOMMU_CAP_EFR 27
Joerg Roedel8d283c32008-06-26 21:27:38 +0200331
332#define MAX_DOMAIN_ID 65536
333
Joerg Roedel9fdb19d2008-12-02 17:46:25 +0100334/* Protection domain flags */
335#define PD_DMA_OPS_MASK (1UL << 0) /* domain used for dma_ops */
Joerg Roedele2dc14a2008-12-10 18:48:59 +0100336#define PD_DEFAULT_MASK (1UL << 1) /* domain is a default dma_ops
337 domain for an IOMMU */
Joerg Roedel0feae532009-08-26 15:26:30 +0200338#define PD_PASSTHROUGH_MASK (1UL << 2) /* domain has no page
339 translation */
Joerg Roedel52815b72011-11-17 17:24:28 +0100340#define PD_IOMMUV2_MASK (1UL << 3) /* domain has gcr3 table */
Joerg Roedel0feae532009-08-26 15:26:30 +0200341
Joerg Roedelfefda112009-05-20 12:21:42 +0200342extern bool amd_iommu_dump;
343#define DUMP_printk(format, arg...) \
344 do { \
345 if (amd_iommu_dump) \
Joerg Roedel4c6f40d2009-09-01 16:43:58 +0200346 printk(KERN_INFO "AMD-Vi: " format, ## arg); \
Joerg Roedelfefda112009-05-20 12:21:42 +0200347 } while(0);
Joerg Roedel9fdb19d2008-12-02 17:46:25 +0100348
Joerg Roedel318afd42009-11-23 18:32:38 +0100349/* global flag if IOMMUs cache non-present entries */
350extern bool amd_iommu_np_cache;
Joerg Roedel60f723b2011-04-05 12:50:24 +0200351/* Only true if all IOMMUs support device IOTLBs */
352extern bool amd_iommu_iotlb_sup;
Joerg Roedel318afd42009-11-23 18:32:38 +0100353
Joerg Roedel05152a02012-06-15 16:53:51 +0200354#define MAX_IRQS_PER_TABLE 256
355#define IRQ_TABLE_ALIGNMENT 128
356
Joerg Roedel0ea2c422012-06-15 18:05:20 +0200357struct irq_remap_table {
358 spinlock_t lock;
359 unsigned min_index;
360 u32 *table;
361};
362
363extern struct irq_remap_table **irq_lookup_table;
364
Joerg Roedel05152a02012-06-15 16:53:51 +0200365/* Interrupt remapping feature used? */
366extern bool amd_iommu_irq_remap;
367
368/* kmem_cache to get tables with 128 byte alignement */
369extern struct kmem_cache *amd_iommu_irq_cache;
370
Joerg Roedel56947032008-07-11 17:14:20 +0200371/*
Joerg Roedel3bd22172009-05-04 15:06:20 +0200372 * Make iterating over all IOMMUs easier
373 */
374#define for_each_iommu(iommu) \
375 list_for_each_entry((iommu), &amd_iommu_list, list)
376#define for_each_iommu_safe(iommu, next) \
377 list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list)
378
Joerg Roedel384de722009-05-15 12:30:05 +0200379#define APERTURE_RANGE_SHIFT 27 /* 128 MB */
380#define APERTURE_RANGE_SIZE (1ULL << APERTURE_RANGE_SHIFT)
381#define APERTURE_RANGE_PAGES (APERTURE_RANGE_SIZE >> PAGE_SHIFT)
382#define APERTURE_MAX_RANGES 32 /* allows 4GB of DMA address space */
383#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT)
384#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL)
Joerg Roedel8d283c32008-06-26 21:27:38 +0200385
Joerg Roedel72e1dcc2011-11-10 19:13:51 +0100386
387/*
388 * This struct is used to pass information about
389 * incoming PPR faults around.
390 */
391struct amd_iommu_fault {
392 u64 address; /* IO virtual address of the fault*/
393 u32 pasid; /* Address space identifier */
394 u16 device_id; /* Originating PCI device id */
395 u16 tag; /* PPR tag */
396 u16 flags; /* Fault flags */
397
398};
399
Joerg Roedel72e1dcc2011-11-10 19:13:51 +0100400
Joerg Roedelf3572db2011-11-23 12:36:25 +0100401struct iommu_domain;
Jiang Liu7c71d302015-04-13 14:11:33 +0800402struct irq_domain;
Joerg Roedelf3572db2011-11-23 12:36:25 +0100403
Joerg Roedel56947032008-07-11 17:14:20 +0200404/*
405 * This structure contains generic data for IOMMU protection domains
406 * independent of their use.
407 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200408struct protection_domain {
Joerg Roedelaeb26f52009-11-20 16:44:01 +0100409 struct list_head list; /* for list of all protection domains */
Joerg Roedel7c392cb2009-11-26 11:13:32 +0100410 struct list_head dev_list; /* List of all devices in this domain */
Joerg Roedel3f4b87b2015-03-26 13:43:07 +0100411 struct iommu_domain domain; /* generic domain handle used by
412 iommu core code */
Joerg Roedel9fdb19d2008-12-02 17:46:25 +0100413 spinlock_t lock; /* mostly used to lock the page table*/
Joerg Roedel5d214fe2010-02-08 14:44:49 +0100414 struct mutex api_lock; /* protect page tables in the iommu-api path */
Joerg Roedel9fdb19d2008-12-02 17:46:25 +0100415 u16 id; /* the domain id written to the device table */
416 int mode; /* paging mode (0-6 levels) */
417 u64 *pt_root; /* page table root pointer */
Joerg Roedel52815b72011-11-17 17:24:28 +0100418 int glx; /* Number of levels for GCR3 table */
419 u64 *gcr3_tbl; /* Guest CR3 table */
Joerg Roedel9fdb19d2008-12-02 17:46:25 +0100420 unsigned long flags; /* flags to find out type of domain */
Joerg Roedel04bfdd82009-09-02 16:00:23 +0200421 bool updated; /* complete domain flush required */
Joerg Roedel863c74e2008-12-02 17:56:36 +0100422 unsigned dev_cnt; /* devices assigned to this domain */
Joerg Roedelc4596112009-11-20 14:57:32 +0100423 unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200424};
425
Joerg Roedel56947032008-07-11 17:14:20 +0200426/*
Joerg Roedel56947032008-07-11 17:14:20 +0200427 * Structure where we save information about one hardware AMD IOMMU in the
428 * system.
429 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200430struct amd_iommu {
431 struct list_head list;
Joerg Roedel56947032008-07-11 17:14:20 +0200432
Joerg Roedelbb527772009-11-20 14:31:51 +0100433 /* Index within the IOMMU array */
434 int index;
435
Joerg Roedel56947032008-07-11 17:14:20 +0200436 /* locks the accesses to the hardware */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200437 spinlock_t lock;
438
Joerg Roedel3eaf28a2008-09-08 15:55:10 +0200439 /* Pointer to PCI device of this IOMMU */
440 struct pci_dev *dev;
441
Joerg Roedelc1bf94e2012-05-31 17:38:11 +0200442 /* Cache pdev to root device for resume quirks */
443 struct pci_dev *root_pdev;
444
Joerg Roedel56947032008-07-11 17:14:20 +0200445 /* physical address of MMIO space */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200446 u64 mmio_phys;
Steven L Kinney30861dd2013-06-05 16:11:48 -0500447
448 /* physical end address of MMIO space */
449 u64 mmio_phys_end;
450
Joerg Roedel56947032008-07-11 17:14:20 +0200451 /* virtual address of MMIO space */
Joerg Roedel98f1ad22012-07-06 13:28:37 +0200452 u8 __iomem *mmio_base;
Joerg Roedel56947032008-07-11 17:14:20 +0200453
454 /* capabilities of that IOMMU read from ACPI */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200455 u32 cap;
Joerg Roedel56947032008-07-11 17:14:20 +0200456
Joerg Roedele9bf5192010-09-20 14:33:07 +0200457 /* flags read from acpi table */
458 u8 acpi_flags;
459
Joerg Roedeld99ddec2011-04-11 11:03:18 +0200460 /* Extended features */
461 u64 features;
462
Joerg Roedel400a28a2011-11-28 15:11:02 +0100463 /* IOMMUv2 */
464 bool is_iommu_v2;
465
Joerg Roedel23c742d2012-06-12 11:47:34 +0200466 /* PCI device id of the IOMMU device */
467 u16 devid;
468
Richard Kennedyeac9fbc2008-11-24 13:53:24 +0000469 /*
470 * Capability pointer. There could be more than one IOMMU per PCI
471 * device function if there are more than one AMD IOMMU capability
472 * pointers.
473 */
474 u16 cap_ptr;
475
Joerg Roedelee893c22008-09-08 14:48:04 +0200476 /* pci domain of this IOMMU */
477 u16 pci_seg;
478
Joerg Roedel56947032008-07-11 17:14:20 +0200479 /* start of exclusion range of that IOMMU */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200480 u64 exclusion_start;
Joerg Roedel56947032008-07-11 17:14:20 +0200481 /* length of exclusion range of that IOMMU */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200482 u64 exclusion_length;
483
Joerg Roedel56947032008-07-11 17:14:20 +0200484 /* command buffer virtual address */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200485 u8 *cmd_buf;
Joerg Roedel8d283c32008-06-26 21:27:38 +0200486
Richard Kennedyeac9fbc2008-11-24 13:53:24 +0000487 /* event buffer virtual address */
488 u8 *evt_buf;
Joerg Roedel335503e2008-09-05 14:29:07 +0200489
Joerg Roedel1a29ac02011-11-10 15:41:40 +0100490 /* Base of the PPR log, if present */
491 u8 *ppr_log;
492
Joerg Roedela80dc3e2008-09-11 16:51:41 +0200493 /* true if interrupts for this IOMMU are already enabled */
494 bool int_enabled;
495
Richard Kennedyeac9fbc2008-11-24 13:53:24 +0000496 /* if one, we need to send a completion wait command */
Joerg Roedel0cfd7aa2008-12-10 19:58:00 +0100497 bool need_sync;
Richard Kennedyeac9fbc2008-11-24 13:53:24 +0000498
Alex Williamson066f2e92014-06-12 16:12:37 -0600499 /* IOMMU sysfs device */
500 struct device *iommu_dev;
501
Joerg Roedel4c894f42010-09-23 15:15:19 +0200502 /*
Matthew Garrett5bcd7572010-10-04 14:59:31 -0400503 * We can't rely on the BIOS to restore all values on reinit, so we
504 * need to stash them
Joerg Roedel4c894f42010-09-23 15:15:19 +0200505 */
Matthew Garrett5bcd7572010-10-04 14:59:31 -0400506
507 /* The iommu BAR */
508 u32 stored_addr_lo;
509 u32 stored_addr_hi;
510
511 /*
512 * Each iommu has 6 l1s, each of which is documented as having 0x12
513 * registers
514 */
515 u32 stored_l1[6][0x12];
516
517 /* The l2 indirect registers */
518 u32 stored_l2[0x83];
Steven L Kinney30861dd2013-06-05 16:11:48 -0500519
520 /* The maximum PC banks and counters/bank (PCSup=1) */
521 u8 max_banks;
522 u8 max_counters;
Jiang Liu7c71d302015-04-13 14:11:33 +0800523#ifdef CONFIG_IRQ_REMAP
524 struct irq_domain *ir_domain;
525 struct irq_domain *msi_domain;
526#endif
Joerg Roedel4bf5bee2016-09-14 11:41:59 +0200527
528 volatile u64 __aligned(8) cmd_sem;
Joerg Roedel8d283c32008-06-26 21:27:38 +0200529};
530
Wan Zongshun2a0cb4e2016-04-01 09:06:00 -0400531#define ACPIHID_UID_LEN 256
532#define ACPIHID_HID_LEN 9
533
534struct acpihid_map_entry {
535 struct list_head list;
536 u8 uid[ACPIHID_UID_LEN];
537 u8 hid[ACPIHID_HID_LEN];
538 u16 devid;
539 u16 root_devid;
540 bool cmd_line;
541 struct iommu_group *group;
542};
543
Joerg Roedel6efed632012-06-14 15:52:58 +0200544struct devid_map {
545 struct list_head list;
546 u8 id;
547 u16 devid;
Joerg Roedel31cff672013-04-09 16:53:58 +0200548 bool cmd_line;
Joerg Roedel6efed632012-06-14 15:52:58 +0200549};
550
551/* Map HPET and IOAPIC ids to the devid used by the IOMMU */
552extern struct list_head ioapic_map;
553extern struct list_head hpet_map;
Wan Zongshun2a0cb4e2016-04-01 09:06:00 -0400554extern struct list_head acpihid_map;
Joerg Roedel6efed632012-06-14 15:52:58 +0200555
Joerg Roedel56947032008-07-11 17:14:20 +0200556/*
557 * List with all IOMMUs in the system. This list is not locked because it is
558 * only written and read at driver initialization or suspend time
559 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200560extern struct list_head amd_iommu_list;
561
Joerg Roedel56947032008-07-11 17:14:20 +0200562/*
Joerg Roedelbb527772009-11-20 14:31:51 +0100563 * Array with pointers to each IOMMU struct
564 * The indices are referenced in the protection domains
565 */
566extern struct amd_iommu *amd_iommus[MAX_IOMMUS];
567
568/* Number of IOMMUs present in the system */
569extern int amd_iommus_present;
570
571/*
Joerg Roedelaeb26f52009-11-20 16:44:01 +0100572 * Declarations for the global list of all protection domains
573 */
574extern spinlock_t amd_iommu_pd_lock;
575extern struct list_head amd_iommu_pd_list;
576
577/*
Joerg Roedel56947032008-07-11 17:14:20 +0200578 * Structure defining one entry in the device table
579 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200580struct dev_table_entry {
Joerg Roedelee6c2862011-11-09 12:06:03 +0100581 u64 data[4];
Joerg Roedel8d283c32008-06-26 21:27:38 +0200582};
583
Joerg Roedel56947032008-07-11 17:14:20 +0200584/*
585 * One entry for unity mappings parsed out of the ACPI table.
586 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200587struct unity_map_entry {
588 struct list_head list;
Joerg Roedel56947032008-07-11 17:14:20 +0200589
590 /* starting device id this entry is used for (including) */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200591 u16 devid_start;
Joerg Roedel56947032008-07-11 17:14:20 +0200592 /* end device id this entry is used for (including) */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200593 u16 devid_end;
Joerg Roedel56947032008-07-11 17:14:20 +0200594
595 /* start address to unity map (including) */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200596 u64 address_start;
Joerg Roedel56947032008-07-11 17:14:20 +0200597 /* end address to unity map (including) */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200598 u64 address_end;
Joerg Roedel56947032008-07-11 17:14:20 +0200599
600 /* required protection */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200601 int prot;
602};
603
Joerg Roedel56947032008-07-11 17:14:20 +0200604/*
605 * List of all unity mappings. It is not locked because as runtime it is only
606 * read. It is created at ACPI table parsing time.
607 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200608extern struct list_head amd_iommu_unity_map;
609
Joerg Roedel56947032008-07-11 17:14:20 +0200610/*
611 * Data structures for device handling
612 */
613
614/*
615 * Device table used by hardware. Read and write accesses by software are
616 * locked with the amd_iommu_pd_table lock.
617 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200618extern struct dev_table_entry *amd_iommu_dev_table;
Joerg Roedel56947032008-07-11 17:14:20 +0200619
620/*
621 * Alias table to find requestor ids to device ids. Not locked because only
622 * read on runtime.
623 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200624extern u16 *amd_iommu_alias_table;
Joerg Roedel56947032008-07-11 17:14:20 +0200625
626/*
627 * Reverse lookup table to find the IOMMU which translates a specific device.
628 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200629extern struct amd_iommu **amd_iommu_rlookup_table;
630
Joerg Roedel56947032008-07-11 17:14:20 +0200631/* size of the dma_ops aperture as power of 2 */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200632extern unsigned amd_iommu_aperture_order;
633
Joerg Roedel56947032008-07-11 17:14:20 +0200634/* largest PCI device id we expect translation requests for */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200635extern u16 amd_iommu_last_bdf;
636
Joerg Roedel56947032008-07-11 17:14:20 +0200637/* allocation bitmap for domain ids */
Joerg Roedel8d283c32008-06-26 21:27:38 +0200638extern unsigned long *amd_iommu_pd_alloc_bitmap;
639
FUJITA Tomonoriafa9fdc2008-09-20 01:23:30 +0900640/*
641 * If true, the addresses will be flushed on unmap time, not when
642 * they are reused
643 */
Viresh Kumar621a5f72015-09-26 15:04:07 -0700644extern bool amd_iommu_unmap_flush;
FUJITA Tomonoriafa9fdc2008-09-20 01:23:30 +0900645
Suravee Suthikulpanita919a012014-03-05 18:54:18 -0600646/* Smallest max PASID supported by any IOMMU in the system */
647extern u32 amd_iommu_max_pasid;
Joerg Roedel62f71ab2011-11-10 14:41:57 +0100648
Joerg Roedel400a28a2011-11-28 15:11:02 +0100649extern bool amd_iommu_v2_present;
650
Joerg Roedel5abcdba2011-12-01 15:49:45 +0100651extern bool amd_iommu_force_isolation;
652
Joerg Roedel52815b72011-11-17 17:24:28 +0100653/* Max levels of glxval supported */
654extern int amd_iommu_max_glx_val;
655
Joerg Roedel98f1ad22012-07-06 13:28:37 +0200656/*
657 * This function flushes all internal caches of
658 * the IOMMU used by this driver.
659 */
660extern void iommu_flush_all_caches(struct amd_iommu *iommu);
661
Joerg Roedel6efed632012-06-14 15:52:58 +0200662static inline int get_ioapic_devid(int id)
663{
664 struct devid_map *entry;
665
666 list_for_each_entry(entry, &ioapic_map, list) {
667 if (entry->id == id)
668 return entry->devid;
669 }
670
671 return -EINVAL;
672}
673
674static inline int get_hpet_devid(int id)
675{
676 struct devid_map *entry;
677
678 list_for_each_entry(entry, &hpet_map, list) {
679 if (entry->id == id)
680 return entry->devid;
681 }
682
683 return -EINVAL;
684}
685
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700686#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */