Merge branches 'arm/io-pgtable', 'arm/rockchip', 'arm/omap', 'x86/vt-d', 'ppc/pamu', 'core' and 'x86/amd' into next
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 374c129..c9ccc74 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -19,6 +19,8 @@
#include <linux/ratelimit.h>
#include <linux/pci.h>
+#include <linux/acpi.h>
+#include <linux/amba/bus.h>
#include <linux/pci-ats.h>
#include <linux/bitmap.h>
#include <linux/slab.h>
@@ -72,6 +74,7 @@
LIST_HEAD(ioapic_map);
LIST_HEAD(hpet_map);
+LIST_HEAD(acpihid_map);
/*
* Domain for untranslated devices - only allocated
@@ -92,6 +95,7 @@
struct list_head dev_data_list; /* For global dev_data_list */
struct protection_domain *domain; /* Domain the device is bound to */
u16 devid; /* PCI Device ID */
+ u16 alias; /* Alias Device ID */
bool iommu_v2; /* Device can make use of IOMMUv2 */
bool passthrough; /* Device is identity mapped */
struct {
@@ -161,6 +165,60 @@
*
****************************************************************************/
+static inline int match_hid_uid(struct device *dev,
+ struct acpihid_map_entry *entry)
+{
+ const char *hid, *uid;
+
+ hid = acpi_device_hid(ACPI_COMPANION(dev));
+ uid = acpi_device_uid(ACPI_COMPANION(dev));
+
+ if (!hid || !(*hid))
+ return -ENODEV;
+
+ if (!uid || !(*uid))
+ return strcmp(hid, entry->hid);
+
+ if (!(*entry->uid))
+ return strcmp(hid, entry->hid);
+
+ return (strcmp(hid, entry->hid) || strcmp(uid, entry->uid));
+}
+
+static inline u16 get_pci_device_id(struct device *dev)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+
+ return PCI_DEVID(pdev->bus->number, pdev->devfn);
+}
+
+static inline int get_acpihid_device_id(struct device *dev,
+ struct acpihid_map_entry **entry)
+{
+ struct acpihid_map_entry *p;
+
+ list_for_each_entry(p, &acpihid_map, list) {
+ if (!match_hid_uid(dev, p)) {
+ if (entry)
+ *entry = p;
+ return p->devid;
+ }
+ }
+ return -EINVAL;
+}
+
+static inline int get_device_id(struct device *dev)
+{
+ int devid;
+
+ if (dev_is_pci(dev))
+ devid = get_pci_device_id(dev);
+ else
+ devid = get_acpihid_device_id(dev, NULL);
+
+ return devid;
+}
+
static struct protection_domain *to_pdomain(struct iommu_domain *dom)
{
return container_of(dom, struct protection_domain, domain);
@@ -203,6 +261,69 @@
return dev_data;
}
+static int __last_alias(struct pci_dev *pdev, u16 alias, void *data)
+{
+ *(u16 *)data = alias;
+ return 0;
+}
+
+static u16 get_alias(struct device *dev)
+{
+ struct pci_dev *pdev = to_pci_dev(dev);
+ u16 devid, ivrs_alias, pci_alias;
+
+ /* The callers make sure that get_device_id() does not fail here */
+ devid = get_device_id(dev);
+ ivrs_alias = amd_iommu_alias_table[devid];
+ pci_for_each_dma_alias(pdev, __last_alias, &pci_alias);
+
+ if (ivrs_alias == pci_alias)
+ return ivrs_alias;
+
+ /*
+ * DMA alias showdown
+ *
+ * The IVRS is fairly reliable in telling us about aliases, but it
+ * can't know about every screwy device. If we don't have an IVRS
+ * reported alias, use the PCI reported alias. In that case we may
+ * still need to initialize the rlookup and dev_table entries if the
+ * alias is to a non-existent device.
+ */
+ if (ivrs_alias == devid) {
+ if (!amd_iommu_rlookup_table[pci_alias]) {
+ amd_iommu_rlookup_table[pci_alias] =
+ amd_iommu_rlookup_table[devid];
+ memcpy(amd_iommu_dev_table[pci_alias].data,
+ amd_iommu_dev_table[devid].data,
+ sizeof(amd_iommu_dev_table[pci_alias].data));
+ }
+
+ return pci_alias;
+ }
+
+ pr_info("AMD-Vi: Using IVRS reported alias %02x:%02x.%d "
+ "for device %s[%04x:%04x], kernel reported alias "
+ "%02x:%02x.%d\n", PCI_BUS_NUM(ivrs_alias), PCI_SLOT(ivrs_alias),
+ PCI_FUNC(ivrs_alias), dev_name(dev), pdev->vendor, pdev->device,
+ PCI_BUS_NUM(pci_alias), PCI_SLOT(pci_alias),
+ PCI_FUNC(pci_alias));
+
+ /*
+ * If we don't have a PCI DMA alias and the IVRS alias is on the same
+ * bus, then the IVRS table may know about a quirk that we don't.
+ */
+ if (pci_alias == devid &&
+ PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) {
+ pdev->dev_flags |= PCI_DEV_FLAGS_DMA_ALIAS_DEVFN;
+ pdev->dma_alias_devfn = ivrs_alias & 0xff;
+ pr_info("AMD-Vi: Added PCI DMA alias %02x.%d for %s\n",
+ PCI_SLOT(ivrs_alias), PCI_FUNC(ivrs_alias),
+ dev_name(dev));
+ }
+
+ return ivrs_alias;
+}
+
static struct iommu_dev_data *find_dev_data(u16 devid)
{
struct iommu_dev_data *dev_data;
@@ -215,18 +336,34 @@
return dev_data;
}
-static inline u16 get_device_id(struct device *dev)
-{
- struct pci_dev *pdev = to_pci_dev(dev);
-
- return PCI_DEVID(pdev->bus->number, pdev->devfn);
-}
-
static struct iommu_dev_data *get_dev_data(struct device *dev)
{
return dev->archdata.iommu;
}
+/*
+* Find or create an IOMMU group for a acpihid device.
+*/
+static struct iommu_group *acpihid_device_group(struct device *dev)
+{
+ struct acpihid_map_entry *p, *entry = NULL;
+ int devid;
+
+ devid = get_acpihid_device_id(dev, &entry);
+ if (devid < 0)
+ return ERR_PTR(devid);
+
+ list_for_each_entry(p, &acpihid_map, list) {
+ if ((devid == p->devid) && p->group)
+ entry->group = p->group;
+ }
+
+ if (!entry->group)
+ entry->group = generic_device_group(dev);
+
+ return entry->group;
+}
+
static bool pci_iommuv2_capable(struct pci_dev *pdev)
{
static const int caps[] = {
@@ -278,9 +415,11 @@
struct dma_ops_domain *dma_dom)
{
struct unity_map_entry *e;
- u16 devid;
+ int devid;
devid = get_device_id(dev);
+ if (devid < 0)
+ return;
list_for_each_entry(e, &amd_iommu_unity_map, list) {
if (!(devid >= e->devid_start && devid <= e->devid_end))
@@ -295,16 +434,14 @@
*/
static bool check_device(struct device *dev)
{
- u16 devid;
+ int devid;
if (!dev || !dev->dma_mask)
return false;
- /* No PCI device */
- if (!dev_is_pci(dev))
- return false;
-
devid = get_device_id(dev);
+ if (devid < 0)
+ return false;
/* Out of our scope? */
if (devid > amd_iommu_last_bdf)
@@ -339,20 +476,26 @@
static int iommu_init_device(struct device *dev)
{
- struct pci_dev *pdev = to_pci_dev(dev);
struct iommu_dev_data *dev_data;
+ int devid;
if (dev->archdata.iommu)
return 0;
- dev_data = find_dev_data(get_device_id(dev));
+ devid = get_device_id(dev);
+ if (devid < 0)
+ return devid;
+
+ dev_data = find_dev_data(devid);
if (!dev_data)
return -ENOMEM;
- if (pci_iommuv2_capable(pdev)) {
+ dev_data->alias = get_alias(dev);
+
+ if (dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) {
struct amd_iommu *iommu;
- iommu = amd_iommu_rlookup_table[dev_data->devid];
+ iommu = amd_iommu_rlookup_table[dev_data->devid];
dev_data->iommu_v2 = iommu->is_iommu_v2;
}
@@ -366,10 +509,14 @@
static void iommu_ignore_device(struct device *dev)
{
- u16 devid, alias;
+ u16 alias;
+ int devid;
devid = get_device_id(dev);
- alias = amd_iommu_alias_table[devid];
+ if (devid < 0)
+ return;
+
+ alias = get_alias(dev);
memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
@@ -380,8 +527,14 @@
static void iommu_uninit_device(struct device *dev)
{
- struct iommu_dev_data *dev_data = search_dev_data(get_device_id(dev));
+ int devid;
+ struct iommu_dev_data *dev_data;
+ devid = get_device_id(dev);
+ if (devid < 0)
+ return;
+
+ dev_data = search_dev_data(devid);
if (!dev_data)
return;
@@ -402,70 +555,6 @@
*/
}
-#ifdef CONFIG_AMD_IOMMU_STATS
-
-/*
- * Initialization code for statistics collection
- */
-
-DECLARE_STATS_COUNTER(compl_wait);
-DECLARE_STATS_COUNTER(cnt_map_single);
-DECLARE_STATS_COUNTER(cnt_unmap_single);
-DECLARE_STATS_COUNTER(cnt_map_sg);
-DECLARE_STATS_COUNTER(cnt_unmap_sg);
-DECLARE_STATS_COUNTER(cnt_alloc_coherent);
-DECLARE_STATS_COUNTER(cnt_free_coherent);
-DECLARE_STATS_COUNTER(cross_page);
-DECLARE_STATS_COUNTER(domain_flush_single);
-DECLARE_STATS_COUNTER(domain_flush_all);
-DECLARE_STATS_COUNTER(alloced_io_mem);
-DECLARE_STATS_COUNTER(total_map_requests);
-DECLARE_STATS_COUNTER(complete_ppr);
-DECLARE_STATS_COUNTER(invalidate_iotlb);
-DECLARE_STATS_COUNTER(invalidate_iotlb_all);
-DECLARE_STATS_COUNTER(pri_requests);
-
-static struct dentry *stats_dir;
-static struct dentry *de_fflush;
-
-static void amd_iommu_stats_add(struct __iommu_counter *cnt)
-{
- if (stats_dir == NULL)
- return;
-
- cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
- &cnt->value);
-}
-
-static void amd_iommu_stats_init(void)
-{
- stats_dir = debugfs_create_dir("amd-iommu", NULL);
- if (stats_dir == NULL)
- return;
-
- de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
- &amd_iommu_unmap_flush);
-
- amd_iommu_stats_add(&compl_wait);
- amd_iommu_stats_add(&cnt_map_single);
- amd_iommu_stats_add(&cnt_unmap_single);
- amd_iommu_stats_add(&cnt_map_sg);
- amd_iommu_stats_add(&cnt_unmap_sg);
- amd_iommu_stats_add(&cnt_alloc_coherent);
- amd_iommu_stats_add(&cnt_free_coherent);
- amd_iommu_stats_add(&cross_page);
- amd_iommu_stats_add(&domain_flush_single);
- amd_iommu_stats_add(&domain_flush_all);
- amd_iommu_stats_add(&alloced_io_mem);
- amd_iommu_stats_add(&total_map_requests);
- amd_iommu_stats_add(&complete_ppr);
- amd_iommu_stats_add(&invalidate_iotlb);
- amd_iommu_stats_add(&invalidate_iotlb_all);
- amd_iommu_stats_add(&pri_requests);
-}
-
-#endif
-
/****************************************************************************
*
* Interrupt handling functions
@@ -588,8 +677,6 @@
{
struct amd_iommu_fault fault;
- INC_STATS_COUNTER(pri_requests);
-
if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
return;
@@ -1061,7 +1148,7 @@
int ret;
iommu = amd_iommu_rlookup_table[dev_data->devid];
- alias = amd_iommu_alias_table[dev_data->devid];
+ alias = dev_data->alias;
ret = iommu_flush_dte(iommu, dev_data->devid);
if (!ret && alias != dev_data->devid)
@@ -2039,7 +2126,7 @@
bool ats;
iommu = amd_iommu_rlookup_table[dev_data->devid];
- alias = amd_iommu_alias_table[dev_data->devid];
+ alias = dev_data->alias;
ats = dev_data->ats.enabled;
/* Update data structures */
@@ -2073,7 +2160,7 @@
return;
iommu = amd_iommu_rlookup_table[dev_data->devid];
- alias = amd_iommu_alias_table[dev_data->devid];
+ alias = dev_data->alias;
/* decrease reference counters */
dev_data->domain->dev_iommu[iommu->index] -= 1;
@@ -2219,13 +2306,17 @@
static int attach_device(struct device *dev,
struct protection_domain *domain)
{
- struct pci_dev *pdev = to_pci_dev(dev);
+ struct pci_dev *pdev;
struct iommu_dev_data *dev_data;
unsigned long flags;
int ret;
dev_data = get_dev_data(dev);
+ if (!dev_is_pci(dev))
+ goto skip_ats_check;
+
+ pdev = to_pci_dev(dev);
if (domain->flags & PD_IOMMUV2_MASK) {
if (!dev_data->passthrough)
return -EINVAL;
@@ -2244,6 +2335,7 @@
dev_data->ats.qdep = pci_ats_queue_depth(pdev);
}
+skip_ats_check:
write_lock_irqsave(&amd_iommu_devtable_lock, flags);
ret = __attach_device(dev_data, domain);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
@@ -2300,6 +2392,9 @@
__detach_device(dev_data);
write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+ if (!dev_is_pci(dev))
+ return;
+
if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)
pdev_iommuv2_disable(to_pci_dev(dev));
else if (dev_data->ats.enabled)
@@ -2313,13 +2408,15 @@
struct iommu_dev_data *dev_data;
struct iommu_domain *domain;
struct amd_iommu *iommu;
- u16 devid;
- int ret;
+ int ret, devid;
if (!check_device(dev) || get_dev_data(dev))
return 0;
devid = get_device_id(dev);
+ if (devid < 0)
+ return devid;
+
iommu = amd_iommu_rlookup_table[devid];
ret = iommu_init_device(dev);
@@ -2357,18 +2454,29 @@
static void amd_iommu_remove_device(struct device *dev)
{
struct amd_iommu *iommu;
- u16 devid;
+ int devid;
if (!check_device(dev))
return;
devid = get_device_id(dev);
+ if (devid < 0)
+ return;
+
iommu = amd_iommu_rlookup_table[devid];
iommu_uninit_device(dev);
iommu_completion_wait(iommu);
}
+static struct iommu_group *amd_iommu_device_group(struct device *dev)
+{
+ if (dev_is_pci(dev))
+ return pci_device_group(dev);
+
+ return acpihid_device_group(dev);
+}
+
/*****************************************************************************
*
* The next functions belong to the dma_ops mapping/unmapping code.
@@ -2533,11 +2641,6 @@
pages = iommu_num_pages(paddr, size, PAGE_SIZE);
paddr &= PAGE_MASK;
- INC_STATS_COUNTER(total_map_requests);
-
- if (pages > 1)
- INC_STATS_COUNTER(cross_page);
-
if (align)
align_mask = (1UL << get_order(size)) - 1;
@@ -2558,8 +2661,6 @@
}
address += offset;
- ADD_STATS_COUNTER(alloced_io_mem, size);
-
if (unlikely(amd_iommu_np_cache)) {
domain_flush_pages(&dma_dom->domain, address, size);
domain_flush_complete(&dma_dom->domain);
@@ -2607,8 +2708,6 @@
start += PAGE_SIZE;
}
- SUB_STATS_COUNTER(alloced_io_mem, size);
-
dma_ops_free_addresses(dma_dom, dma_addr, pages);
}
@@ -2624,8 +2723,6 @@
struct protection_domain *domain;
u64 dma_mask;
- INC_STATS_COUNTER(cnt_map_single);
-
domain = get_domain(dev);
if (PTR_ERR(domain) == -EINVAL)
return (dma_addr_t)paddr;
@@ -2646,8 +2743,6 @@
{
struct protection_domain *domain;
- INC_STATS_COUNTER(cnt_unmap_single);
-
domain = get_domain(dev);
if (IS_ERR(domain))
return;
@@ -2670,8 +2765,6 @@
int mapped_elems = 0;
u64 dma_mask;
- INC_STATS_COUNTER(cnt_map_sg);
-
domain = get_domain(dev);
if (IS_ERR(domain))
return 0;
@@ -2717,8 +2810,6 @@
struct scatterlist *s;
int i;
- INC_STATS_COUNTER(cnt_unmap_sg);
-
domain = get_domain(dev);
if (IS_ERR(domain))
return;
@@ -2741,8 +2832,6 @@
struct protection_domain *domain;
struct page *page;
- INC_STATS_COUNTER(cnt_alloc_coherent);
-
domain = get_domain(dev);
if (PTR_ERR(domain) == -EINVAL) {
page = alloc_pages(flag, get_order(size));
@@ -2796,8 +2885,6 @@
struct protection_domain *domain;
struct page *page;
- INC_STATS_COUNTER(cnt_free_coherent);
-
page = virt_to_page(virt_addr);
size = PAGE_ALIGN(size);
@@ -2862,7 +2949,17 @@
int __init amd_iommu_init_api(void)
{
- return bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
+ int err = 0;
+
+ err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
+ if (err)
+ return err;
+#ifdef CONFIG_ARM_AMBA
+ err = bus_set_iommu(&amba_bustype, &amd_iommu_ops);
+ if (err)
+ return err;
+#endif
+ return 0;
}
int __init amd_iommu_init_dma_ops(void)
@@ -2879,8 +2976,6 @@
if (!swiotlb)
dma_ops = &nommu_dma_ops;
- amd_iommu_stats_init();
-
if (amd_iommu_unmap_flush)
pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
else
@@ -3034,12 +3129,14 @@
{
struct iommu_dev_data *dev_data = dev->archdata.iommu;
struct amd_iommu *iommu;
- u16 devid;
+ int devid;
if (!check_device(dev))
return;
devid = get_device_id(dev);
+ if (devid < 0)
+ return;
if (dev_data->domain != NULL)
detach_device(dev);
@@ -3157,9 +3254,11 @@
struct list_head *head)
{
struct unity_map_entry *entry;
- u16 devid;
+ int devid;
devid = get_device_id(dev);
+ if (devid < 0)
+ return;
list_for_each_entry(entry, &amd_iommu_unity_map, list) {
struct iommu_dm_region *region;
@@ -3206,7 +3305,7 @@
.iova_to_phys = amd_iommu_iova_to_phys,
.add_device = amd_iommu_add_device,
.remove_device = amd_iommu_remove_device,
- .device_group = pci_device_group,
+ .device_group = amd_iommu_device_group,
.get_dm_regions = amd_iommu_get_dm_regions,
.put_dm_regions = amd_iommu_put_dm_regions,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
@@ -3367,8 +3466,6 @@
static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
u64 address)
{
- INC_STATS_COUNTER(invalidate_iotlb);
-
return __flush_pasid(domain, pasid, address, false);
}
@@ -3389,8 +3486,6 @@
static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
{
- INC_STATS_COUNTER(invalidate_iotlb_all);
-
return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
true);
}
@@ -3510,8 +3605,6 @@
struct amd_iommu *iommu;
struct iommu_cmd cmd;
- INC_STATS_COUNTER(complete_ppr);
-
dev_data = get_dev_data(&pdev->dev);
iommu = amd_iommu_rlookup_table[dev_data->devid];
@@ -3861,6 +3954,9 @@
case X86_IRQ_ALLOC_TYPE_MSI:
case X86_IRQ_ALLOC_TYPE_MSIX:
devid = get_device_id(&info->msi_dev->dev);
+ if (devid < 0)
+ return NULL;
+
iommu = amd_iommu_rlookup_table[devid];
if (iommu)
return iommu->msi_domain;