Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 1 | |
| 2 | #define pr_fmt(fmt) "DMAR-IR: " fmt |
| 3 | |
Yinghai Lu | 5aeecaf | 2008-08-19 20:49:59 -0700 | [diff] [blame] | 4 | #include <linux/interrupt.h> |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 5 | #include <linux/dmar.h> |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 6 | #include <linux/spinlock.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 7 | #include <linux/slab.h> |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 8 | #include <linux/jiffies.h> |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 9 | #include <linux/hpet.h> |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 10 | #include <linux/pci.h> |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 11 | #include <linux/irq.h> |
Lv Zheng | 8b48463 | 2013-12-03 08:49:16 +0800 | [diff] [blame] | 12 | #include <linux/intel-iommu.h> |
| 13 | #include <linux/acpi.h> |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 14 | #include <asm/io_apic.h> |
Yinghai Lu | 17483a1 | 2008-12-12 13:14:18 -0800 | [diff] [blame] | 15 | #include <asm/smp.h> |
Jaswinder Singh Rajput | 6d652ea | 2009-01-07 21:38:59 +0530 | [diff] [blame] | 16 | #include <asm/cpu.h> |
Suresh Siddha | 8a8f422 | 2012-03-30 11:47:08 -0700 | [diff] [blame] | 17 | #include <asm/irq_remapping.h> |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 18 | #include <asm/pci-direct.h> |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 19 | #include <asm/msidef.h> |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 20 | |
Suresh Siddha | 8a8f422 | 2012-03-30 11:47:08 -0700 | [diff] [blame] | 21 | #include "irq_remapping.h" |
Joerg Roedel | 736baef | 2012-03-30 11:47:00 -0700 | [diff] [blame] | 22 | |
Joerg Roedel | eef93fd | 2012-03-30 11:46:59 -0700 | [diff] [blame] | 23 | struct ioapic_scope { |
| 24 | struct intel_iommu *iommu; |
| 25 | unsigned int id; |
| 26 | unsigned int bus; /* PCI bus number */ |
| 27 | unsigned int devfn; /* PCI devfn number */ |
| 28 | }; |
| 29 | |
| 30 | struct hpet_scope { |
| 31 | struct intel_iommu *iommu; |
| 32 | u8 id; |
| 33 | unsigned int bus; |
| 34 | unsigned int devfn; |
| 35 | }; |
| 36 | |
| 37 | #define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0) |
Jiang Liu | 13d09b6 | 2015-01-07 15:31:37 +0800 | [diff] [blame] | 38 | #define IRTE_DEST(dest) ((eim_mode) ? dest : dest << 8) |
Joerg Roedel | eef93fd | 2012-03-30 11:46:59 -0700 | [diff] [blame] | 39 | |
Jiang Liu | 13d09b6 | 2015-01-07 15:31:37 +0800 | [diff] [blame] | 40 | static int __read_mostly eim_mode; |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 41 | static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 42 | static struct hpet_scope ir_hpet[MAX_HPET_TBS]; |
Chris Wright | d1423d5 | 2010-07-20 11:06:49 -0700 | [diff] [blame] | 43 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 44 | /* |
| 45 | * Lock ordering: |
| 46 | * ->dmar_global_lock |
| 47 | * ->irq_2_ir_lock |
| 48 | * ->qi->q_lock |
| 49 | * ->iommu->register_lock |
| 50 | * Note: |
| 51 | * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called |
| 52 | * in single-threaded environment with interrupt disabled, so no need to tabke |
| 53 | * the dmar_global_lock. |
| 54 | */ |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 55 | static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 56 | |
Jiang Liu | 694835d | 2014-01-06 14:18:16 +0800 | [diff] [blame] | 57 | static int __init parse_ioapics_under_ir(void); |
| 58 | |
Yinghai Lu | e420dfb | 2008-08-19 20:50:21 -0700 | [diff] [blame] | 59 | static struct irq_2_iommu *irq_2_iommu(unsigned int irq) |
| 60 | { |
Jiang Liu | 91411da | 2014-10-27 16:12:09 +0800 | [diff] [blame] | 61 | struct irq_cfg *cfg = irq_cfg(irq); |
Thomas Gleixner | 349d676 | 2010-10-10 12:29:27 +0200 | [diff] [blame] | 62 | return cfg ? &cfg->irq_2_iommu : NULL; |
Yinghai Lu | 0b8f1ef | 2008-12-05 18:58:31 -0800 | [diff] [blame] | 63 | } |
| 64 | |
Rashika Kheria | 6a7885c | 2013-12-18 12:04:27 +0530 | [diff] [blame] | 65 | static int get_irte(int irq, struct irte *entry) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 66 | { |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 67 | struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); |
Suresh Siddha | 4c5502b | 2009-03-16 17:04:53 -0700 | [diff] [blame] | 68 | unsigned long flags; |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 69 | int index; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 70 | |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 71 | if (!entry || !irq_iommu) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 72 | return -1; |
| 73 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 74 | raw_spin_lock_irqsave(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 75 | |
Greg Edwards | af43746 | 2014-07-23 10:13:26 -0600 | [diff] [blame] | 76 | if (unlikely(!irq_iommu->iommu)) { |
| 77 | raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); |
| 78 | return -1; |
| 79 | } |
| 80 | |
Yinghai Lu | e420dfb | 2008-08-19 20:50:21 -0700 | [diff] [blame] | 81 | index = irq_iommu->irte_index + irq_iommu->sub_handle; |
| 82 | *entry = *(irq_iommu->iommu->ir_table->base + index); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 83 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 84 | raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 85 | return 0; |
| 86 | } |
| 87 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 88 | static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 89 | { |
| 90 | struct ir_table *table = iommu->ir_table; |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 91 | struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); |
Jiang Liu | 91411da | 2014-10-27 16:12:09 +0800 | [diff] [blame] | 92 | struct irq_cfg *cfg = irq_cfg(irq); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 93 | unsigned int mask = 0; |
Suresh Siddha | 4c5502b | 2009-03-16 17:04:53 -0700 | [diff] [blame] | 94 | unsigned long flags; |
Dan Carpenter | 9f4c744 | 2014-01-09 08:32:36 +0300 | [diff] [blame] | 95 | int index; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 96 | |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 97 | if (!count || !irq_iommu) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 98 | return -1; |
| 99 | |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 100 | if (count > 1) { |
| 101 | count = __roundup_pow_of_two(count); |
| 102 | mask = ilog2(count); |
| 103 | } |
| 104 | |
| 105 | if (mask > ecap_max_handle_mask(iommu->ecap)) { |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 106 | pr_err("Requested mask %x exceeds the max invalidation handle" |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 107 | " mask value %Lx\n", mask, |
| 108 | ecap_max_handle_mask(iommu->ecap)); |
| 109 | return -1; |
| 110 | } |
| 111 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 112 | raw_spin_lock_irqsave(&irq_2_ir_lock, flags); |
Jiang Liu | 360eb3c5 | 2014-01-06 14:18:08 +0800 | [diff] [blame] | 113 | index = bitmap_find_free_region(table->bitmap, |
| 114 | INTR_REMAP_TABLE_ENTRIES, mask); |
| 115 | if (index < 0) { |
| 116 | pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id); |
| 117 | } else { |
| 118 | cfg->remapped = 1; |
| 119 | irq_iommu->iommu = iommu; |
| 120 | irq_iommu->irte_index = index; |
| 121 | irq_iommu->sub_handle = 0; |
| 122 | irq_iommu->irte_mask = mask; |
| 123 | } |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 124 | raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 125 | |
| 126 | return index; |
| 127 | } |
| 128 | |
Yu Zhao | 704126a | 2009-01-04 16:28:52 +0800 | [diff] [blame] | 129 | static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 130 | { |
| 131 | struct qi_desc desc; |
| 132 | |
| 133 | desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) |
| 134 | | QI_IEC_SELECTIVE; |
| 135 | desc.high = 0; |
| 136 | |
Yu Zhao | 704126a | 2009-01-04 16:28:52 +0800 | [diff] [blame] | 137 | return qi_submit_sync(&desc, iommu); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 138 | } |
| 139 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 140 | static int map_irq_to_irte_handle(int irq, u16 *sub_handle) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 141 | { |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 142 | struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); |
Suresh Siddha | 4c5502b | 2009-03-16 17:04:53 -0700 | [diff] [blame] | 143 | unsigned long flags; |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 144 | int index; |
| 145 | |
| 146 | if (!irq_iommu) |
| 147 | return -1; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 148 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 149 | raw_spin_lock_irqsave(&irq_2_ir_lock, flags); |
Yinghai Lu | e420dfb | 2008-08-19 20:50:21 -0700 | [diff] [blame] | 150 | *sub_handle = irq_iommu->sub_handle; |
| 151 | index = irq_iommu->irte_index; |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 152 | raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 153 | return index; |
| 154 | } |
| 155 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 156 | static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 157 | { |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 158 | struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); |
Jiang Liu | 91411da | 2014-10-27 16:12:09 +0800 | [diff] [blame] | 159 | struct irq_cfg *cfg = irq_cfg(irq); |
Suresh Siddha | 4c5502b | 2009-03-16 17:04:53 -0700 | [diff] [blame] | 160 | unsigned long flags; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 161 | |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 162 | if (!irq_iommu) |
Yinghai Lu | 0b8f1ef | 2008-12-05 18:58:31 -0800 | [diff] [blame] | 163 | return -1; |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 164 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 165 | raw_spin_lock_irqsave(&irq_2_ir_lock, flags); |
Yinghai Lu | 0b8f1ef | 2008-12-05 18:58:31 -0800 | [diff] [blame] | 166 | |
Joerg Roedel | 9b1b0e4 | 2012-09-26 12:44:45 +0200 | [diff] [blame] | 167 | cfg->remapped = 1; |
Yinghai Lu | e420dfb | 2008-08-19 20:50:21 -0700 | [diff] [blame] | 168 | irq_iommu->iommu = iommu; |
| 169 | irq_iommu->irte_index = index; |
| 170 | irq_iommu->sub_handle = subhandle; |
| 171 | irq_iommu->irte_mask = 0; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 172 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 173 | raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 174 | |
| 175 | return 0; |
| 176 | } |
| 177 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 178 | static int modify_irte(int irq, struct irte *irte_modified) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 179 | { |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 180 | struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 181 | struct intel_iommu *iommu; |
Suresh Siddha | 4c5502b | 2009-03-16 17:04:53 -0700 | [diff] [blame] | 182 | unsigned long flags; |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 183 | struct irte *irte; |
| 184 | int rc, index; |
| 185 | |
| 186 | if (!irq_iommu) |
| 187 | return -1; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 188 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 189 | raw_spin_lock_irqsave(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 190 | |
Yinghai Lu | e420dfb | 2008-08-19 20:50:21 -0700 | [diff] [blame] | 191 | iommu = irq_iommu->iommu; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 192 | |
Yinghai Lu | e420dfb | 2008-08-19 20:50:21 -0700 | [diff] [blame] | 193 | index = irq_iommu->irte_index + irq_iommu->sub_handle; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 194 | irte = &iommu->ir_table->base[index]; |
| 195 | |
Linus Torvalds | c513b67 | 2010-08-06 11:02:31 -0700 | [diff] [blame] | 196 | set_64bit(&irte->low, irte_modified->low); |
| 197 | set_64bit(&irte->high, irte_modified->high); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 198 | __iommu_flush_cache(iommu, irte, sizeof(*irte)); |
| 199 | |
Yu Zhao | 704126a | 2009-01-04 16:28:52 +0800 | [diff] [blame] | 200 | rc = qi_flush_iec(iommu, index, 0); |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 201 | raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); |
Yu Zhao | 704126a | 2009-01-04 16:28:52 +0800 | [diff] [blame] | 202 | |
| 203 | return rc; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 204 | } |
| 205 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 206 | static struct intel_iommu *map_hpet_to_ir(u8 hpet_id) |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 207 | { |
| 208 | int i; |
| 209 | |
| 210 | for (i = 0; i < MAX_HPET_TBS; i++) |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 211 | if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu) |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 212 | return ir_hpet[i].iommu; |
| 213 | return NULL; |
| 214 | } |
| 215 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 216 | static struct intel_iommu *map_ioapic_to_ir(int apic) |
Suresh Siddha | 89027d3 | 2008-07-10 11:16:56 -0700 | [diff] [blame] | 217 | { |
| 218 | int i; |
| 219 | |
| 220 | for (i = 0; i < MAX_IO_APICS; i++) |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 221 | if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu) |
Suresh Siddha | 89027d3 | 2008-07-10 11:16:56 -0700 | [diff] [blame] | 222 | return ir_ioapic[i].iommu; |
| 223 | return NULL; |
| 224 | } |
| 225 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 226 | static struct intel_iommu *map_dev_to_ir(struct pci_dev *dev) |
Suresh Siddha | 75c46fa | 2008-07-10 11:16:57 -0700 | [diff] [blame] | 227 | { |
| 228 | struct dmar_drhd_unit *drhd; |
| 229 | |
| 230 | drhd = dmar_find_matched_drhd_unit(dev); |
| 231 | if (!drhd) |
| 232 | return NULL; |
| 233 | |
| 234 | return drhd->iommu; |
| 235 | } |
| 236 | |
Weidong Han | c4658b4 | 2009-05-23 00:41:14 +0800 | [diff] [blame] | 237 | static int clear_entries(struct irq_2_iommu *irq_iommu) |
| 238 | { |
| 239 | struct irte *start, *entry, *end; |
| 240 | struct intel_iommu *iommu; |
| 241 | int index; |
| 242 | |
| 243 | if (irq_iommu->sub_handle) |
| 244 | return 0; |
| 245 | |
| 246 | iommu = irq_iommu->iommu; |
| 247 | index = irq_iommu->irte_index + irq_iommu->sub_handle; |
| 248 | |
| 249 | start = iommu->ir_table->base + index; |
| 250 | end = start + (1 << irq_iommu->irte_mask); |
| 251 | |
| 252 | for (entry = start; entry < end; entry++) { |
Linus Torvalds | c513b67 | 2010-08-06 11:02:31 -0700 | [diff] [blame] | 253 | set_64bit(&entry->low, 0); |
| 254 | set_64bit(&entry->high, 0); |
Weidong Han | c4658b4 | 2009-05-23 00:41:14 +0800 | [diff] [blame] | 255 | } |
Jiang Liu | 360eb3c5 | 2014-01-06 14:18:08 +0800 | [diff] [blame] | 256 | bitmap_release_region(iommu->ir_table->bitmap, index, |
| 257 | irq_iommu->irte_mask); |
Weidong Han | c4658b4 | 2009-05-23 00:41:14 +0800 | [diff] [blame] | 258 | |
| 259 | return qi_flush_iec(iommu, index, irq_iommu->irte_mask); |
| 260 | } |
| 261 | |
Joerg Roedel | 9d619f6 | 2012-03-30 11:47:04 -0700 | [diff] [blame] | 262 | static int free_irte(int irq) |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 263 | { |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 264 | struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); |
Suresh Siddha | 4c5502b | 2009-03-16 17:04:53 -0700 | [diff] [blame] | 265 | unsigned long flags; |
Thomas Gleixner | d585d06 | 2010-10-10 12:34:27 +0200 | [diff] [blame] | 266 | int rc; |
| 267 | |
| 268 | if (!irq_iommu) |
| 269 | return -1; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 270 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 271 | raw_spin_lock_irqsave(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 272 | |
Weidong Han | c4658b4 | 2009-05-23 00:41:14 +0800 | [diff] [blame] | 273 | rc = clear_entries(irq_iommu); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 274 | |
Yinghai Lu | e420dfb | 2008-08-19 20:50:21 -0700 | [diff] [blame] | 275 | irq_iommu->iommu = NULL; |
| 276 | irq_iommu->irte_index = 0; |
| 277 | irq_iommu->sub_handle = 0; |
| 278 | irq_iommu->irte_mask = 0; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 279 | |
Thomas Gleixner | 96f8e98 | 2011-07-19 16:28:19 +0200 | [diff] [blame] | 280 | raw_spin_unlock_irqrestore(&irq_2_ir_lock, flags); |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 281 | |
Yu Zhao | 704126a | 2009-01-04 16:28:52 +0800 | [diff] [blame] | 282 | return rc; |
Suresh Siddha | b6fcb33 | 2008-07-10 11:16:44 -0700 | [diff] [blame] | 283 | } |
| 284 | |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 285 | /* |
| 286 | * source validation type |
| 287 | */ |
| 288 | #define SVT_NO_VERIFY 0x0 /* no verification is required */ |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 289 | #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */ |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 290 | #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */ |
| 291 | |
| 292 | /* |
| 293 | * source-id qualifier |
| 294 | */ |
| 295 | #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */ |
| 296 | #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore |
| 297 | * the third least significant bit |
| 298 | */ |
| 299 | #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore |
| 300 | * the second and third least significant bits |
| 301 | */ |
| 302 | #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore |
| 303 | * the least three significant bits |
| 304 | */ |
| 305 | |
| 306 | /* |
| 307 | * set SVT, SQ and SID fields of irte to verify |
| 308 | * source ids of interrupt requests |
| 309 | */ |
| 310 | static void set_irte_sid(struct irte *irte, unsigned int svt, |
| 311 | unsigned int sq, unsigned int sid) |
| 312 | { |
Chris Wright | d1423d5 | 2010-07-20 11:06:49 -0700 | [diff] [blame] | 313 | if (disable_sourceid_checking) |
| 314 | svt = SVT_NO_VERIFY; |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 315 | irte->svt = svt; |
| 316 | irte->sq = sq; |
| 317 | irte->sid = sid; |
| 318 | } |
| 319 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 320 | static int set_ioapic_sid(struct irte *irte, int apic) |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 321 | { |
| 322 | int i; |
| 323 | u16 sid = 0; |
| 324 | |
| 325 | if (!irte) |
| 326 | return -1; |
| 327 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 328 | down_read(&dmar_global_lock); |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 329 | for (i = 0; i < MAX_IO_APICS; i++) { |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 330 | if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 331 | sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; |
| 332 | break; |
| 333 | } |
| 334 | } |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 335 | up_read(&dmar_global_lock); |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 336 | |
| 337 | if (sid == 0) { |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 338 | pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 339 | return -1; |
| 340 | } |
| 341 | |
Jiang Liu | 2fe2c60 | 2014-01-06 14:18:17 +0800 | [diff] [blame] | 342 | set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, sid); |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 343 | |
| 344 | return 0; |
| 345 | } |
| 346 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 347 | static int set_hpet_sid(struct irte *irte, u8 id) |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 348 | { |
| 349 | int i; |
| 350 | u16 sid = 0; |
| 351 | |
| 352 | if (!irte) |
| 353 | return -1; |
| 354 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 355 | down_read(&dmar_global_lock); |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 356 | for (i = 0; i < MAX_HPET_TBS; i++) { |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 357 | if (ir_hpet[i].iommu && ir_hpet[i].id == id) { |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 358 | sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; |
| 359 | break; |
| 360 | } |
| 361 | } |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 362 | up_read(&dmar_global_lock); |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 363 | |
| 364 | if (sid == 0) { |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 365 | pr_warn("Failed to set source-id of HPET block (%d)\n", id); |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 366 | return -1; |
| 367 | } |
| 368 | |
| 369 | /* |
| 370 | * Should really use SQ_ALL_16. Some platforms are broken. |
| 371 | * While we figure out the right quirks for these broken platforms, use |
| 372 | * SQ_13_IGNORE_3 for now. |
| 373 | */ |
| 374 | set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid); |
| 375 | |
| 376 | return 0; |
| 377 | } |
| 378 | |
Alex Williamson | 579305f | 2014-07-03 09:51:43 -0600 | [diff] [blame] | 379 | struct set_msi_sid_data { |
| 380 | struct pci_dev *pdev; |
| 381 | u16 alias; |
| 382 | }; |
| 383 | |
| 384 | static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque) |
| 385 | { |
| 386 | struct set_msi_sid_data *data = opaque; |
| 387 | |
| 388 | data->pdev = pdev; |
| 389 | data->alias = alias; |
| 390 | |
| 391 | return 0; |
| 392 | } |
| 393 | |
Joerg Roedel | 263b5e8 | 2012-03-30 11:47:06 -0700 | [diff] [blame] | 394 | static int set_msi_sid(struct irte *irte, struct pci_dev *dev) |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 395 | { |
Alex Williamson | 579305f | 2014-07-03 09:51:43 -0600 | [diff] [blame] | 396 | struct set_msi_sid_data data; |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 397 | |
| 398 | if (!irte || !dev) |
| 399 | return -1; |
| 400 | |
Alex Williamson | 579305f | 2014-07-03 09:51:43 -0600 | [diff] [blame] | 401 | pci_for_each_dma_alias(dev, set_msi_sid_cb, &data); |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 402 | |
Alex Williamson | 579305f | 2014-07-03 09:51:43 -0600 | [diff] [blame] | 403 | /* |
| 404 | * DMA alias provides us with a PCI device and alias. The only case |
| 405 | * where the it will return an alias on a different bus than the |
| 406 | * device is the case of a PCIe-to-PCI bridge, where the alias is for |
| 407 | * the subordinate bus. In this case we can only verify the bus. |
| 408 | * |
| 409 | * If the alias device is on a different bus than our source device |
| 410 | * then we have a topology based alias, use it. |
| 411 | * |
| 412 | * Otherwise, the alias is for a device DMA quirk and we cannot |
| 413 | * assume that MSI uses the same requester ID. Therefore use the |
| 414 | * original device. |
| 415 | */ |
| 416 | if (PCI_BUS_NUM(data.alias) != data.pdev->bus->number) |
| 417 | set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16, |
| 418 | PCI_DEVID(PCI_BUS_NUM(data.alias), |
| 419 | dev->bus->number)); |
| 420 | else if (data.pdev->bus->number != dev->bus->number) |
| 421 | set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, data.alias); |
| 422 | else |
| 423 | set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16, |
| 424 | PCI_DEVID(dev->bus->number, dev->devfn)); |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 425 | |
| 426 | return 0; |
| 427 | } |
| 428 | |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 429 | static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 430 | { |
| 431 | u64 addr; |
David Woodhouse | c416daa | 2009-05-10 20:30:58 +0100 | [diff] [blame] | 432 | u32 sts; |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 433 | unsigned long flags; |
| 434 | |
| 435 | addr = virt_to_phys((void *)iommu->ir_table->base); |
| 436 | |
Thomas Gleixner | 1f5b3c3 | 2011-07-19 16:19:51 +0200 | [diff] [blame] | 437 | raw_spin_lock_irqsave(&iommu->register_lock, flags); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 438 | |
| 439 | dmar_writeq(iommu->reg + DMAR_IRTA_REG, |
| 440 | (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE); |
| 441 | |
| 442 | /* Set interrupt-remapping table pointer */ |
Jan Kiszka | f63ef69 | 2014-08-11 13:13:25 +0200 | [diff] [blame] | 443 | writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 444 | |
| 445 | IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, |
| 446 | readl, (sts & DMA_GSTS_IRTPS), sts); |
Thomas Gleixner | 1f5b3c3 | 2011-07-19 16:19:51 +0200 | [diff] [blame] | 447 | raw_spin_unlock_irqrestore(&iommu->register_lock, flags); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 448 | |
| 449 | /* |
| 450 | * global invalidation of interrupt entry cache before enabling |
| 451 | * interrupt-remapping. |
| 452 | */ |
| 453 | qi_global_iec(iommu); |
| 454 | |
Thomas Gleixner | 1f5b3c3 | 2011-07-19 16:19:51 +0200 | [diff] [blame] | 455 | raw_spin_lock_irqsave(&iommu->register_lock, flags); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 456 | |
| 457 | /* Enable interrupt-remapping */ |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 458 | iommu->gcmd |= DMA_GCMD_IRE; |
Andy Lutomirski | af8d102 | 2013-02-01 14:57:43 -0800 | [diff] [blame] | 459 | iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */ |
David Woodhouse | c416daa | 2009-05-10 20:30:58 +0100 | [diff] [blame] | 460 | writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 461 | |
| 462 | IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, |
| 463 | readl, (sts & DMA_GSTS_IRES), sts); |
| 464 | |
Andy Lutomirski | af8d102 | 2013-02-01 14:57:43 -0800 | [diff] [blame] | 465 | /* |
| 466 | * With CFI clear in the Global Command register, we should be |
| 467 | * protected from dangerous (i.e. compatibility) interrupts |
| 468 | * regardless of x2apic status. Check just to be sure. |
| 469 | */ |
| 470 | if (sts & DMA_GSTS_CFIS) |
| 471 | WARN(1, KERN_WARNING |
| 472 | "Compatibility-format IRQs enabled despite intr remapping;\n" |
| 473 | "you are vulnerable to IRQ injection.\n"); |
| 474 | |
Thomas Gleixner | 1f5b3c3 | 2011-07-19 16:19:51 +0200 | [diff] [blame] | 475 | raw_spin_unlock_irqrestore(&iommu->register_lock, flags); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 476 | } |
| 477 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 478 | static int intel_setup_irq_remapping(struct intel_iommu *iommu) |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 479 | { |
| 480 | struct ir_table *ir_table; |
| 481 | struct page *pages; |
Jiang Liu | 360eb3c5 | 2014-01-06 14:18:08 +0800 | [diff] [blame] | 482 | unsigned long *bitmap; |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 483 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 484 | if (iommu->ir_table) |
| 485 | return 0; |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 486 | |
Thomas Gleixner | e3a981d | 2015-01-07 15:31:30 +0800 | [diff] [blame] | 487 | ir_table = kzalloc(sizeof(struct ir_table), GFP_KERNEL); |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 488 | if (!ir_table) |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 489 | return -ENOMEM; |
| 490 | |
Thomas Gleixner | e3a981d | 2015-01-07 15:31:30 +0800 | [diff] [blame] | 491 | pages = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, |
Suresh Siddha | 824cd75 | 2009-10-02 11:01:23 -0700 | [diff] [blame] | 492 | INTR_REMAP_PAGE_ORDER); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 493 | |
| 494 | if (!pages) { |
Jiang Liu | 360eb3c5 | 2014-01-06 14:18:08 +0800 | [diff] [blame] | 495 | pr_err("IR%d: failed to allocate pages of order %d\n", |
| 496 | iommu->seq_id, INTR_REMAP_PAGE_ORDER); |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 497 | goto out_free_table; |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 498 | } |
| 499 | |
Jiang Liu | 360eb3c5 | 2014-01-06 14:18:08 +0800 | [diff] [blame] | 500 | bitmap = kcalloc(BITS_TO_LONGS(INTR_REMAP_TABLE_ENTRIES), |
| 501 | sizeof(long), GFP_ATOMIC); |
| 502 | if (bitmap == NULL) { |
| 503 | pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 504 | goto out_free_pages; |
Jiang Liu | 360eb3c5 | 2014-01-06 14:18:08 +0800 | [diff] [blame] | 505 | } |
| 506 | |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 507 | ir_table->base = page_address(pages); |
Jiang Liu | 360eb3c5 | 2014-01-06 14:18:08 +0800 | [diff] [blame] | 508 | ir_table->bitmap = bitmap; |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 509 | iommu->ir_table = ir_table; |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 510 | |
| 511 | /* |
| 512 | * If the queued invalidation is already initialized, |
| 513 | * shouldn't disable it. |
| 514 | */ |
| 515 | if (!iommu->qi) { |
| 516 | /* |
| 517 | * Clear previous faults. |
| 518 | */ |
| 519 | dmar_fault(-1, iommu); |
| 520 | dmar_disable_qi(iommu); |
| 521 | |
| 522 | if (dmar_enable_qi(iommu)) { |
| 523 | pr_err("Failed to enable queued invalidation\n"); |
| 524 | goto out_free_bitmap; |
| 525 | } |
| 526 | } |
| 527 | |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 528 | return 0; |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 529 | |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 530 | out_free_bitmap: |
| 531 | kfree(bitmap); |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 532 | out_free_pages: |
| 533 | __free_pages(pages, INTR_REMAP_PAGE_ORDER); |
| 534 | out_free_table: |
| 535 | kfree(ir_table); |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 536 | |
| 537 | iommu->ir_table = NULL; |
| 538 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 539 | return -ENOMEM; |
| 540 | } |
| 541 | |
| 542 | static void intel_teardown_irq_remapping(struct intel_iommu *iommu) |
| 543 | { |
| 544 | if (iommu && iommu->ir_table) { |
| 545 | free_pages((unsigned long)iommu->ir_table->base, |
| 546 | INTR_REMAP_PAGE_ORDER); |
| 547 | kfree(iommu->ir_table->bitmap); |
| 548 | kfree(iommu->ir_table); |
| 549 | iommu->ir_table = NULL; |
| 550 | } |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 551 | } |
| 552 | |
Suresh Siddha | eba67e5 | 2009-03-16 17:04:56 -0700 | [diff] [blame] | 553 | /* |
| 554 | * Disable Interrupt Remapping. |
| 555 | */ |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 556 | static void iommu_disable_irq_remapping(struct intel_iommu *iommu) |
Suresh Siddha | eba67e5 | 2009-03-16 17:04:56 -0700 | [diff] [blame] | 557 | { |
| 558 | unsigned long flags; |
| 559 | u32 sts; |
| 560 | |
| 561 | if (!ecap_ir_support(iommu->ecap)) |
| 562 | return; |
| 563 | |
Fenghua Yu | b24696b | 2009-03-27 14:22:44 -0700 | [diff] [blame] | 564 | /* |
| 565 | * global invalidation of interrupt entry cache before disabling |
| 566 | * interrupt-remapping. |
| 567 | */ |
| 568 | qi_global_iec(iommu); |
| 569 | |
Thomas Gleixner | 1f5b3c3 | 2011-07-19 16:19:51 +0200 | [diff] [blame] | 570 | raw_spin_lock_irqsave(&iommu->register_lock, flags); |
Suresh Siddha | eba67e5 | 2009-03-16 17:04:56 -0700 | [diff] [blame] | 571 | |
| 572 | sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); |
| 573 | if (!(sts & DMA_GSTS_IRES)) |
| 574 | goto end; |
| 575 | |
| 576 | iommu->gcmd &= ~DMA_GCMD_IRE; |
| 577 | writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); |
| 578 | |
| 579 | IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, |
| 580 | readl, !(sts & DMA_GSTS_IRES), sts); |
| 581 | |
| 582 | end: |
Thomas Gleixner | 1f5b3c3 | 2011-07-19 16:19:51 +0200 | [diff] [blame] | 583 | raw_spin_unlock_irqrestore(&iommu->register_lock, flags); |
Suresh Siddha | eba67e5 | 2009-03-16 17:04:56 -0700 | [diff] [blame] | 584 | } |
| 585 | |
Suresh Siddha | 41750d3 | 2011-08-23 17:05:18 -0700 | [diff] [blame] | 586 | static int __init dmar_x2apic_optout(void) |
| 587 | { |
| 588 | struct acpi_table_dmar *dmar; |
| 589 | dmar = (struct acpi_table_dmar *)dmar_tbl; |
| 590 | if (!dmar || no_x2apic_optout) |
| 591 | return 0; |
| 592 | return dmar->flags & DMAR_X2APIC_OPT_OUT; |
| 593 | } |
| 594 | |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 595 | static void __init intel_cleanup_irq_remapping(void) |
| 596 | { |
| 597 | struct dmar_drhd_unit *drhd; |
| 598 | struct intel_iommu *iommu; |
| 599 | |
| 600 | for_each_iommu(iommu, drhd) { |
| 601 | if (ecap_ir_support(iommu->ecap)) { |
| 602 | iommu_disable_irq_remapping(iommu); |
| 603 | intel_teardown_irq_remapping(iommu); |
| 604 | } |
| 605 | } |
| 606 | |
| 607 | if (x2apic_supported()) |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 608 | pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 609 | } |
| 610 | |
| 611 | static int __init intel_prepare_irq_remapping(void) |
| 612 | { |
| 613 | struct dmar_drhd_unit *drhd; |
| 614 | struct intel_iommu *iommu; |
Joerg Roedel | 23256d0 | 2015-06-12 14:15:49 +0200 | [diff] [blame] | 615 | int eim = 0; |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 616 | |
Jiang Liu | 2966d95 | 2015-01-07 15:31:35 +0800 | [diff] [blame] | 617 | if (irq_remap_broken) { |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 618 | pr_warn("This system BIOS has enabled interrupt remapping\n" |
Jiang Liu | 2966d95 | 2015-01-07 15:31:35 +0800 | [diff] [blame] | 619 | "on a chipset that contains an erratum making that\n" |
| 620 | "feature unstable. To maintain system stability\n" |
| 621 | "interrupt remapping is being disabled. Please\n" |
| 622 | "contact your BIOS vendor for an update\n"); |
| 623 | add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); |
Jiang Liu | 2966d95 | 2015-01-07 15:31:35 +0800 | [diff] [blame] | 624 | return -ENODEV; |
| 625 | } |
| 626 | |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 627 | if (dmar_table_init() < 0) |
Jiang Liu | 2966d95 | 2015-01-07 15:31:35 +0800 | [diff] [blame] | 628 | return -ENODEV; |
| 629 | |
| 630 | if (!dmar_ir_support()) |
| 631 | return -ENODEV; |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 632 | |
| 633 | if (parse_ioapics_under_ir() != 1) { |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 634 | pr_info("Not enabling interrupt remapping\n"); |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 635 | goto error; |
| 636 | } |
| 637 | |
Joerg Roedel | 69cf1d8 | 2015-01-07 15:31:36 +0800 | [diff] [blame] | 638 | /* First make sure all IOMMUs support IRQ remapping */ |
Jiang Liu | 2966d95 | 2015-01-07 15:31:35 +0800 | [diff] [blame] | 639 | for_each_iommu(iommu, drhd) |
Joerg Roedel | 69cf1d8 | 2015-01-07 15:31:36 +0800 | [diff] [blame] | 640 | if (!ecap_ir_support(iommu->ecap)) |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 641 | goto error; |
Joerg Roedel | 69cf1d8 | 2015-01-07 15:31:36 +0800 | [diff] [blame] | 642 | |
Joerg Roedel | 23256d0 | 2015-06-12 14:15:49 +0200 | [diff] [blame] | 643 | /* Detect remapping mode: lapic or x2apic */ |
| 644 | if (x2apic_supported()) { |
| 645 | eim = !dmar_x2apic_optout(); |
| 646 | if (!eim) { |
| 647 | pr_info("x2apic is disabled because BIOS sets x2apic opt out bit."); |
| 648 | pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); |
| 649 | } |
| 650 | } |
| 651 | |
| 652 | for_each_iommu(iommu, drhd) { |
| 653 | if (eim && !ecap_eim_support(iommu->ecap)) { |
| 654 | pr_info("%s does not support EIM\n", iommu->name); |
| 655 | eim = 0; |
| 656 | } |
| 657 | } |
| 658 | |
| 659 | eim_mode = eim; |
| 660 | if (eim) |
| 661 | pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); |
| 662 | |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 663 | /* Do the initializations early */ |
| 664 | for_each_iommu(iommu, drhd) { |
| 665 | if (intel_setup_irq_remapping(iommu)) { |
| 666 | pr_err("Failed to setup irq remapping for %s\n", |
| 667 | iommu->name); |
Joerg Roedel | 69cf1d8 | 2015-01-07 15:31:36 +0800 | [diff] [blame] | 668 | goto error; |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 669 | } |
| 670 | } |
Joerg Roedel | 69cf1d8 | 2015-01-07 15:31:36 +0800 | [diff] [blame] | 671 | |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 672 | return 0; |
Jiang Liu | 2966d95 | 2015-01-07 15:31:35 +0800 | [diff] [blame] | 673 | |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 674 | error: |
| 675 | intel_cleanup_irq_remapping(); |
Jiang Liu | 2966d95 | 2015-01-07 15:31:35 +0800 | [diff] [blame] | 676 | return -ENODEV; |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 677 | } |
| 678 | |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 679 | static int __init intel_enable_irq_remapping(void) |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 680 | { |
| 681 | struct dmar_drhd_unit *drhd; |
Jiang Liu | 7c91977 | 2014-01-06 14:18:18 +0800 | [diff] [blame] | 682 | struct intel_iommu *iommu; |
Quentin Lambert | 2f119c7 | 2015-02-06 10:59:53 +0100 | [diff] [blame] | 683 | bool setup = false; |
Suresh Siddha | 41750d3 | 2011-08-23 17:05:18 -0700 | [diff] [blame] | 684 | |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 685 | for_each_iommu(iommu, drhd) |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 686 | iommu_disable_irq_remapping(iommu); |
Suresh Siddha | 1531a6a | 2009-03-16 17:04:57 -0700 | [diff] [blame] | 687 | |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 688 | /* |
| 689 | * Setup Interrupt-remapping for all the DRHD's now. |
| 690 | */ |
Jiang Liu | 7c91977 | 2014-01-06 14:18:18 +0800 | [diff] [blame] | 691 | for_each_iommu(iommu, drhd) { |
Joerg Roedel | 23256d0 | 2015-06-12 14:15:49 +0200 | [diff] [blame] | 692 | iommu_set_irq_remapping(iommu, eim_mode); |
Quentin Lambert | 2f119c7 | 2015-02-06 10:59:53 +0100 | [diff] [blame] | 693 | setup = true; |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 694 | } |
| 695 | |
| 696 | if (!setup) |
| 697 | goto error; |
| 698 | |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 699 | irq_remapping_enabled = 1; |
Joerg Roedel | afcc8a4 | 2012-09-26 12:44:36 +0200 | [diff] [blame] | 700 | |
| 701 | /* |
| 702 | * VT-d has a different layout for IO-APIC entries when |
| 703 | * interrupt remapping is enabled. So it needs a special routine |
| 704 | * to print IO-APIC entries for debugging purposes too. |
| 705 | */ |
| 706 | x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries; |
| 707 | |
Joerg Roedel | 23256d0 | 2015-06-12 14:15:49 +0200 | [diff] [blame] | 708 | pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic"); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 709 | |
Joerg Roedel | 23256d0 | 2015-06-12 14:15:49 +0200 | [diff] [blame] | 710 | return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 711 | |
| 712 | error: |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 713 | intel_cleanup_irq_remapping(); |
Suresh Siddha | 2ae2101 | 2008-07-10 11:16:43 -0700 | [diff] [blame] | 714 | return -1; |
| 715 | } |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 716 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 717 | static int ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope, |
| 718 | struct intel_iommu *iommu, |
| 719 | struct acpi_dmar_hardware_unit *drhd) |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 720 | { |
| 721 | struct acpi_dmar_pci_path *path; |
| 722 | u8 bus; |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 723 | int count, free = -1; |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 724 | |
| 725 | bus = scope->bus; |
| 726 | path = (struct acpi_dmar_pci_path *)(scope + 1); |
| 727 | count = (scope->length - sizeof(struct acpi_dmar_device_scope)) |
| 728 | / sizeof(struct acpi_dmar_pci_path); |
| 729 | |
| 730 | while (--count > 0) { |
| 731 | /* |
| 732 | * Access PCI directly due to the PCI |
| 733 | * subsystem isn't initialized yet. |
| 734 | */ |
Lv Zheng | fa5f508 | 2013-10-31 09:30:22 +0800 | [diff] [blame] | 735 | bus = read_pci_config_byte(bus, path->device, path->function, |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 736 | PCI_SECONDARY_BUS); |
| 737 | path++; |
| 738 | } |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 739 | |
| 740 | for (count = 0; count < MAX_HPET_TBS; count++) { |
| 741 | if (ir_hpet[count].iommu == iommu && |
| 742 | ir_hpet[count].id == scope->enumeration_id) |
| 743 | return 0; |
| 744 | else if (ir_hpet[count].iommu == NULL && free == -1) |
| 745 | free = count; |
| 746 | } |
| 747 | if (free == -1) { |
| 748 | pr_warn("Exceeded Max HPET blocks\n"); |
| 749 | return -ENOSPC; |
| 750 | } |
| 751 | |
| 752 | ir_hpet[free].iommu = iommu; |
| 753 | ir_hpet[free].id = scope->enumeration_id; |
| 754 | ir_hpet[free].bus = bus; |
| 755 | ir_hpet[free].devfn = PCI_DEVFN(path->device, path->function); |
| 756 | pr_info("HPET id %d under DRHD base 0x%Lx\n", |
| 757 | scope->enumeration_id, drhd->address); |
| 758 | |
| 759 | return 0; |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 760 | } |
| 761 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 762 | static int ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope, |
| 763 | struct intel_iommu *iommu, |
| 764 | struct acpi_dmar_hardware_unit *drhd) |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 765 | { |
| 766 | struct acpi_dmar_pci_path *path; |
| 767 | u8 bus; |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 768 | int count, free = -1; |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 769 | |
| 770 | bus = scope->bus; |
| 771 | path = (struct acpi_dmar_pci_path *)(scope + 1); |
| 772 | count = (scope->length - sizeof(struct acpi_dmar_device_scope)) |
| 773 | / sizeof(struct acpi_dmar_pci_path); |
| 774 | |
| 775 | while (--count > 0) { |
| 776 | /* |
| 777 | * Access PCI directly due to the PCI |
| 778 | * subsystem isn't initialized yet. |
| 779 | */ |
Lv Zheng | fa5f508 | 2013-10-31 09:30:22 +0800 | [diff] [blame] | 780 | bus = read_pci_config_byte(bus, path->device, path->function, |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 781 | PCI_SECONDARY_BUS); |
| 782 | path++; |
| 783 | } |
| 784 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 785 | for (count = 0; count < MAX_IO_APICS; count++) { |
| 786 | if (ir_ioapic[count].iommu == iommu && |
| 787 | ir_ioapic[count].id == scope->enumeration_id) |
| 788 | return 0; |
| 789 | else if (ir_ioapic[count].iommu == NULL && free == -1) |
| 790 | free = count; |
| 791 | } |
| 792 | if (free == -1) { |
| 793 | pr_warn("Exceeded Max IO APICS\n"); |
| 794 | return -ENOSPC; |
| 795 | } |
| 796 | |
| 797 | ir_ioapic[free].bus = bus; |
| 798 | ir_ioapic[free].devfn = PCI_DEVFN(path->device, path->function); |
| 799 | ir_ioapic[free].iommu = iommu; |
| 800 | ir_ioapic[free].id = scope->enumeration_id; |
| 801 | pr_info("IOAPIC id %d under DRHD base 0x%Lx IOMMU %d\n", |
| 802 | scope->enumeration_id, drhd->address, iommu->seq_id); |
| 803 | |
| 804 | return 0; |
Weidong Han | f007e99 | 2009-05-23 00:41:15 +0800 | [diff] [blame] | 805 | } |
| 806 | |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 807 | static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header, |
| 808 | struct intel_iommu *iommu) |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 809 | { |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 810 | int ret = 0; |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 811 | struct acpi_dmar_hardware_unit *drhd; |
| 812 | struct acpi_dmar_device_scope *scope; |
| 813 | void *start, *end; |
| 814 | |
| 815 | drhd = (struct acpi_dmar_hardware_unit *)header; |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 816 | start = (void *)(drhd + 1); |
| 817 | end = ((void *)drhd) + header->length; |
| 818 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 819 | while (start < end && ret == 0) { |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 820 | scope = start; |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 821 | if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) |
| 822 | ret = ir_parse_one_ioapic_scope(scope, iommu, drhd); |
| 823 | else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) |
| 824 | ret = ir_parse_one_hpet_scope(scope, iommu, drhd); |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 825 | start += scope->length; |
| 826 | } |
| 827 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 828 | return ret; |
| 829 | } |
| 830 | |
| 831 | static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu) |
| 832 | { |
| 833 | int i; |
| 834 | |
| 835 | for (i = 0; i < MAX_HPET_TBS; i++) |
| 836 | if (ir_hpet[i].iommu == iommu) |
| 837 | ir_hpet[i].iommu = NULL; |
| 838 | |
| 839 | for (i = 0; i < MAX_IO_APICS; i++) |
| 840 | if (ir_ioapic[i].iommu == iommu) |
| 841 | ir_ioapic[i].iommu = NULL; |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 842 | } |
| 843 | |
| 844 | /* |
| 845 | * Finds the assocaition between IOAPIC's and its Interrupt-remapping |
| 846 | * hardware unit. |
| 847 | */ |
Jiang Liu | 694835d | 2014-01-06 14:18:16 +0800 | [diff] [blame] | 848 | static int __init parse_ioapics_under_ir(void) |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 849 | { |
| 850 | struct dmar_drhd_unit *drhd; |
Jiang Liu | 7c91977 | 2014-01-06 14:18:18 +0800 | [diff] [blame] | 851 | struct intel_iommu *iommu; |
Quentin Lambert | 2f119c7 | 2015-02-06 10:59:53 +0100 | [diff] [blame] | 852 | bool ir_supported = false; |
Seth Forshee | 32ab31e | 2012-08-08 08:27:03 -0500 | [diff] [blame] | 853 | int ioapic_idx; |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 854 | |
Jiang Liu | 7c91977 | 2014-01-06 14:18:18 +0800 | [diff] [blame] | 855 | for_each_iommu(iommu, drhd) |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 856 | if (ecap_ir_support(iommu->ecap)) { |
Suresh Siddha | 20f3097 | 2009-08-04 12:07:08 -0700 | [diff] [blame] | 857 | if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu)) |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 858 | return -1; |
| 859 | |
Quentin Lambert | 2f119c7 | 2015-02-06 10:59:53 +0100 | [diff] [blame] | 860 | ir_supported = true; |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 861 | } |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 862 | |
Seth Forshee | 32ab31e | 2012-08-08 08:27:03 -0500 | [diff] [blame] | 863 | if (!ir_supported) |
| 864 | return 0; |
| 865 | |
| 866 | for (ioapic_idx = 0; ioapic_idx < nr_ioapics; ioapic_idx++) { |
| 867 | int ioapic_id = mpc_ioapic_id(ioapic_idx); |
| 868 | if (!map_ioapic_to_ir(ioapic_id)) { |
| 869 | pr_err(FW_BUG "ioapic %d has no mapping iommu, " |
| 870 | "interrupt remapping will be disabled\n", |
| 871 | ioapic_id); |
| 872 | return -1; |
| 873 | } |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 874 | } |
| 875 | |
Seth Forshee | 32ab31e | 2012-08-08 08:27:03 -0500 | [diff] [blame] | 876 | return 1; |
Suresh Siddha | ad3ad3f | 2008-07-10 11:16:40 -0700 | [diff] [blame] | 877 | } |
Fenghua Yu | b24696b | 2009-03-27 14:22:44 -0700 | [diff] [blame] | 878 | |
Rashika Kheria | 6a7885c | 2013-12-18 12:04:27 +0530 | [diff] [blame] | 879 | static int __init ir_dev_scope_init(void) |
Suresh Siddha | c2c7286 | 2011-08-23 17:05:19 -0700 | [diff] [blame] | 880 | { |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 881 | int ret; |
| 882 | |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 883 | if (!irq_remapping_enabled) |
Suresh Siddha | c2c7286 | 2011-08-23 17:05:19 -0700 | [diff] [blame] | 884 | return 0; |
| 885 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 886 | down_write(&dmar_global_lock); |
| 887 | ret = dmar_dev_scope_init(); |
| 888 | up_write(&dmar_global_lock); |
| 889 | |
| 890 | return ret; |
Suresh Siddha | c2c7286 | 2011-08-23 17:05:19 -0700 | [diff] [blame] | 891 | } |
| 892 | rootfs_initcall(ir_dev_scope_init); |
| 893 | |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 894 | static void disable_irq_remapping(void) |
Fenghua Yu | b24696b | 2009-03-27 14:22:44 -0700 | [diff] [blame] | 895 | { |
| 896 | struct dmar_drhd_unit *drhd; |
| 897 | struct intel_iommu *iommu = NULL; |
| 898 | |
| 899 | /* |
| 900 | * Disable Interrupt-remapping for all the DRHD's now. |
| 901 | */ |
| 902 | for_each_iommu(iommu, drhd) { |
| 903 | if (!ecap_ir_support(iommu->ecap)) |
| 904 | continue; |
| 905 | |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 906 | iommu_disable_irq_remapping(iommu); |
Fenghua Yu | b24696b | 2009-03-27 14:22:44 -0700 | [diff] [blame] | 907 | } |
| 908 | } |
| 909 | |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 910 | static int reenable_irq_remapping(int eim) |
Fenghua Yu | b24696b | 2009-03-27 14:22:44 -0700 | [diff] [blame] | 911 | { |
| 912 | struct dmar_drhd_unit *drhd; |
Quentin Lambert | 2f119c7 | 2015-02-06 10:59:53 +0100 | [diff] [blame] | 913 | bool setup = false; |
Fenghua Yu | b24696b | 2009-03-27 14:22:44 -0700 | [diff] [blame] | 914 | struct intel_iommu *iommu = NULL; |
| 915 | |
| 916 | for_each_iommu(iommu, drhd) |
| 917 | if (iommu->qi) |
| 918 | dmar_reenable_qi(iommu); |
| 919 | |
| 920 | /* |
| 921 | * Setup Interrupt-remapping for all the DRHD's now. |
| 922 | */ |
| 923 | for_each_iommu(iommu, drhd) { |
| 924 | if (!ecap_ir_support(iommu->ecap)) |
| 925 | continue; |
| 926 | |
| 927 | /* Set up interrupt remapping for iommu.*/ |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 928 | iommu_set_irq_remapping(iommu, eim); |
Quentin Lambert | 2f119c7 | 2015-02-06 10:59:53 +0100 | [diff] [blame] | 929 | setup = true; |
Fenghua Yu | b24696b | 2009-03-27 14:22:44 -0700 | [diff] [blame] | 930 | } |
| 931 | |
| 932 | if (!setup) |
| 933 | goto error; |
| 934 | |
| 935 | return 0; |
| 936 | |
| 937 | error: |
| 938 | /* |
| 939 | * handle error condition gracefully here! |
| 940 | */ |
| 941 | return -1; |
| 942 | } |
| 943 | |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 944 | static void prepare_irte(struct irte *irte, int vector, |
| 945 | unsigned int dest) |
| 946 | { |
| 947 | memset(irte, 0, sizeof(*irte)); |
| 948 | |
| 949 | irte->present = 1; |
| 950 | irte->dst_mode = apic->irq_dest_mode; |
| 951 | /* |
| 952 | * Trigger mode in the IRTE will always be edge, and for IO-APIC, the |
| 953 | * actual level or edge trigger will be setup in the IO-APIC |
| 954 | * RTE. This will help simplify level triggered irq migration. |
| 955 | * For more details, see the comments (in io_apic.c) explainig IO-APIC |
| 956 | * irq migration in the presence of interrupt-remapping. |
| 957 | */ |
| 958 | irte->trigger_mode = 0; |
| 959 | irte->dlvry_mode = apic->irq_delivery_mode; |
| 960 | irte->vector = vector; |
| 961 | irte->dest_id = IRTE_DEST(dest); |
| 962 | irte->redir_hint = 1; |
| 963 | } |
| 964 | |
| 965 | static int intel_setup_ioapic_entry(int irq, |
| 966 | struct IO_APIC_route_entry *route_entry, |
| 967 | unsigned int destination, int vector, |
| 968 | struct io_apic_irq_attr *attr) |
| 969 | { |
| 970 | int ioapic_id = mpc_ioapic_id(attr->ioapic); |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 971 | struct intel_iommu *iommu; |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 972 | struct IR_IO_APIC_route_entry *entry; |
| 973 | struct irte irte; |
| 974 | int index; |
| 975 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 976 | down_read(&dmar_global_lock); |
| 977 | iommu = map_ioapic_to_ir(ioapic_id); |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 978 | if (!iommu) { |
| 979 | pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 980 | index = -ENODEV; |
| 981 | } else { |
| 982 | index = alloc_irte(iommu, irq, 1); |
| 983 | if (index < 0) { |
| 984 | pr_warn("Failed to allocate IRTE for ioapic %d\n", |
| 985 | ioapic_id); |
| 986 | index = -ENOMEM; |
| 987 | } |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 988 | } |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 989 | up_read(&dmar_global_lock); |
| 990 | if (index < 0) |
| 991 | return index; |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 992 | |
| 993 | prepare_irte(&irte, vector, destination); |
| 994 | |
| 995 | /* Set source-id of interrupt request */ |
| 996 | set_ioapic_sid(&irte, ioapic_id); |
| 997 | |
| 998 | modify_irte(irq, &irte); |
| 999 | |
| 1000 | apic_printk(APIC_VERBOSE, KERN_DEBUG "IOAPIC[%d]: " |
| 1001 | "Set IRTE entry (P:%d FPD:%d Dst_Mode:%d " |
| 1002 | "Redir_hint:%d Trig_Mode:%d Dlvry_Mode:%X " |
| 1003 | "Avail:%X Vector:%02X Dest:%08X " |
| 1004 | "SID:%04X SQ:%X SVT:%X)\n", |
| 1005 | attr->ioapic, irte.present, irte.fpd, irte.dst_mode, |
| 1006 | irte.redir_hint, irte.trigger_mode, irte.dlvry_mode, |
| 1007 | irte.avail, irte.vector, irte.dest_id, |
| 1008 | irte.sid, irte.sq, irte.svt); |
| 1009 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1010 | entry = (struct IR_IO_APIC_route_entry *)route_entry; |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 1011 | memset(entry, 0, sizeof(*entry)); |
| 1012 | |
| 1013 | entry->index2 = (index >> 15) & 0x1; |
| 1014 | entry->zero = 0; |
| 1015 | entry->format = 1; |
| 1016 | entry->index = (index & 0x7fff); |
| 1017 | /* |
| 1018 | * IO-APIC RTE will be configured with virtual vector. |
| 1019 | * irq handler will do the explicit EOI to the io-apic. |
| 1020 | */ |
| 1021 | entry->vector = attr->ioapic_pin; |
| 1022 | entry->mask = 0; /* enable IRQ */ |
| 1023 | entry->trigger = attr->trigger; |
| 1024 | entry->polarity = attr->polarity; |
| 1025 | |
| 1026 | /* Mask level triggered irqs. |
| 1027 | * Use IRQ_DELAYED_DISABLE for edge triggered irqs. |
| 1028 | */ |
| 1029 | if (attr->trigger) |
| 1030 | entry->mask = 1; |
| 1031 | |
| 1032 | return 0; |
| 1033 | } |
| 1034 | |
Joerg Roedel | 4c1bad6 | 2012-03-30 11:47:03 -0700 | [diff] [blame] | 1035 | /* |
| 1036 | * Migrate the IO-APIC irq in the presence of intr-remapping. |
| 1037 | * |
| 1038 | * For both level and edge triggered, irq migration is a simple atomic |
| 1039 | * update(of vector and cpu destination) of IRTE and flush the hardware cache. |
| 1040 | * |
| 1041 | * For level triggered, we eliminate the io-apic RTE modification (with the |
| 1042 | * updated vector information), by using a virtual vector (io-apic pin number). |
| 1043 | * Real vector that is used for interrupting cpu will be coming from |
| 1044 | * the interrupt-remapping table entry. |
| 1045 | * |
| 1046 | * As the migration is a simple atomic update of IRTE, the same mechanism |
| 1047 | * is used to migrate MSI irq's in the presence of interrupt-remapping. |
| 1048 | */ |
| 1049 | static int |
| 1050 | intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, |
| 1051 | bool force) |
| 1052 | { |
Jiang Liu | 91411da | 2014-10-27 16:12:09 +0800 | [diff] [blame] | 1053 | struct irq_cfg *cfg = irqd_cfg(data); |
Joerg Roedel | 4c1bad6 | 2012-03-30 11:47:03 -0700 | [diff] [blame] | 1054 | unsigned int dest, irq = data->irq; |
| 1055 | struct irte irte; |
Alexander Gordeev | ff16432 | 2012-06-07 15:15:59 +0200 | [diff] [blame] | 1056 | int err; |
Joerg Roedel | 4c1bad6 | 2012-03-30 11:47:03 -0700 | [diff] [blame] | 1057 | |
Suresh Siddha | 7eb9ae0 | 2012-06-14 18:28:49 -0700 | [diff] [blame] | 1058 | if (!config_enabled(CONFIG_SMP)) |
| 1059 | return -EINVAL; |
| 1060 | |
Joerg Roedel | 4c1bad6 | 2012-03-30 11:47:03 -0700 | [diff] [blame] | 1061 | if (!cpumask_intersects(mask, cpu_online_mask)) |
| 1062 | return -EINVAL; |
| 1063 | |
| 1064 | if (get_irte(irq, &irte)) |
| 1065 | return -EBUSY; |
| 1066 | |
Alexander Gordeev | ff16432 | 2012-06-07 15:15:59 +0200 | [diff] [blame] | 1067 | err = assign_irq_vector(irq, cfg, mask); |
| 1068 | if (err) |
| 1069 | return err; |
Joerg Roedel | 4c1bad6 | 2012-03-30 11:47:03 -0700 | [diff] [blame] | 1070 | |
Alexander Gordeev | ff16432 | 2012-06-07 15:15:59 +0200 | [diff] [blame] | 1071 | err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest); |
| 1072 | if (err) { |
Dan Carpenter | ed88bed | 2012-06-12 19:26:33 +0300 | [diff] [blame] | 1073 | if (assign_irq_vector(irq, cfg, data->affinity)) |
Alexander Gordeev | ff16432 | 2012-06-07 15:15:59 +0200 | [diff] [blame] | 1074 | pr_err("Failed to recover vector for irq %d\n", irq); |
| 1075 | return err; |
| 1076 | } |
Joerg Roedel | 4c1bad6 | 2012-03-30 11:47:03 -0700 | [diff] [blame] | 1077 | |
| 1078 | irte.vector = cfg->vector; |
| 1079 | irte.dest_id = IRTE_DEST(dest); |
| 1080 | |
| 1081 | /* |
| 1082 | * Atomically updates the IRTE with the new destination, vector |
| 1083 | * and flushes the interrupt entry cache. |
| 1084 | */ |
| 1085 | modify_irte(irq, &irte); |
| 1086 | |
| 1087 | /* |
| 1088 | * After this point, all the interrupts will start arriving |
| 1089 | * at the new destination. So, time to cleanup the previous |
| 1090 | * vector allocation. |
| 1091 | */ |
| 1092 | if (cfg->move_in_progress) |
| 1093 | send_cleanup_vector(cfg); |
| 1094 | |
| 1095 | cpumask_copy(data->affinity, mask); |
| 1096 | return 0; |
| 1097 | } |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 1098 | |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1099 | static void intel_compose_msi_msg(struct pci_dev *pdev, |
| 1100 | unsigned int irq, unsigned int dest, |
| 1101 | struct msi_msg *msg, u8 hpet_id) |
| 1102 | { |
| 1103 | struct irq_cfg *cfg; |
| 1104 | struct irte irte; |
Suresh Siddha | c558df4 | 2012-05-08 00:08:54 -0700 | [diff] [blame] | 1105 | u16 sub_handle = 0; |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1106 | int ir_index; |
| 1107 | |
Jiang Liu | 91411da | 2014-10-27 16:12:09 +0800 | [diff] [blame] | 1108 | cfg = irq_cfg(irq); |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1109 | |
| 1110 | ir_index = map_irq_to_irte_handle(irq, &sub_handle); |
| 1111 | BUG_ON(ir_index == -1); |
| 1112 | |
| 1113 | prepare_irte(&irte, cfg->vector, dest); |
| 1114 | |
| 1115 | /* Set source-id of interrupt request */ |
| 1116 | if (pdev) |
| 1117 | set_msi_sid(&irte, pdev); |
| 1118 | else |
| 1119 | set_hpet_sid(&irte, hpet_id); |
| 1120 | |
| 1121 | modify_irte(irq, &irte); |
| 1122 | |
| 1123 | msg->address_hi = MSI_ADDR_BASE_HI; |
| 1124 | msg->data = sub_handle; |
| 1125 | msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT | |
| 1126 | MSI_ADDR_IR_SHV | |
| 1127 | MSI_ADDR_IR_INDEX1(ir_index) | |
| 1128 | MSI_ADDR_IR_INDEX2(ir_index); |
| 1129 | } |
| 1130 | |
| 1131 | /* |
| 1132 | * Map the PCI dev to the corresponding remapping hardware unit |
| 1133 | * and allocate 'nvec' consecutive interrupt-remapping table entries |
| 1134 | * in it. |
| 1135 | */ |
| 1136 | static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) |
| 1137 | { |
| 1138 | struct intel_iommu *iommu; |
| 1139 | int index; |
| 1140 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1141 | down_read(&dmar_global_lock); |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1142 | iommu = map_dev_to_ir(dev); |
| 1143 | if (!iommu) { |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 1144 | pr_err("Unable to map PCI %s to iommu\n", pci_name(dev)); |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1145 | index = -ENOENT; |
| 1146 | } else { |
| 1147 | index = alloc_irte(iommu, irq, nvec); |
| 1148 | if (index < 0) { |
Joerg Roedel | 9f10e5b | 2015-06-12 09:57:06 +0200 | [diff] [blame] | 1149 | pr_err("Unable to allocate %d IRTE for PCI %s\n", |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1150 | nvec, pci_name(dev)); |
| 1151 | index = -ENOSPC; |
| 1152 | } |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1153 | } |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1154 | up_read(&dmar_global_lock); |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1155 | |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1156 | return index; |
| 1157 | } |
| 1158 | |
| 1159 | static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, |
| 1160 | int index, int sub_handle) |
| 1161 | { |
| 1162 | struct intel_iommu *iommu; |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1163 | int ret = -ENOENT; |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1164 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1165 | down_read(&dmar_global_lock); |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1166 | iommu = map_dev_to_ir(pdev); |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1167 | if (iommu) { |
| 1168 | /* |
| 1169 | * setup the mapping between the irq and the IRTE |
| 1170 | * base index, the sub_handle pointing to the |
| 1171 | * appropriate interrupt remap table entry. |
| 1172 | */ |
| 1173 | set_irte_irq(irq, iommu, index, sub_handle); |
| 1174 | ret = 0; |
| 1175 | } |
| 1176 | up_read(&dmar_global_lock); |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1177 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1178 | return ret; |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1179 | } |
| 1180 | |
Yijing Wang | 5fc24d8 | 2014-09-17 17:32:19 +0800 | [diff] [blame] | 1181 | static int intel_alloc_hpet_msi(unsigned int irq, unsigned int id) |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1182 | { |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1183 | int ret = -1; |
| 1184 | struct intel_iommu *iommu; |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1185 | int index; |
| 1186 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1187 | down_read(&dmar_global_lock); |
| 1188 | iommu = map_hpet_to_ir(id); |
| 1189 | if (iommu) { |
| 1190 | index = alloc_irte(iommu, irq, 1); |
| 1191 | if (index >= 0) |
| 1192 | ret = 0; |
| 1193 | } |
| 1194 | up_read(&dmar_global_lock); |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1195 | |
Jiang Liu | 3a5670e | 2014-02-19 14:07:33 +0800 | [diff] [blame] | 1196 | return ret; |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1197 | } |
| 1198 | |
Joerg Roedel | 736baef | 2012-03-30 11:47:00 -0700 | [diff] [blame] | 1199 | struct irq_remap_ops intel_irq_remap_ops = { |
Thomas Gleixner | 1119030 | 2015-01-07 15:31:29 +0800 | [diff] [blame] | 1200 | .prepare = intel_prepare_irq_remapping, |
Suresh Siddha | 95a02e9 | 2012-03-30 11:47:07 -0700 | [diff] [blame] | 1201 | .enable = intel_enable_irq_remapping, |
| 1202 | .disable = disable_irq_remapping, |
| 1203 | .reenable = reenable_irq_remapping, |
Joerg Roedel | 4f3d8b6 | 2012-03-30 11:47:01 -0700 | [diff] [blame] | 1204 | .enable_faulting = enable_drhd_fault_handling, |
Joerg Roedel | 0c3f173 | 2012-03-30 11:47:02 -0700 | [diff] [blame] | 1205 | .setup_ioapic_entry = intel_setup_ioapic_entry, |
Joerg Roedel | 4c1bad6 | 2012-03-30 11:47:03 -0700 | [diff] [blame] | 1206 | .set_affinity = intel_ioapic_set_affinity, |
Joerg Roedel | 9d619f6 | 2012-03-30 11:47:04 -0700 | [diff] [blame] | 1207 | .free_irq = free_irte, |
Joerg Roedel | 5e2b930 | 2012-03-30 11:47:05 -0700 | [diff] [blame] | 1208 | .compose_msi_msg = intel_compose_msi_msg, |
| 1209 | .msi_alloc_irq = intel_msi_alloc_irq, |
| 1210 | .msi_setup_irq = intel_msi_setup_irq, |
Yijing Wang | 5fc24d8 | 2014-09-17 17:32:19 +0800 | [diff] [blame] | 1211 | .alloc_hpet_msi = intel_alloc_hpet_msi, |
Joerg Roedel | 736baef | 2012-03-30 11:47:00 -0700 | [diff] [blame] | 1212 | }; |
Jiang Liu | 6b19724 | 2014-11-09 22:47:58 +0800 | [diff] [blame] | 1213 | |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 1214 | /* |
| 1215 | * Support of Interrupt Remapping Unit Hotplug |
| 1216 | */ |
| 1217 | static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) |
| 1218 | { |
| 1219 | int ret; |
| 1220 | int eim = x2apic_enabled(); |
| 1221 | |
| 1222 | if (eim && !ecap_eim_support(iommu->ecap)) { |
| 1223 | pr_info("DRHD %Lx: EIM not supported by DRHD, ecap %Lx\n", |
| 1224 | iommu->reg_phys, iommu->ecap); |
| 1225 | return -ENODEV; |
| 1226 | } |
| 1227 | |
| 1228 | if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) { |
| 1229 | pr_warn("DRHD %Lx: failed to parse managed IOAPIC/HPET\n", |
| 1230 | iommu->reg_phys); |
| 1231 | return -ENODEV; |
| 1232 | } |
| 1233 | |
| 1234 | /* TODO: check all IOAPICs are covered by IOMMU */ |
| 1235 | |
| 1236 | /* Setup Interrupt-remapping now. */ |
| 1237 | ret = intel_setup_irq_remapping(iommu); |
| 1238 | if (ret) { |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 1239 | pr_err("Failed to setup irq remapping for %s\n", |
| 1240 | iommu->name); |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 1241 | intel_teardown_irq_remapping(iommu); |
| 1242 | ir_remove_ioapic_hpet_scope(iommu); |
Joerg Roedel | 9e4e49d | 2015-06-12 14:23:56 +0200 | [diff] [blame^] | 1243 | } else { |
| 1244 | iommu_set_irq_remapping(iommu, eim); |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 1245 | } |
| 1246 | |
| 1247 | return ret; |
| 1248 | } |
| 1249 | |
Jiang Liu | 6b19724 | 2014-11-09 22:47:58 +0800 | [diff] [blame] | 1250 | int dmar_ir_hotplug(struct dmar_drhd_unit *dmaru, bool insert) |
| 1251 | { |
Jiang Liu | a7a3dad | 2014-11-09 22:48:00 +0800 | [diff] [blame] | 1252 | int ret = 0; |
| 1253 | struct intel_iommu *iommu = dmaru->iommu; |
| 1254 | |
| 1255 | if (!irq_remapping_enabled) |
| 1256 | return 0; |
| 1257 | if (iommu == NULL) |
| 1258 | return -EINVAL; |
| 1259 | if (!ecap_ir_support(iommu->ecap)) |
| 1260 | return 0; |
| 1261 | |
| 1262 | if (insert) { |
| 1263 | if (!iommu->ir_table) |
| 1264 | ret = dmar_ir_add(dmaru, iommu); |
| 1265 | } else { |
| 1266 | if (iommu->ir_table) { |
| 1267 | if (!bitmap_empty(iommu->ir_table->bitmap, |
| 1268 | INTR_REMAP_TABLE_ENTRIES)) { |
| 1269 | ret = -EBUSY; |
| 1270 | } else { |
| 1271 | iommu_disable_irq_remapping(iommu); |
| 1272 | intel_teardown_irq_remapping(iommu); |
| 1273 | ir_remove_ioapic_hpet_scope(iommu); |
| 1274 | } |
| 1275 | } |
| 1276 | } |
| 1277 | |
| 1278 | return ret; |
Jiang Liu | 6b19724 | 2014-11-09 22:47:58 +0800 | [diff] [blame] | 1279 | } |