Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 1 | /* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 2 | * |
| 3 | * This program is free software; you can redistribute it and/or modify |
| 4 | * it under the terms of the GNU General Public License version 2 and |
| 5 | * only version 2 as published by the Free Software Foundation. |
| 6 | * |
| 7 | * This program is distributed in the hope that it will be useful, |
| 8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 10 | * GNU General Public License for more details. |
| 11 | * |
| 12 | * You should have received a copy of the GNU General Public License |
| 13 | * along with this program; if not, write to the Free Software |
| 14 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
| 15 | * 02110-1301, USA. |
| 16 | */ |
| 17 | |
| 18 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| 19 | #include <linux/kernel.h> |
| 20 | #include <linux/module.h> |
| 21 | #include <linux/platform_device.h> |
| 22 | #include <linux/errno.h> |
| 23 | #include <linux/io.h> |
| 24 | #include <linux/interrupt.h> |
| 25 | #include <linux/list.h> |
| 26 | #include <linux/spinlock.h> |
| 27 | #include <linux/slab.h> |
| 28 | #include <linux/iommu.h> |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 29 | #include <linux/clk.h> |
Sricharan R | f7f125e | 2016-06-13 17:06:04 +0530 | [diff] [blame^] | 30 | #include <linux/err.h> |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 31 | |
| 32 | #include <asm/cacheflush.h> |
| 33 | #include <asm/sizes.h> |
| 34 | |
Stephen Boyd | 0b559df | 2013-07-24 13:54:33 -0700 | [diff] [blame] | 35 | #include "msm_iommu_hw-8xxx.h" |
| 36 | #include "msm_iommu.h" |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 37 | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 38 | #define MRC(reg, processor, op1, crn, crm, op2) \ |
| 39 | __asm__ __volatile__ ( \ |
| 40 | " mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ |
| 41 | : "=r" (reg)) |
| 42 | |
| 43 | #define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) |
| 44 | #define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) |
| 45 | |
Ohad Ben-Cohen | 8342727 | 2011-11-10 11:32:28 +0200 | [diff] [blame] | 46 | /* bitmap of the page sizes currently supported */ |
| 47 | #define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) |
| 48 | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 49 | static int msm_iommu_tex_class[4]; |
| 50 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 51 | DEFINE_SPINLOCK(msm_iommu_lock); |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 52 | static LIST_HEAD(qcom_iommu_devices); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 53 | |
| 54 | struct msm_priv { |
| 55 | unsigned long *pgtable; |
| 56 | struct list_head list_attached; |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 57 | struct iommu_domain domain; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 58 | }; |
| 59 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 60 | static struct msm_priv *to_msm_priv(struct iommu_domain *dom) |
| 61 | { |
| 62 | return container_of(dom, struct msm_priv, domain); |
| 63 | } |
| 64 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 65 | static int __enable_clocks(struct msm_iommu_dev *iommu) |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 66 | { |
| 67 | int ret; |
| 68 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 69 | ret = clk_enable(iommu->pclk); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 70 | if (ret) |
| 71 | goto fail; |
| 72 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 73 | if (iommu->clk) { |
| 74 | ret = clk_enable(iommu->clk); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 75 | if (ret) |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 76 | clk_disable(iommu->pclk); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 77 | } |
| 78 | fail: |
| 79 | return ret; |
| 80 | } |
| 81 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 82 | static void __disable_clocks(struct msm_iommu_dev *iommu) |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 83 | { |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 84 | if (iommu->clk) |
| 85 | clk_disable(iommu->clk); |
| 86 | clk_disable(iommu->pclk); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 87 | } |
| 88 | |
Sricharan R | f7f125e | 2016-06-13 17:06:04 +0530 | [diff] [blame^] | 89 | static void msm_iommu_reset(void __iomem *base, int ncb) |
| 90 | { |
| 91 | int ctx; |
| 92 | |
| 93 | SET_RPUE(base, 0); |
| 94 | SET_RPUEIE(base, 0); |
| 95 | SET_ESRRESTORE(base, 0); |
| 96 | SET_TBE(base, 0); |
| 97 | SET_CR(base, 0); |
| 98 | SET_SPDMBE(base, 0); |
| 99 | SET_TESTBUSCR(base, 0); |
| 100 | SET_TLBRSW(base, 0); |
| 101 | SET_GLOBAL_TLBIALL(base, 0); |
| 102 | SET_RPU_ACR(base, 0); |
| 103 | SET_TLBLKCRWE(base, 1); |
| 104 | |
| 105 | for (ctx = 0; ctx < ncb; ctx++) { |
| 106 | SET_BPRCOSH(base, ctx, 0); |
| 107 | SET_BPRCISH(base, ctx, 0); |
| 108 | SET_BPRCNSH(base, ctx, 0); |
| 109 | SET_BPSHCFG(base, ctx, 0); |
| 110 | SET_BPMTCFG(base, ctx, 0); |
| 111 | SET_ACTLR(base, ctx, 0); |
| 112 | SET_SCTLR(base, ctx, 0); |
| 113 | SET_FSRRESTORE(base, ctx, 0); |
| 114 | SET_TTBR0(base, ctx, 0); |
| 115 | SET_TTBR1(base, ctx, 0); |
| 116 | SET_TTBCR(base, ctx, 0); |
| 117 | SET_BFBCR(base, ctx, 0); |
| 118 | SET_PAR(base, ctx, 0); |
| 119 | SET_FAR(base, ctx, 0); |
| 120 | SET_CTX_TLBIALL(base, ctx, 0); |
| 121 | SET_TLBFLPTER(base, ctx, 0); |
| 122 | SET_TLBSLPTER(base, ctx, 0); |
| 123 | SET_TLBLKCR(base, ctx, 0); |
| 124 | SET_PRRR(base, ctx, 0); |
| 125 | SET_NMRR(base, ctx, 0); |
| 126 | SET_CONTEXTIDR(base, ctx, 0); |
| 127 | } |
| 128 | } |
| 129 | |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 130 | static int __flush_iotlb(struct iommu_domain *domain) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 131 | { |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 132 | struct msm_priv *priv = to_msm_priv(domain); |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 133 | struct msm_iommu_dev *iommu = NULL; |
| 134 | struct msm_iommu_ctx_dev *master; |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 135 | int ret = 0; |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 136 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 137 | #ifndef CONFIG_IOMMU_PGTABLES_L2 |
| 138 | unsigned long *fl_table = priv->pgtable; |
| 139 | int i; |
| 140 | |
Stepan Moskovchenko | f6f41eb | 2010-11-12 19:29:54 -0800 | [diff] [blame] | 141 | if (!list_empty(&priv->list_attached)) { |
| 142 | dmac_flush_range(fl_table, fl_table + SZ_16K); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 143 | |
Stepan Moskovchenko | f6f41eb | 2010-11-12 19:29:54 -0800 | [diff] [blame] | 144 | for (i = 0; i < NUM_FL_PTE; i++) |
| 145 | if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { |
| 146 | void *sl_table = __va(fl_table[i] & |
| 147 | FL_BASE_MASK); |
| 148 | dmac_flush_range(sl_table, sl_table + SZ_4K); |
| 149 | } |
| 150 | } |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 151 | #endif |
| 152 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 153 | list_for_each_entry(iommu, &priv->list_attached, dom_node) { |
| 154 | ret = __enable_clocks(iommu); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 155 | if (ret) |
| 156 | goto fail; |
| 157 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 158 | list_for_each_entry(master, &iommu->ctx_list, list) |
| 159 | SET_CTX_TLBIALL(iommu->base, master->num, 0); |
| 160 | |
| 161 | __disable_clocks(iommu); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 162 | } |
| 163 | fail: |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 164 | return ret; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 165 | } |
| 166 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 167 | static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end) |
| 168 | { |
| 169 | int idx; |
| 170 | |
| 171 | do { |
| 172 | idx = find_next_zero_bit(map, end, start); |
| 173 | if (idx == end) |
| 174 | return -ENOSPC; |
| 175 | } while (test_and_set_bit(idx, map)); |
| 176 | |
| 177 | return idx; |
| 178 | } |
| 179 | |
| 180 | static void msm_iommu_free_ctx(unsigned long *map, int idx) |
| 181 | { |
| 182 | clear_bit(idx, map); |
| 183 | } |
| 184 | |
| 185 | static void config_mids(struct msm_iommu_dev *iommu, |
| 186 | struct msm_iommu_ctx_dev *master) |
| 187 | { |
| 188 | int mid, ctx, i; |
| 189 | |
| 190 | for (i = 0; i < master->num_mids; i++) { |
| 191 | mid = master->mids[i]; |
| 192 | ctx = master->num; |
| 193 | |
| 194 | SET_M2VCBR_N(iommu->base, mid, 0); |
| 195 | SET_CBACR_N(iommu->base, ctx, 0); |
| 196 | |
| 197 | /* Set VMID = 0 */ |
| 198 | SET_VMID(iommu->base, mid, 0); |
| 199 | |
| 200 | /* Set the context number for that MID to this context */ |
| 201 | SET_CBNDX(iommu->base, mid, ctx); |
| 202 | |
| 203 | /* Set MID associated with this context bank to 0*/ |
| 204 | SET_CBVMID(iommu->base, ctx, 0); |
| 205 | |
| 206 | /* Set the ASID for TLB tagging for this context */ |
| 207 | SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx); |
| 208 | |
| 209 | /* Set security bit override to be Non-secure */ |
| 210 | SET_NSCFG(iommu->base, mid, 3); |
| 211 | } |
| 212 | } |
| 213 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 214 | static void __reset_context(void __iomem *base, int ctx) |
| 215 | { |
| 216 | SET_BPRCOSH(base, ctx, 0); |
| 217 | SET_BPRCISH(base, ctx, 0); |
| 218 | SET_BPRCNSH(base, ctx, 0); |
| 219 | SET_BPSHCFG(base, ctx, 0); |
| 220 | SET_BPMTCFG(base, ctx, 0); |
| 221 | SET_ACTLR(base, ctx, 0); |
| 222 | SET_SCTLR(base, ctx, 0); |
| 223 | SET_FSRRESTORE(base, ctx, 0); |
| 224 | SET_TTBR0(base, ctx, 0); |
| 225 | SET_TTBR1(base, ctx, 0); |
| 226 | SET_TTBCR(base, ctx, 0); |
| 227 | SET_BFBCR(base, ctx, 0); |
| 228 | SET_PAR(base, ctx, 0); |
| 229 | SET_FAR(base, ctx, 0); |
| 230 | SET_CTX_TLBIALL(base, ctx, 0); |
| 231 | SET_TLBFLPTER(base, ctx, 0); |
| 232 | SET_TLBSLPTER(base, ctx, 0); |
| 233 | SET_TLBLKCR(base, ctx, 0); |
| 234 | SET_PRRR(base, ctx, 0); |
| 235 | SET_NMRR(base, ctx, 0); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 236 | } |
| 237 | |
| 238 | static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) |
| 239 | { |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 240 | unsigned int prrr, nmrr; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 241 | __reset_context(base, ctx); |
| 242 | |
| 243 | /* Set up HTW mode */ |
| 244 | /* TLB miss configuration: perform HTW on miss */ |
| 245 | SET_TLBMCFG(base, ctx, 0x3); |
| 246 | |
| 247 | /* V2P configuration: HTW for access */ |
| 248 | SET_V2PCFG(base, ctx, 0x3); |
| 249 | |
| 250 | SET_TTBCR(base, ctx, 0); |
| 251 | SET_TTBR0_PA(base, ctx, (pgtable >> 14)); |
| 252 | |
| 253 | /* Invalidate the TLB for this context */ |
| 254 | SET_CTX_TLBIALL(base, ctx, 0); |
| 255 | |
| 256 | /* Set interrupt number to "secure" interrupt */ |
| 257 | SET_IRPTNDX(base, ctx, 0); |
| 258 | |
| 259 | /* Enable context fault interrupt */ |
| 260 | SET_CFEIE(base, ctx, 1); |
| 261 | |
| 262 | /* Stall access on a context fault and let the handler deal with it */ |
| 263 | SET_CFCFG(base, ctx, 1); |
| 264 | |
| 265 | /* Redirect all cacheable requests to L2 slave port. */ |
| 266 | SET_RCISH(base, ctx, 1); |
| 267 | SET_RCOSH(base, ctx, 1); |
| 268 | SET_RCNSH(base, ctx, 1); |
| 269 | |
| 270 | /* Turn on TEX Remap */ |
| 271 | SET_TRE(base, ctx, 1); |
| 272 | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 273 | /* Set TEX remap attributes */ |
| 274 | RCP15_PRRR(prrr); |
| 275 | RCP15_NMRR(nmrr); |
| 276 | SET_PRRR(base, ctx, prrr); |
| 277 | SET_NMRR(base, ctx, nmrr); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 278 | |
| 279 | /* Turn on BFB prefetch */ |
| 280 | SET_BFBDFE(base, ctx, 1); |
| 281 | |
| 282 | #ifdef CONFIG_IOMMU_PGTABLES_L2 |
| 283 | /* Configure page tables as inner-cacheable and shareable to reduce |
| 284 | * the TLB miss penalty. |
| 285 | */ |
| 286 | SET_TTBR0_SH(base, ctx, 1); |
| 287 | SET_TTBR1_SH(base, ctx, 1); |
| 288 | |
| 289 | SET_TTBR0_NOS(base, ctx, 1); |
| 290 | SET_TTBR1_NOS(base, ctx, 1); |
| 291 | |
| 292 | SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */ |
| 293 | SET_TTBR0_IRGNL(base, ctx, 1); |
| 294 | |
| 295 | SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */ |
| 296 | SET_TTBR1_IRGNL(base, ctx, 1); |
| 297 | |
| 298 | SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */ |
| 299 | SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */ |
| 300 | #endif |
| 301 | |
| 302 | /* Enable the MMU */ |
| 303 | SET_M(base, ctx, 1); |
| 304 | } |
| 305 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 306 | static struct iommu_domain *msm_iommu_domain_alloc(unsigned type) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 307 | { |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 308 | struct msm_priv *priv; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 309 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 310 | if (type != IOMMU_DOMAIN_UNMANAGED) |
| 311 | return NULL; |
| 312 | |
| 313 | priv = kzalloc(sizeof(*priv), GFP_KERNEL); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 314 | if (!priv) |
| 315 | goto fail_nomem; |
| 316 | |
| 317 | INIT_LIST_HEAD(&priv->list_attached); |
| 318 | priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL, |
| 319 | get_order(SZ_16K)); |
| 320 | |
| 321 | if (!priv->pgtable) |
| 322 | goto fail_nomem; |
| 323 | |
| 324 | memset(priv->pgtable, 0, SZ_16K); |
Joerg Roedel | 4be6a29 | 2012-01-26 19:40:56 +0100 | [diff] [blame] | 325 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 326 | priv->domain.geometry.aperture_start = 0; |
| 327 | priv->domain.geometry.aperture_end = (1ULL << 32) - 1; |
| 328 | priv->domain.geometry.force_aperture = true; |
Joerg Roedel | 4be6a29 | 2012-01-26 19:40:56 +0100 | [diff] [blame] | 329 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 330 | return &priv->domain; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 331 | |
| 332 | fail_nomem: |
| 333 | kfree(priv); |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 334 | return NULL; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 335 | } |
| 336 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 337 | static void msm_iommu_domain_free(struct iommu_domain *domain) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 338 | { |
| 339 | struct msm_priv *priv; |
| 340 | unsigned long flags; |
| 341 | unsigned long *fl_table; |
| 342 | int i; |
| 343 | |
| 344 | spin_lock_irqsave(&msm_iommu_lock, flags); |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 345 | priv = to_msm_priv(domain); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 346 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 347 | fl_table = priv->pgtable; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 348 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 349 | for (i = 0; i < NUM_FL_PTE; i++) |
| 350 | if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) |
| 351 | free_page((unsigned long) __va(((fl_table[i]) & |
| 352 | FL_BASE_MASK))); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 353 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 354 | free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); |
| 355 | priv->pgtable = NULL; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 356 | |
| 357 | kfree(priv); |
| 358 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
| 359 | } |
| 360 | |
| 361 | static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) |
| 362 | { |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 363 | int ret = 0; |
| 364 | unsigned long flags; |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 365 | struct msm_iommu_dev *iommu; |
| 366 | struct msm_priv *priv = to_msm_priv(domain); |
| 367 | struct msm_iommu_ctx_dev *master; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 368 | |
| 369 | spin_lock_irqsave(&msm_iommu_lock, flags); |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 370 | list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) { |
| 371 | master = list_first_entry(&iommu->ctx_list, |
| 372 | struct msm_iommu_ctx_dev, |
| 373 | list); |
| 374 | if (master->of_node == dev->of_node) { |
| 375 | ret = __enable_clocks(iommu); |
| 376 | if (ret) |
| 377 | goto fail; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 378 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 379 | list_for_each_entry(master, &iommu->ctx_list, list) { |
| 380 | if (master->num) { |
| 381 | dev_err(dev, "domain already attached"); |
| 382 | ret = -EEXIST; |
| 383 | goto fail; |
| 384 | } |
| 385 | master->num = |
| 386 | msm_iommu_alloc_ctx(iommu->context_map, |
| 387 | 0, iommu->ncb); |
| 388 | if (IS_ERR_VALUE(master->num)) { |
| 389 | ret = -ENODEV; |
| 390 | goto fail; |
| 391 | } |
| 392 | config_mids(iommu, master); |
| 393 | __program_context(iommu->base, master->num, |
| 394 | __pa(priv->pgtable)); |
| 395 | } |
| 396 | __disable_clocks(iommu); |
| 397 | list_add(&iommu->dom_node, &priv->list_attached); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 398 | } |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 399 | } |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 400 | |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 401 | ret = __flush_iotlb(domain); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 402 | fail: |
| 403 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 404 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 405 | return ret; |
| 406 | } |
| 407 | |
| 408 | static void msm_iommu_detach_dev(struct iommu_domain *domain, |
| 409 | struct device *dev) |
| 410 | { |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 411 | struct msm_priv *priv = to_msm_priv(domain); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 412 | unsigned long flags; |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 413 | struct msm_iommu_dev *iommu; |
| 414 | struct msm_iommu_ctx_dev *master; |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 415 | int ret; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 416 | |
| 417 | spin_lock_irqsave(&msm_iommu_lock, flags); |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 418 | ret = __flush_iotlb(domain); |
| 419 | if (ret) |
| 420 | goto fail; |
| 421 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 422 | list_for_each_entry(iommu, &priv->list_attached, dom_node) { |
| 423 | ret = __enable_clocks(iommu); |
| 424 | if (ret) |
| 425 | goto fail; |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 426 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 427 | list_for_each_entry(master, &iommu->ctx_list, list) { |
| 428 | msm_iommu_free_ctx(iommu->context_map, master->num); |
| 429 | __reset_context(iommu->base, master->num); |
| 430 | } |
| 431 | __disable_clocks(iommu); |
| 432 | } |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 433 | fail: |
| 434 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
| 435 | } |
| 436 | |
| 437 | static int msm_iommu_map(struct iommu_domain *domain, unsigned long va, |
Ohad Ben-Cohen | 5009065 | 2011-11-10 11:32:25 +0200 | [diff] [blame] | 438 | phys_addr_t pa, size_t len, int prot) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 439 | { |
| 440 | struct msm_priv *priv; |
| 441 | unsigned long flags; |
| 442 | unsigned long *fl_table; |
| 443 | unsigned long *fl_pte; |
| 444 | unsigned long fl_offset; |
| 445 | unsigned long *sl_table; |
| 446 | unsigned long *sl_pte; |
| 447 | unsigned long sl_offset; |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 448 | unsigned int pgprot; |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 449 | int ret = 0, tex, sh; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 450 | |
| 451 | spin_lock_irqsave(&msm_iommu_lock, flags); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 452 | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 453 | sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0; |
| 454 | tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK]; |
| 455 | |
| 456 | if (tex < 0 || tex > NUM_TEX_CLASS - 1) { |
| 457 | ret = -EINVAL; |
| 458 | goto fail; |
| 459 | } |
| 460 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 461 | priv = to_msm_priv(domain); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 462 | |
| 463 | fl_table = priv->pgtable; |
| 464 | |
| 465 | if (len != SZ_16M && len != SZ_1M && |
| 466 | len != SZ_64K && len != SZ_4K) { |
| 467 | pr_debug("Bad size: %d\n", len); |
| 468 | ret = -EINVAL; |
| 469 | goto fail; |
| 470 | } |
| 471 | |
| 472 | if (!fl_table) { |
| 473 | pr_debug("Null page table\n"); |
| 474 | ret = -EINVAL; |
| 475 | goto fail; |
| 476 | } |
| 477 | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 478 | if (len == SZ_16M || len == SZ_1M) { |
| 479 | pgprot = sh ? FL_SHARED : 0; |
| 480 | pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0; |
| 481 | pgprot |= tex & 0x02 ? FL_CACHEABLE : 0; |
| 482 | pgprot |= tex & 0x04 ? FL_TEX0 : 0; |
| 483 | } else { |
| 484 | pgprot = sh ? SL_SHARED : 0; |
| 485 | pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0; |
| 486 | pgprot |= tex & 0x02 ? SL_CACHEABLE : 0; |
| 487 | pgprot |= tex & 0x04 ? SL_TEX0 : 0; |
| 488 | } |
| 489 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 490 | fl_offset = FL_OFFSET(va); /* Upper 12 bits */ |
| 491 | fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ |
| 492 | |
| 493 | if (len == SZ_16M) { |
| 494 | int i = 0; |
| 495 | for (i = 0; i < 16; i++) |
| 496 | *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION | |
| 497 | FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT | |
Stepan Moskovchenko | 2e8c8ba | 2011-02-24 18:00:41 -0800 | [diff] [blame] | 498 | FL_SHARED | FL_NG | pgprot; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 499 | } |
| 500 | |
| 501 | if (len == SZ_1M) |
Stepan Moskovchenko | 2e8c8ba | 2011-02-24 18:00:41 -0800 | [diff] [blame] | 502 | *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 503 | FL_TYPE_SECT | FL_SHARED | pgprot; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 504 | |
| 505 | /* Need a 2nd level table */ |
| 506 | if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) { |
| 507 | unsigned long *sl; |
Stepan Moskovchenko | 294b2de | 2010-12-10 14:12:03 -0800 | [diff] [blame] | 508 | sl = (unsigned long *) __get_free_pages(GFP_ATOMIC, |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 509 | get_order(SZ_4K)); |
| 510 | |
| 511 | if (!sl) { |
| 512 | pr_debug("Could not allocate second level table\n"); |
| 513 | ret = -ENOMEM; |
| 514 | goto fail; |
| 515 | } |
| 516 | |
| 517 | memset(sl, 0, SZ_4K); |
| 518 | *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE); |
| 519 | } |
| 520 | |
| 521 | sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); |
| 522 | sl_offset = SL_OFFSET(va); |
| 523 | sl_pte = sl_table + sl_offset; |
| 524 | |
| 525 | |
| 526 | if (len == SZ_4K) |
Stepan Moskovchenko | 2e8c8ba | 2011-02-24 18:00:41 -0800 | [diff] [blame] | 527 | *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 528 | SL_SHARED | SL_TYPE_SMALL | pgprot; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 529 | |
| 530 | if (len == SZ_64K) { |
| 531 | int i; |
| 532 | |
| 533 | for (i = 0; i < 16; i++) |
| 534 | *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | |
Stepan Moskovchenko | 2e8c8ba | 2011-02-24 18:00:41 -0800 | [diff] [blame] | 535 | SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 536 | } |
| 537 | |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 538 | ret = __flush_iotlb(domain); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 539 | fail: |
| 540 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
| 541 | return ret; |
| 542 | } |
| 543 | |
Ohad Ben-Cohen | 5009065 | 2011-11-10 11:32:25 +0200 | [diff] [blame] | 544 | static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, |
| 545 | size_t len) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 546 | { |
| 547 | struct msm_priv *priv; |
| 548 | unsigned long flags; |
| 549 | unsigned long *fl_table; |
| 550 | unsigned long *fl_pte; |
| 551 | unsigned long fl_offset; |
| 552 | unsigned long *sl_table; |
| 553 | unsigned long *sl_pte; |
| 554 | unsigned long sl_offset; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 555 | int i, ret = 0; |
| 556 | |
| 557 | spin_lock_irqsave(&msm_iommu_lock, flags); |
| 558 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 559 | priv = to_msm_priv(domain); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 560 | |
| 561 | fl_table = priv->pgtable; |
| 562 | |
| 563 | if (len != SZ_16M && len != SZ_1M && |
| 564 | len != SZ_64K && len != SZ_4K) { |
| 565 | pr_debug("Bad length: %d\n", len); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 566 | goto fail; |
| 567 | } |
| 568 | |
| 569 | if (!fl_table) { |
| 570 | pr_debug("Null page table\n"); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 571 | goto fail; |
| 572 | } |
| 573 | |
| 574 | fl_offset = FL_OFFSET(va); /* Upper 12 bits */ |
| 575 | fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ |
| 576 | |
| 577 | if (*fl_pte == 0) { |
| 578 | pr_debug("First level PTE is 0\n"); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 579 | goto fail; |
| 580 | } |
| 581 | |
| 582 | /* Unmap supersection */ |
| 583 | if (len == SZ_16M) |
| 584 | for (i = 0; i < 16; i++) |
| 585 | *(fl_pte+i) = 0; |
| 586 | |
| 587 | if (len == SZ_1M) |
| 588 | *fl_pte = 0; |
| 589 | |
| 590 | sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); |
| 591 | sl_offset = SL_OFFSET(va); |
| 592 | sl_pte = sl_table + sl_offset; |
| 593 | |
| 594 | if (len == SZ_64K) { |
| 595 | for (i = 0; i < 16; i++) |
| 596 | *(sl_pte+i) = 0; |
| 597 | } |
| 598 | |
| 599 | if (len == SZ_4K) |
| 600 | *sl_pte = 0; |
| 601 | |
| 602 | if (len == SZ_4K || len == SZ_64K) { |
| 603 | int used = 0; |
| 604 | |
| 605 | for (i = 0; i < NUM_SL_PTE; i++) |
| 606 | if (sl_table[i]) |
| 607 | used = 1; |
| 608 | if (!used) { |
| 609 | free_page((unsigned long)sl_table); |
| 610 | *fl_pte = 0; |
| 611 | } |
| 612 | } |
| 613 | |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 614 | ret = __flush_iotlb(domain); |
Ohad Ben-Cohen | 9e28547 | 2011-09-02 13:32:34 -0400 | [diff] [blame] | 615 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 616 | fail: |
| 617 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
Ohad Ben-Cohen | 5009065 | 2011-11-10 11:32:25 +0200 | [diff] [blame] | 618 | |
| 619 | /* the IOMMU API requires us to return how many bytes were unmapped */ |
| 620 | len = ret ? 0 : len; |
| 621 | return len; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 622 | } |
| 623 | |
| 624 | static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, |
Varun Sethi | bb5547ac | 2013-03-29 01:23:58 +0530 | [diff] [blame] | 625 | dma_addr_t va) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 626 | { |
| 627 | struct msm_priv *priv; |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 628 | struct msm_iommu_dev *iommu; |
| 629 | struct msm_iommu_ctx_dev *master; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 630 | unsigned int par; |
| 631 | unsigned long flags; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 632 | phys_addr_t ret = 0; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 633 | |
| 634 | spin_lock_irqsave(&msm_iommu_lock, flags); |
| 635 | |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 636 | priv = to_msm_priv(domain); |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 637 | iommu = list_first_entry(&priv->list_attached, |
| 638 | struct msm_iommu_dev, dom_node); |
| 639 | |
| 640 | if (list_empty(&iommu->ctx_list)) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 641 | goto fail; |
| 642 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 643 | master = list_first_entry(&iommu->ctx_list, |
| 644 | struct msm_iommu_ctx_dev, list); |
| 645 | if (!master) |
| 646 | goto fail; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 647 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 648 | ret = __enable_clocks(iommu); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 649 | if (ret) |
| 650 | goto fail; |
| 651 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 652 | /* Invalidate context TLB */ |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 653 | SET_CTX_TLBIALL(iommu->base, master->num, 0); |
| 654 | SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 655 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 656 | par = GET_PAR(iommu->base, master->num); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 657 | |
| 658 | /* We are dealing with a supersection */ |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 659 | if (GET_NOFAULT_SS(iommu->base, master->num)) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 660 | ret = (par & 0xFF000000) | (va & 0x00FFFFFF); |
| 661 | else /* Upper 20 bits from PAR, lower 12 from VA */ |
| 662 | ret = (par & 0xFFFFF000) | (va & 0x00000FFF); |
| 663 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 664 | if (GET_FAULT(iommu->base, master->num)) |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 665 | ret = 0; |
| 666 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 667 | __disable_clocks(iommu); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 668 | fail: |
| 669 | spin_unlock_irqrestore(&msm_iommu_lock, flags); |
| 670 | return ret; |
| 671 | } |
| 672 | |
Joerg Roedel | 4480845 | 2014-09-05 10:51:14 +0200 | [diff] [blame] | 673 | static bool msm_iommu_capable(enum iommu_cap cap) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 674 | { |
Joerg Roedel | 4480845 | 2014-09-05 10:51:14 +0200 | [diff] [blame] | 675 | return false; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 676 | } |
| 677 | |
| 678 | static void print_ctx_regs(void __iomem *base, int ctx) |
| 679 | { |
| 680 | unsigned int fsr = GET_FSR(base, ctx); |
| 681 | pr_err("FAR = %08x PAR = %08x\n", |
| 682 | GET_FAR(base, ctx), GET_PAR(base, ctx)); |
| 683 | pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr, |
| 684 | (fsr & 0x02) ? "TF " : "", |
| 685 | (fsr & 0x04) ? "AFF " : "", |
| 686 | (fsr & 0x08) ? "APF " : "", |
| 687 | (fsr & 0x10) ? "TLBMF " : "", |
| 688 | (fsr & 0x20) ? "HTWDEEF " : "", |
| 689 | (fsr & 0x40) ? "HTWSEEF " : "", |
| 690 | (fsr & 0x80) ? "MHF " : "", |
| 691 | (fsr & 0x10000) ? "SL " : "", |
| 692 | (fsr & 0x40000000) ? "SS " : "", |
| 693 | (fsr & 0x80000000) ? "MULTI " : ""); |
| 694 | |
| 695 | pr_err("FSYNR0 = %08x FSYNR1 = %08x\n", |
| 696 | GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx)); |
| 697 | pr_err("TTBR0 = %08x TTBR1 = %08x\n", |
| 698 | GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); |
| 699 | pr_err("SCTLR = %08x ACTLR = %08x\n", |
| 700 | GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); |
| 701 | pr_err("PRRR = %08x NMRR = %08x\n", |
| 702 | GET_PRRR(base, ctx), GET_NMRR(base, ctx)); |
| 703 | } |
| 704 | |
| 705 | irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) |
| 706 | { |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 707 | struct msm_iommu_dev *iommu = dev_id; |
Stepan Moskovchenko | 3306973 | 2010-11-12 19:30:00 -0800 | [diff] [blame] | 708 | unsigned int fsr; |
Stepan Moskovchenko | a43d8c1 | 2011-02-24 18:00:42 -0800 | [diff] [blame] | 709 | int i, ret; |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 710 | |
| 711 | spin_lock(&msm_iommu_lock); |
| 712 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 713 | if (!iommu) { |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 714 | pr_err("Invalid device ID in context interrupt handler\n"); |
| 715 | goto fail; |
| 716 | } |
| 717 | |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 718 | pr_err("Unexpected IOMMU page fault!\n"); |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 719 | pr_err("base = %08x\n", (unsigned int)iommu->base); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 720 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 721 | ret = __enable_clocks(iommu); |
Stepan Moskovchenko | 41f3f51 | 2011-02-24 18:00:39 -0800 | [diff] [blame] | 722 | if (ret) |
| 723 | goto fail; |
| 724 | |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 725 | for (i = 0; i < iommu->ncb; i++) { |
| 726 | fsr = GET_FSR(iommu->base, i); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 727 | if (fsr) { |
| 728 | pr_err("Fault occurred in context %d.\n", i); |
| 729 | pr_err("Interesting registers:\n"); |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 730 | print_ctx_regs(iommu->base, i); |
| 731 | SET_FSR(iommu->base, i, 0x4000000F); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 732 | } |
| 733 | } |
Sricharan R | 109bd48 | 2016-06-13 17:06:02 +0530 | [diff] [blame] | 734 | __disable_clocks(iommu); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 735 | fail: |
| 736 | spin_unlock(&msm_iommu_lock); |
| 737 | return 0; |
| 738 | } |
| 739 | |
Thierry Reding | b22f643 | 2014-06-27 09:03:12 +0200 | [diff] [blame] | 740 | static const struct iommu_ops msm_iommu_ops = { |
Joerg Roedel | 4480845 | 2014-09-05 10:51:14 +0200 | [diff] [blame] | 741 | .capable = msm_iommu_capable, |
Joerg Roedel | 3e116c3 | 2015-03-26 13:43:14 +0100 | [diff] [blame] | 742 | .domain_alloc = msm_iommu_domain_alloc, |
| 743 | .domain_free = msm_iommu_domain_free, |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 744 | .attach_dev = msm_iommu_attach_dev, |
| 745 | .detach_dev = msm_iommu_detach_dev, |
| 746 | .map = msm_iommu_map, |
| 747 | .unmap = msm_iommu_unmap, |
Olav Haugan | 315786e | 2014-10-25 09:55:16 -0700 | [diff] [blame] | 748 | .map_sg = default_iommu_map_sg, |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 749 | .iova_to_phys = msm_iommu_iova_to_phys, |
Ohad Ben-Cohen | 8342727 | 2011-11-10 11:32:28 +0200 | [diff] [blame] | 750 | .pgsize_bitmap = MSM_IOMMU_PGSIZES, |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 751 | }; |
| 752 | |
Sricharan R | f7f125e | 2016-06-13 17:06:04 +0530 | [diff] [blame^] | 753 | static int msm_iommu_probe(struct platform_device *pdev) |
| 754 | { |
| 755 | struct resource *r; |
| 756 | struct msm_iommu_dev *iommu; |
| 757 | int ret, par, val; |
| 758 | |
| 759 | iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); |
| 760 | if (!iommu) |
| 761 | return -ENODEV; |
| 762 | |
| 763 | iommu->dev = &pdev->dev; |
| 764 | INIT_LIST_HEAD(&iommu->ctx_list); |
| 765 | |
| 766 | iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk"); |
| 767 | if (IS_ERR(iommu->pclk)) { |
| 768 | dev_err(iommu->dev, "could not get smmu_pclk\n"); |
| 769 | return PTR_ERR(iommu->pclk); |
| 770 | } |
| 771 | |
| 772 | ret = clk_prepare(iommu->pclk); |
| 773 | if (ret) { |
| 774 | dev_err(iommu->dev, "could not prepare smmu_pclk\n"); |
| 775 | return ret; |
| 776 | } |
| 777 | |
| 778 | iommu->clk = devm_clk_get(iommu->dev, "iommu_clk"); |
| 779 | if (IS_ERR(iommu->clk)) { |
| 780 | dev_err(iommu->dev, "could not get iommu_clk\n"); |
| 781 | clk_unprepare(iommu->pclk); |
| 782 | return PTR_ERR(iommu->clk); |
| 783 | } |
| 784 | |
| 785 | ret = clk_prepare(iommu->clk); |
| 786 | if (ret) { |
| 787 | dev_err(iommu->dev, "could not prepare iommu_clk\n"); |
| 788 | clk_unprepare(iommu->pclk); |
| 789 | return ret; |
| 790 | } |
| 791 | |
| 792 | r = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| 793 | iommu->base = devm_ioremap_resource(iommu->dev, r); |
| 794 | if (IS_ERR(iommu->base)) { |
| 795 | dev_err(iommu->dev, "could not get iommu base\n"); |
| 796 | ret = PTR_ERR(iommu->base); |
| 797 | goto fail; |
| 798 | } |
| 799 | |
| 800 | iommu->irq = platform_get_irq(pdev, 0); |
| 801 | if (iommu->irq < 0) { |
| 802 | dev_err(iommu->dev, "could not get iommu irq\n"); |
| 803 | ret = -ENODEV; |
| 804 | goto fail; |
| 805 | } |
| 806 | |
| 807 | ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val); |
| 808 | if (ret) { |
| 809 | dev_err(iommu->dev, "could not get ncb\n"); |
| 810 | goto fail; |
| 811 | } |
| 812 | iommu->ncb = val; |
| 813 | |
| 814 | msm_iommu_reset(iommu->base, iommu->ncb); |
| 815 | SET_M(iommu->base, 0, 1); |
| 816 | SET_PAR(iommu->base, 0, 0); |
| 817 | SET_V2PCFG(iommu->base, 0, 1); |
| 818 | SET_V2PPR(iommu->base, 0, 0); |
| 819 | par = GET_PAR(iommu->base, 0); |
| 820 | SET_V2PCFG(iommu->base, 0, 0); |
| 821 | SET_M(iommu->base, 0, 0); |
| 822 | |
| 823 | if (!par) { |
| 824 | pr_err("Invalid PAR value detected\n"); |
| 825 | ret = -ENODEV; |
| 826 | goto fail; |
| 827 | } |
| 828 | |
| 829 | ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL, |
| 830 | msm_iommu_fault_handler, |
| 831 | IRQF_ONESHOT | IRQF_SHARED, |
| 832 | "msm_iommu_secure_irpt_handler", |
| 833 | iommu); |
| 834 | if (ret) { |
| 835 | pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret); |
| 836 | goto fail; |
| 837 | } |
| 838 | |
| 839 | list_add(&iommu->dev_node, &qcom_iommu_devices); |
| 840 | |
| 841 | pr_info("device mapped at %p, irq %d with %d ctx banks\n", |
| 842 | iommu->base, iommu->irq, iommu->ncb); |
| 843 | |
| 844 | return ret; |
| 845 | fail: |
| 846 | clk_unprepare(iommu->clk); |
| 847 | clk_unprepare(iommu->pclk); |
| 848 | return ret; |
| 849 | } |
| 850 | |
| 851 | static const struct of_device_id msm_iommu_dt_match[] = { |
| 852 | { .compatible = "qcom,apq8064-iommu" }, |
| 853 | {} |
| 854 | }; |
| 855 | |
| 856 | static int msm_iommu_remove(struct platform_device *pdev) |
| 857 | { |
| 858 | struct msm_iommu_dev *iommu = platform_get_drvdata(pdev); |
| 859 | |
| 860 | clk_unprepare(iommu->clk); |
| 861 | clk_unprepare(iommu->pclk); |
| 862 | return 0; |
| 863 | } |
| 864 | |
| 865 | static struct platform_driver msm_iommu_driver = { |
| 866 | .driver = { |
| 867 | .name = "msm_iommu", |
| 868 | .of_match_table = msm_iommu_dt_match, |
| 869 | }, |
| 870 | .probe = msm_iommu_probe, |
| 871 | .remove = msm_iommu_remove, |
| 872 | }; |
| 873 | |
| 874 | static int __init msm_iommu_driver_init(void) |
| 875 | { |
| 876 | int ret; |
| 877 | |
| 878 | ret = platform_driver_register(&msm_iommu_driver); |
| 879 | if (ret != 0) |
| 880 | pr_err("Failed to register IOMMU driver\n"); |
| 881 | |
| 882 | return ret; |
| 883 | } |
| 884 | |
| 885 | static void __exit msm_iommu_driver_exit(void) |
| 886 | { |
| 887 | platform_driver_unregister(&msm_iommu_driver); |
| 888 | } |
| 889 | |
| 890 | subsys_initcall(msm_iommu_driver_init); |
| 891 | module_exit(msm_iommu_driver_exit); |
| 892 | |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 893 | static int __init get_tex_class(int icp, int ocp, int mt, int nos) |
| 894 | { |
| 895 | int i = 0; |
| 896 | unsigned int prrr = 0; |
| 897 | unsigned int nmrr = 0; |
| 898 | int c_icp, c_ocp, c_mt, c_nos; |
| 899 | |
| 900 | RCP15_PRRR(prrr); |
| 901 | RCP15_NMRR(nmrr); |
| 902 | |
| 903 | for (i = 0; i < NUM_TEX_CLASS; i++) { |
| 904 | c_nos = PRRR_NOS(prrr, i); |
| 905 | c_mt = PRRR_MT(prrr, i); |
| 906 | c_icp = NMRR_ICP(nmrr, i); |
| 907 | c_ocp = NMRR_OCP(nmrr, i); |
| 908 | |
| 909 | if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos) |
| 910 | return i; |
| 911 | } |
| 912 | |
| 913 | return -ENODEV; |
| 914 | } |
| 915 | |
| 916 | static void __init setup_iommu_tex_classes(void) |
| 917 | { |
| 918 | msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] = |
| 919 | get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1); |
| 920 | |
| 921 | msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] = |
| 922 | get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1); |
| 923 | |
| 924 | msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] = |
| 925 | get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1); |
| 926 | |
| 927 | msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] = |
| 928 | get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1); |
| 929 | } |
| 930 | |
Stepan Moskovchenko | 516cbc7 | 2010-11-12 19:29:53 -0800 | [diff] [blame] | 931 | static int __init msm_iommu_init(void) |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 932 | { |
Stepan Moskovchenko | 100832c | 2010-11-15 18:20:08 -0800 | [diff] [blame] | 933 | setup_iommu_tex_classes(); |
Joerg Roedel | 85eebbc | 2011-09-06 17:56:07 +0200 | [diff] [blame] | 934 | bus_set_iommu(&platform_bus_type, &msm_iommu_ops); |
Stepan Moskovchenko | 0720d1f | 2010-08-24 18:31:10 -0700 | [diff] [blame] | 935 | return 0; |
| 936 | } |
| 937 | |
| 938 | subsys_initcall(msm_iommu_init); |
| 939 | |
| 940 | MODULE_LICENSE("GPL v2"); |
| 941 | MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>"); |