blob: 3dc5b65f3990d774e648376f8e814a4eb12f3d5f [file] [log] [blame]
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001/*
2 * omap iommu: tlb and pagetable primitives
3 *
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -08004 * Copyright (C) 2008-2010 Nokia Corporation
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02005 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
7 * Paul Mundt and Toshihiro Kobayashi
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/err.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090015#include <linux/slab.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020016#include <linux/interrupt.h>
17#include <linux/ioport.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020018#include <linux/platform_device.h>
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030019#include <linux/iommu.h>
Tony Lindgrenc8d35c82012-11-02 12:24:03 -070020#include <linux/omap-iommu.h>
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030021#include <linux/mutex.h>
22#include <linux/spinlock.h>
Tony Lindgrened1c7de2012-11-02 12:24:06 -070023#include <linux/io.h>
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -060024#include <linux/pm_runtime.h>
Florian Vaussard3c927482014-02-28 14:42:36 -060025#include <linux/of.h>
26#include <linux/of_iommu.h>
27#include <linux/of_irq.h>
Suman Anna7d682772014-09-04 17:27:30 -050028#include <linux/of_platform.h>
Suman Anna3ca92992015-10-02 18:02:44 -050029#include <linux/regmap.h>
30#include <linux/mfd/syscon.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020031
32#include <asm/cacheflush.h>
33
Tony Lindgren2ab7c842012-11-02 12:24:14 -070034#include <linux/platform_data/iommu-omap.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020035
Ido Yariv2f7702a2012-11-02 12:24:00 -070036#include "omap-iopgtable.h"
Tony Lindgrened1c7de2012-11-02 12:24:06 -070037#include "omap-iommu.h"
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020038
Suman Anna5acc97d2014-03-17 20:31:34 -050039#define to_iommu(dev) \
40 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)))
41
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +020042/* bitmap of the page sizes currently supported */
43#define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
44
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030045/**
46 * struct omap_iommu_domain - omap iommu domain
47 * @pgtable: the page table
48 * @iommu_dev: an omap iommu device attached to this domain. only a single
49 * iommu device can be attached for now.
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050050 * @dev: Device using this domain.
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030051 * @lock: domain lock, should be taken when attaching/detaching
52 */
53struct omap_iommu_domain {
54 u32 *pgtable;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030055 struct omap_iommu *iommu_dev;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050056 struct device *dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030057 spinlock_t lock;
Joerg Roedel8cf851e2015-03-26 13:43:09 +010058 struct iommu_domain domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030059};
60
Ido Yariv7bd9e252012-11-02 12:24:09 -070061#define MMU_LOCK_BASE_SHIFT 10
62#define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT)
63#define MMU_LOCK_BASE(x) \
64 ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT)
65
66#define MMU_LOCK_VICT_SHIFT 4
67#define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT)
68#define MMU_LOCK_VICT(x) \
69 ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT)
70
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020071static struct platform_driver omap_iommu_driver;
72static struct kmem_cache *iopte_cachep;
73
74/**
Joerg Roedel8cf851e2015-03-26 13:43:09 +010075 * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
76 * @dom: generic iommu domain handle
77 **/
78static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom)
79{
80 return container_of(dom, struct omap_iommu_domain, domain);
81}
82
83/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030084 * omap_iommu_save_ctx - Save registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020085 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020086 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020087void omap_iommu_save_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020088{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020089 struct omap_iommu *obj = dev_to_omap_iommu(dev);
Suman Annabd4396f2014-10-22 17:22:27 -050090 u32 *p = obj->ctx;
91 int i;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020092
Suman Annabd4396f2014-10-22 17:22:27 -050093 for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
94 p[i] = iommu_read_reg(obj, i * sizeof(u32));
95 dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);
96 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020097}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030098EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020099
100/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300101 * omap_iommu_restore_ctx - Restore registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200102 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200103 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200104void omap_iommu_restore_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200105{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200106 struct omap_iommu *obj = dev_to_omap_iommu(dev);
Suman Annabd4396f2014-10-22 17:22:27 -0500107 u32 *p = obj->ctx;
108 int i;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200109
Suman Annabd4396f2014-10-22 17:22:27 -0500110 for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
111 iommu_write_reg(obj, p[i], i * sizeof(u32));
112 dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);
113 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200114}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300115EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200116
Suman Anna3ca92992015-10-02 18:02:44 -0500117static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable)
118{
119 u32 val, mask;
120
121 if (!obj->syscfg)
122 return;
123
124 mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT));
125 val = enable ? mask : 0;
126 regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val);
127}
128
Suman Annabd4396f2014-10-22 17:22:27 -0500129static void __iommu_set_twl(struct omap_iommu *obj, bool on)
130{
131 u32 l = iommu_read_reg(obj, MMU_CNTL);
132
133 if (on)
134 iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE);
135 else
136 iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE);
137
138 l &= ~MMU_CNTL_MASK;
139 if (on)
140 l |= (MMU_CNTL_MMU_EN | MMU_CNTL_TWL_EN);
141 else
142 l |= (MMU_CNTL_MMU_EN);
143
144 iommu_write_reg(obj, l, MMU_CNTL);
145}
146
147static int omap2_iommu_enable(struct omap_iommu *obj)
148{
149 u32 l, pa;
150
151 if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd, SZ_16K))
152 return -EINVAL;
153
154 pa = virt_to_phys(obj->iopgd);
155 if (!IS_ALIGNED(pa, SZ_16K))
156 return -EINVAL;
157
158 l = iommu_read_reg(obj, MMU_REVISION);
159 dev_info(obj->dev, "%s: version %d.%d\n", obj->name,
160 (l >> 4) & 0xf, l & 0xf);
161
162 iommu_write_reg(obj, pa, MMU_TTB);
163
Suman Anna3ca92992015-10-02 18:02:44 -0500164 dra7_cfg_dspsys_mmu(obj, true);
165
Suman Annabd4396f2014-10-22 17:22:27 -0500166 if (obj->has_bus_err_back)
167 iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);
168
169 __iommu_set_twl(obj, true);
170
171 return 0;
172}
173
174static void omap2_iommu_disable(struct omap_iommu *obj)
175{
176 u32 l = iommu_read_reg(obj, MMU_CNTL);
177
178 l &= ~MMU_CNTL_MASK;
179 iommu_write_reg(obj, l, MMU_CNTL);
Suman Anna3ca92992015-10-02 18:02:44 -0500180 dra7_cfg_dspsys_mmu(obj, false);
Suman Annabd4396f2014-10-22 17:22:27 -0500181
182 dev_dbg(obj->dev, "%s is shutting down\n", obj->name);
183}
184
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300185static int iommu_enable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200186{
187 int err;
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600188 struct platform_device *pdev = to_platform_device(obj->dev);
Kiran Padwal99cb9ae2014-10-30 11:59:47 +0530189 struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200190
Florian Vaussard90e569c2014-02-28 14:42:34 -0600191 if (pdata && pdata->deassert_reset) {
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600192 err = pdata->deassert_reset(pdev, pdata->reset_name);
193 if (err) {
194 dev_err(obj->dev, "deassert_reset failed: %d\n", err);
195 return err;
196 }
197 }
198
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600199 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200200
Suman Annabd4396f2014-10-22 17:22:27 -0500201 err = omap2_iommu_enable(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200202
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200203 return err;
204}
205
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300206static void iommu_disable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200207{
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600208 struct platform_device *pdev = to_platform_device(obj->dev);
Kiran Padwal99cb9ae2014-10-30 11:59:47 +0530209 struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600210
Suman Annabd4396f2014-10-22 17:22:27 -0500211 omap2_iommu_disable(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200212
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600213 pm_runtime_put_sync(obj->dev);
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600214
Florian Vaussard90e569c2014-02-28 14:42:34 -0600215 if (pdata && pdata->assert_reset)
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600216 pdata->assert_reset(pdev, pdata->reset_name);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200217}
218
219/*
220 * TLB operations
221 */
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300222static u32 iotlb_cr_to_virt(struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200223{
Suman Annabd4396f2014-10-22 17:22:27 -0500224 u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK;
225 u32 mask = get_cam_va_mask(cr->cam & page_size);
226
227 return cr->cam & mask;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200228}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200229
230static u32 get_iopte_attr(struct iotlb_entry *e)
231{
Suman Annabd4396f2014-10-22 17:22:27 -0500232 u32 attr;
233
234 attr = e->mixed << 5;
235 attr |= e->endian;
236 attr |= e->elsz >> 3;
237 attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) ||
238 (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6);
239 return attr;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200240}
241
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300242static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200243{
Suman Annabd4396f2014-10-22 17:22:27 -0500244 u32 status, fault_addr;
245
246 status = iommu_read_reg(obj, MMU_IRQSTATUS);
247 status &= MMU_IRQ_MASK;
248 if (!status) {
249 *da = 0;
250 return 0;
251 }
252
253 fault_addr = iommu_read_reg(obj, MMU_FAULT_AD);
254 *da = fault_addr;
255
256 iommu_write_reg(obj, status, MMU_IRQSTATUS);
257
258 return status;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200259}
260
Suman Anna69c2c192015-07-20 17:33:25 -0500261void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200262{
263 u32 val;
264
265 val = iommu_read_reg(obj, MMU_LOCK);
266
267 l->base = MMU_LOCK_BASE(val);
268 l->vict = MMU_LOCK_VICT(val);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200269}
270
Suman Anna69c2c192015-07-20 17:33:25 -0500271void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200272{
273 u32 val;
274
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200275 val = (l->base << MMU_LOCK_BASE_SHIFT);
276 val |= (l->vict << MMU_LOCK_VICT_SHIFT);
277
278 iommu_write_reg(obj, val, MMU_LOCK);
279}
280
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300281static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200282{
Suman Annabd4396f2014-10-22 17:22:27 -0500283 cr->cam = iommu_read_reg(obj, MMU_READ_CAM);
284 cr->ram = iommu_read_reg(obj, MMU_READ_RAM);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200285}
286
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300287static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200288{
Suman Annabd4396f2014-10-22 17:22:27 -0500289 iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM);
290 iommu_write_reg(obj, cr->ram, MMU_RAM);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200291
292 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
293 iommu_write_reg(obj, 1, MMU_LD_TLB);
294}
295
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000296/* only used in iotlb iteration for-loop */
Suman Anna69c2c192015-07-20 17:33:25 -0500297struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000298{
299 struct cr_regs cr;
300 struct iotlb_lock l;
301
302 iotlb_lock_get(obj, &l);
303 l.vict = n;
304 iotlb_lock_set(obj, &l);
305 iotlb_read_cr(obj, &cr);
306
307 return cr;
308}
309
Suman Annabd4396f2014-10-22 17:22:27 -0500310#ifdef PREFETCH_IOTLB
311static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,
312 struct iotlb_entry *e)
313{
314 struct cr_regs *cr;
315
316 if (!e)
317 return NULL;
318
319 if (e->da & ~(get_cam_va_mask(e->pgsz))) {
320 dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__,
321 e->da);
322 return ERR_PTR(-EINVAL);
323 }
324
325 cr = kmalloc(sizeof(*cr), GFP_KERNEL);
326 if (!cr)
327 return ERR_PTR(-ENOMEM);
328
329 cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid;
330 cr->ram = e->pa | e->endian | e->elsz | e->mixed;
331
332 return cr;
333}
334
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200335/**
336 * load_iotlb_entry - Set an iommu tlb entry
337 * @obj: target iommu
338 * @e: an iommu tlb entry info
339 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300340static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200341{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200342 int err = 0;
343 struct iotlb_lock l;
344 struct cr_regs *cr;
345
346 if (!obj || !obj->nr_tlb_entries || !e)
347 return -EINVAL;
348
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600349 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200350
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000351 iotlb_lock_get(obj, &l);
352 if (l.base == obj->nr_tlb_entries) {
353 dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200354 err = -EBUSY;
355 goto out;
356 }
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000357 if (!e->prsvd) {
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000358 int i;
359 struct cr_regs tmp;
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000360
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000361 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000362 if (!iotlb_cr_valid(&tmp))
363 break;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000364
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000365 if (i == obj->nr_tlb_entries) {
366 dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
367 err = -EBUSY;
368 goto out;
369 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000370
371 iotlb_lock_get(obj, &l);
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000372 } else {
373 l.vict = l.base;
374 iotlb_lock_set(obj, &l);
375 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200376
377 cr = iotlb_alloc_cr(obj, e);
378 if (IS_ERR(cr)) {
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600379 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200380 return PTR_ERR(cr);
381 }
382
383 iotlb_load_cr(obj, cr);
384 kfree(cr);
385
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000386 if (e->prsvd)
387 l.base++;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200388 /* increment victim for next tlb load */
389 if (++l.vict == obj->nr_tlb_entries)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000390 l.vict = l.base;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200391 iotlb_lock_set(obj, &l);
392out:
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600393 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200394 return err;
395}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200396
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300397#else /* !PREFETCH_IOTLB */
398
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300399static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300400{
401 return 0;
402}
403
404#endif /* !PREFETCH_IOTLB */
405
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300406static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300407{
408 return load_iotlb_entry(obj, e);
409}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200410
411/**
412 * flush_iotlb_page - Clear an iommu tlb entry
413 * @obj: target iommu
414 * @da: iommu device virtual address
415 *
416 * Clear an iommu tlb entry which includes 'da' address.
417 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300418static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200419{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200420 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000421 struct cr_regs cr;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200422
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600423 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200424
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000425 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200426 u32 start;
427 size_t bytes;
428
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200429 if (!iotlb_cr_valid(&cr))
430 continue;
431
432 start = iotlb_cr_to_virt(&cr);
433 bytes = iopgsz_to_bytes(cr.cam & 3);
434
435 if ((start <= da) && (da < start + bytes)) {
436 dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",
437 __func__, start, da, bytes);
Hari Kanigeri0fa035e2010-08-20 13:50:18 +0000438 iotlb_load_cr(obj, &cr);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200439 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
Laurent Pinchartf7129a02014-03-07 23:47:03 +0100440 break;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200441 }
442 }
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600443 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200444
445 if (i == obj->nr_tlb_entries)
446 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
447}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200448
449/**
450 * flush_iotlb_all - Clear all iommu tlb entries
451 * @obj: target iommu
452 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300453static void flush_iotlb_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200454{
455 struct iotlb_lock l;
456
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600457 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200458
459 l.base = 0;
460 l.vict = 0;
461 iotlb_lock_set(obj, &l);
462
463 iommu_write_reg(obj, 1, MMU_GFLUSH);
464
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600465 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200466}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200467
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200468/*
469 * H/W pagetable operations
470 */
471static void flush_iopgd_range(u32 *first, u32 *last)
472{
473 /* FIXME: L2 cache should be taken care of if it exists */
474 do {
475 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
476 : : "r" (first));
477 first += L1_CACHE_BYTES / sizeof(*first);
478 } while (first <= last);
479}
480
481static void flush_iopte_range(u32 *first, u32 *last)
482{
483 /* FIXME: L2 cache should be taken care of if it exists */
484 do {
485 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
486 : : "r" (first));
487 first += L1_CACHE_BYTES / sizeof(*first);
488 } while (first <= last);
489}
490
491static void iopte_free(u32 *iopte)
492{
493 /* Note: freed iopte's must be clean ready for re-use */
Zhouyi Zhoue28045a2014-03-05 18:20:19 +0800494 if (iopte)
495 kmem_cache_free(iopte_cachep, iopte);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200496}
497
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300498static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200499{
500 u32 *iopte;
501
502 /* a table has already existed */
503 if (*iopgd)
504 goto pte_ready;
505
506 /*
507 * do the allocation outside the page table lock
508 */
509 spin_unlock(&obj->page_table_lock);
510 iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
511 spin_lock(&obj->page_table_lock);
512
513 if (!*iopgd) {
514 if (!iopte)
515 return ERR_PTR(-ENOMEM);
516
517 *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
518 flush_iopgd_range(iopgd, iopgd);
519
520 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
521 } else {
522 /* We raced, free the reduniovant table */
523 iopte_free(iopte);
524 }
525
526pte_ready:
527 iopte = iopte_offset(iopgd, da);
528
529 dev_vdbg(obj->dev,
530 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
531 __func__, da, iopgd, *iopgd, iopte, *iopte);
532
533 return iopte;
534}
535
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300536static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200537{
538 u32 *iopgd = iopgd_offset(obj, da);
539
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300540 if ((da | pa) & ~IOSECTION_MASK) {
541 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
542 __func__, da, pa, IOSECTION_SIZE);
543 return -EINVAL;
544 }
545
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200546 *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
547 flush_iopgd_range(iopgd, iopgd);
548 return 0;
549}
550
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300551static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200552{
553 u32 *iopgd = iopgd_offset(obj, da);
554 int i;
555
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300556 if ((da | pa) & ~IOSUPER_MASK) {
557 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
558 __func__, da, pa, IOSUPER_SIZE);
559 return -EINVAL;
560 }
561
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200562 for (i = 0; i < 16; i++)
563 *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
564 flush_iopgd_range(iopgd, iopgd + 15);
565 return 0;
566}
567
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300568static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200569{
570 u32 *iopgd = iopgd_offset(obj, da);
571 u32 *iopte = iopte_alloc(obj, iopgd, da);
572
573 if (IS_ERR(iopte))
574 return PTR_ERR(iopte);
575
576 *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
577 flush_iopte_range(iopte, iopte);
578
579 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
580 __func__, da, pa, iopte, *iopte);
581
582 return 0;
583}
584
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300585static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200586{
587 u32 *iopgd = iopgd_offset(obj, da);
588 u32 *iopte = iopte_alloc(obj, iopgd, da);
589 int i;
590
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300591 if ((da | pa) & ~IOLARGE_MASK) {
592 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
593 __func__, da, pa, IOLARGE_SIZE);
594 return -EINVAL;
595 }
596
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200597 if (IS_ERR(iopte))
598 return PTR_ERR(iopte);
599
600 for (i = 0; i < 16; i++)
601 *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
602 flush_iopte_range(iopte, iopte + 15);
603 return 0;
604}
605
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300606static int
607iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200608{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300609 int (*fn)(struct omap_iommu *, u32, u32, u32);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200610 u32 prot;
611 int err;
612
613 if (!obj || !e)
614 return -EINVAL;
615
616 switch (e->pgsz) {
617 case MMU_CAM_PGSZ_16M:
618 fn = iopgd_alloc_super;
619 break;
620 case MMU_CAM_PGSZ_1M:
621 fn = iopgd_alloc_section;
622 break;
623 case MMU_CAM_PGSZ_64K:
624 fn = iopte_alloc_large;
625 break;
626 case MMU_CAM_PGSZ_4K:
627 fn = iopte_alloc_page;
628 break;
629 default:
630 fn = NULL;
631 BUG();
632 break;
633 }
634
635 prot = get_iopte_attr(e);
636
637 spin_lock(&obj->page_table_lock);
638 err = fn(obj, e->da, e->pa, prot);
639 spin_unlock(&obj->page_table_lock);
640
641 return err;
642}
643
644/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300645 * omap_iopgtable_store_entry - Make an iommu pte entry
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200646 * @obj: target iommu
647 * @e: an iommu tlb entry info
648 **/
Suman Anna4899a562014-10-22 17:22:32 -0500649static int
650omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200651{
652 int err;
653
654 flush_iotlb_page(obj, e->da);
655 err = iopgtable_store_entry_core(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200656 if (!err)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300657 prefetch_iotlb_entry(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200658 return err;
659}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200660
661/**
662 * iopgtable_lookup_entry - Lookup an iommu pte entry
663 * @obj: target iommu
664 * @da: iommu device virtual address
665 * @ppgd: iommu pgd entry pointer to be returned
666 * @ppte: iommu pte entry pointer to be returned
667 **/
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300668static void
669iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200670{
671 u32 *iopgd, *iopte = NULL;
672
673 iopgd = iopgd_offset(obj, da);
674 if (!*iopgd)
675 goto out;
676
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300677 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200678 iopte = iopte_offset(iopgd, da);
679out:
680 *ppgd = iopgd;
681 *ppte = iopte;
682}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200683
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300684static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200685{
686 size_t bytes;
687 u32 *iopgd = iopgd_offset(obj, da);
688 int nent = 1;
689
690 if (!*iopgd)
691 return 0;
692
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300693 if (iopgd_is_table(*iopgd)) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200694 int i;
695 u32 *iopte = iopte_offset(iopgd, da);
696
697 bytes = IOPTE_SIZE;
698 if (*iopte & IOPTE_LARGE) {
699 nent *= 16;
700 /* rewind to the 1st entry */
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -0800701 iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200702 }
703 bytes *= nent;
704 memset(iopte, 0, nent * sizeof(*iopte));
705 flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
706
707 /*
708 * do table walk to check if this table is necessary or not
709 */
710 iopte = iopte_offset(iopgd, 0);
711 for (i = 0; i < PTRS_PER_IOPTE; i++)
712 if (iopte[i])
713 goto out;
714
715 iopte_free(iopte);
716 nent = 1; /* for the next L1 entry */
717 } else {
718 bytes = IOPGD_SIZE;
Hiroshi DOYUdcc730d2009-10-22 14:46:32 -0700719 if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200720 nent *= 16;
721 /* rewind to the 1st entry */
Hiroshi DOYU8d33ea52010-02-15 10:03:32 -0800722 iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200723 }
724 bytes *= nent;
725 }
726 memset(iopgd, 0, nent * sizeof(*iopgd));
727 flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
728out:
729 return bytes;
730}
731
732/**
733 * iopgtable_clear_entry - Remove an iommu pte entry
734 * @obj: target iommu
735 * @da: iommu device virtual address
736 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300737static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200738{
739 size_t bytes;
740
741 spin_lock(&obj->page_table_lock);
742
743 bytes = iopgtable_clear_entry_core(obj, da);
744 flush_iotlb_page(obj, da);
745
746 spin_unlock(&obj->page_table_lock);
747
748 return bytes;
749}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200750
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300751static void iopgtable_clear_entry_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200752{
753 int i;
754
755 spin_lock(&obj->page_table_lock);
756
757 for (i = 0; i < PTRS_PER_IOPGD; i++) {
758 u32 da;
759 u32 *iopgd;
760
761 da = i << IOPGD_SHIFT;
762 iopgd = iopgd_offset(obj, da);
763
764 if (!*iopgd)
765 continue;
766
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300767 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200768 iopte_free(iopte_offset(iopgd, 0));
769
770 *iopgd = 0;
771 flush_iopgd_range(iopgd, iopgd);
772 }
773
774 flush_iotlb_all(obj);
775
776 spin_unlock(&obj->page_table_lock);
777}
778
779/*
780 * Device IOMMU generic operations
781 */
782static irqreturn_t iommu_fault_handler(int irq, void *data)
783{
David Cohend594f1f2011-02-16 19:35:51 +0000784 u32 da, errs;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200785 u32 *iopgd, *iopte;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300786 struct omap_iommu *obj = data;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400787 struct iommu_domain *domain = obj->domain;
Joerg Roedel8cf851e2015-03-26 13:43:09 +0100788 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200789
Suman Anna2088ecb2014-10-22 17:22:19 -0500790 if (!omap_domain->iommu_dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200791 return IRQ_NONE;
792
David Cohend594f1f2011-02-16 19:35:51 +0000793 errs = iommu_report_fault(obj, &da);
Laurent Pinchartc56b2dd2011-05-10 16:56:46 +0200794 if (errs == 0)
795 return IRQ_HANDLED;
David Cohend594f1f2011-02-16 19:35:51 +0000796
797 /* Fault callback or TLB/PTE Dynamic loading */
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400798 if (!report_iommu_fault(domain, obj->dev, da, 0))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200799 return IRQ_HANDLED;
800
Hiroshi DOYU37b29812010-05-24 02:01:52 +0000801 iommu_disable(obj);
802
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200803 iopgd = iopgd_offset(obj, da);
804
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300805 if (!iopgd_is_table(*iopgd)) {
Suman Annab6c2e092013-05-30 18:10:59 -0500806 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",
Suman Anna5835b6a2015-07-20 17:33:32 -0500807 obj->name, errs, da, iopgd, *iopgd);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200808 return IRQ_NONE;
809 }
810
811 iopte = iopte_offset(iopgd, da);
812
Suman Annab6c2e092013-05-30 18:10:59 -0500813 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
Suman Anna5835b6a2015-07-20 17:33:32 -0500814 obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200815
816 return IRQ_NONE;
817}
818
819static int device_match_by_alias(struct device *dev, void *data)
820{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300821 struct omap_iommu *obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200822 const char *name = data;
823
824 pr_debug("%s: %s %s\n", __func__, obj->name, name);
825
826 return strcmp(obj->name, name) == 0;
827}
828
829/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300830 * omap_iommu_attach() - attach iommu device to an iommu domain
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200831 * @name: name of target omap iommu device
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300832 * @iopgd: page table
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200833 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200834static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200835{
Suman Anna7ee08b9e2014-02-28 14:42:33 -0600836 int err;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200837 struct device *dev;
838 struct omap_iommu *obj;
839
Suman Anna5835b6a2015-07-20 17:33:32 -0500840 dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name,
841 device_match_by_alias);
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200842 if (!dev)
Suman Anna7ee08b9e2014-02-28 14:42:33 -0600843 return ERR_PTR(-ENODEV);
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200844
845 obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200846
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300847 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200848
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300849 obj->iopgd = iopgd;
850 err = iommu_enable(obj);
851 if (err)
852 goto err_enable;
853 flush_iotlb_all(obj);
854
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300855 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200856
857 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
858 return obj;
859
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200860err_enable:
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300861 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200862 return ERR_PTR(err);
863}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200864
865/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300866 * omap_iommu_detach - release iommu device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200867 * @obj: target iommu
868 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300869static void omap_iommu_detach(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200870{
Roel Kluinacf9d462010-01-08 10:29:05 -0800871 if (!obj || IS_ERR(obj))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200872 return;
873
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300874 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200875
Suman Anna2088ecb2014-10-22 17:22:19 -0500876 iommu_disable(obj);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300877 obj->iopgd = NULL;
878
879 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200880
881 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
882}
David Cohend594f1f2011-02-16 19:35:51 +0000883
Suman Anna3ca92992015-10-02 18:02:44 -0500884static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev,
885 struct omap_iommu *obj)
886{
887 struct device_node *np = pdev->dev.of_node;
888 int ret;
889
890 if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu"))
891 return 0;
892
893 if (!of_property_read_bool(np, "ti,syscon-mmuconfig")) {
894 dev_err(&pdev->dev, "ti,syscon-mmuconfig property is missing\n");
895 return -EINVAL;
896 }
897
898 obj->syscfg =
899 syscon_regmap_lookup_by_phandle(np, "ti,syscon-mmuconfig");
900 if (IS_ERR(obj->syscfg)) {
901 /* can fail with -EPROBE_DEFER */
902 ret = PTR_ERR(obj->syscfg);
903 return ret;
904 }
905
906 if (of_property_read_u32_index(np, "ti,syscon-mmuconfig", 1,
907 &obj->id)) {
908 dev_err(&pdev->dev, "couldn't get the IOMMU instance id within subsystem\n");
909 return -EINVAL;
910 }
911
912 if (obj->id != 0 && obj->id != 1) {
913 dev_err(&pdev->dev, "invalid IOMMU instance id\n");
914 return -EINVAL;
915 }
916
917 return 0;
918}
919
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200920/*
921 * OMAP Device MMU(IOMMU) detection
922 */
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -0800923static int omap_iommu_probe(struct platform_device *pdev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200924{
925 int err = -ENODEV;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200926 int irq;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300927 struct omap_iommu *obj;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200928 struct resource *res;
Kiran Padwal99cb9ae2014-10-30 11:59:47 +0530929 struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
Florian Vaussard3c927482014-02-28 14:42:36 -0600930 struct device_node *of = pdev->dev.of_node;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200931
Suman Annaf129b3d2014-02-28 14:42:32 -0600932 obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200933 if (!obj)
934 return -ENOMEM;
935
Florian Vaussard3c927482014-02-28 14:42:36 -0600936 if (of) {
937 obj->name = dev_name(&pdev->dev);
938 obj->nr_tlb_entries = 32;
939 err = of_property_read_u32(of, "ti,#tlb-entries",
940 &obj->nr_tlb_entries);
941 if (err && err != -EINVAL)
942 return err;
943 if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
944 return -EINVAL;
Suman Annab148d5f2014-02-28 14:42:37 -0600945 if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
946 obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
Florian Vaussard3c927482014-02-28 14:42:36 -0600947 } else {
948 obj->nr_tlb_entries = pdata->nr_tlb_entries;
949 obj->name = pdata->name;
Florian Vaussard3c927482014-02-28 14:42:36 -0600950 }
Florian Vaussard3c927482014-02-28 14:42:36 -0600951
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200952 obj->dev = &pdev->dev;
953 obj->ctx = (void *)obj + sizeof(*obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200954
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300955 spin_lock_init(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200956 spin_lock_init(&obj->page_table_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200957
958 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Suman Annaf129b3d2014-02-28 14:42:32 -0600959 obj->regbase = devm_ioremap_resource(obj->dev, res);
960 if (IS_ERR(obj->regbase))
961 return PTR_ERR(obj->regbase);
Aaro Koskinenda4a0f72011-03-14 12:28:32 +0000962
Suman Anna3ca92992015-10-02 18:02:44 -0500963 err = omap_iommu_dra7_get_dsp_system_cfg(pdev, obj);
964 if (err)
965 return err;
966
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200967 irq = platform_get_irq(pdev, 0);
Suman Annaf129b3d2014-02-28 14:42:32 -0600968 if (irq < 0)
969 return -ENODEV;
970
971 err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,
972 dev_name(obj->dev), obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200973 if (err < 0)
Suman Annaf129b3d2014-02-28 14:42:32 -0600974 return err;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200975 platform_set_drvdata(pdev, obj);
976
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600977 pm_runtime_irq_safe(obj->dev);
978 pm_runtime_enable(obj->dev);
979
Suman Anna61c75352014-10-22 17:22:30 -0500980 omap_iommu_debugfs_add(obj);
981
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200982 dev_info(&pdev->dev, "%s registered\n", obj->name);
983 return 0;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200984}
985
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -0800986static int omap_iommu_remove(struct platform_device *pdev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200987{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300988 struct omap_iommu *obj = platform_get_drvdata(pdev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200989
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200990 iopgtable_clear_entry_all(obj);
Suman Anna61c75352014-10-22 17:22:30 -0500991 omap_iommu_debugfs_remove(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200992
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600993 pm_runtime_disable(obj->dev);
994
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200995 dev_info(&pdev->dev, "%s removed\n", obj->name);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200996 return 0;
997}
998
Kiran Padwald943b0f2014-09-11 19:07:36 +0530999static const struct of_device_id omap_iommu_of_match[] = {
Florian Vaussard3c927482014-02-28 14:42:36 -06001000 { .compatible = "ti,omap2-iommu" },
1001 { .compatible = "ti,omap4-iommu" },
1002 { .compatible = "ti,dra7-iommu" },
Suman Anna3ca92992015-10-02 18:02:44 -05001003 { .compatible = "ti,dra7-dsp-iommu" },
Florian Vaussard3c927482014-02-28 14:42:36 -06001004 {},
1005};
Florian Vaussard3c927482014-02-28 14:42:36 -06001006
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001007static struct platform_driver omap_iommu_driver = {
1008 .probe = omap_iommu_probe,
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08001009 .remove = omap_iommu_remove,
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001010 .driver = {
1011 .name = "omap-iommu",
Florian Vaussard3c927482014-02-28 14:42:36 -06001012 .of_match_table = of_match_ptr(omap_iommu_of_match),
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001013 },
1014};
1015
1016static void iopte_cachep_ctor(void *iopte)
1017{
1018 clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
1019}
1020
Laurent Pinchart286f6002014-03-08 00:44:38 +01001021static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
Tony Lindgrened1c7de2012-11-02 12:24:06 -07001022{
1023 memset(e, 0, sizeof(*e));
1024
1025 e->da = da;
1026 e->pa = pa;
Suman Annad760e3e2014-03-17 20:31:32 -05001027 e->valid = MMU_CAM_V;
Laurent Pinchart286f6002014-03-08 00:44:38 +01001028 e->pgsz = pgsz;
1029 e->endian = MMU_RAM_ENDIAN_LITTLE;
1030 e->elsz = MMU_RAM_ELSZ_8;
1031 e->mixed = 0;
Tony Lindgrened1c7de2012-11-02 12:24:06 -07001032
1033 return iopgsz_to_bytes(e->pgsz);
1034}
1035
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001036static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
Suman Anna5835b6a2015-07-20 17:33:32 -05001037 phys_addr_t pa, size_t bytes, int prot)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001038{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001039 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001040 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001041 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001042 struct iotlb_entry e;
1043 int omap_pgsz;
Laurent Pinchart286f6002014-03-08 00:44:38 +01001044 u32 ret;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001045
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001046 omap_pgsz = bytes_to_iopgsz(bytes);
1047 if (omap_pgsz < 0) {
1048 dev_err(dev, "invalid size to map: %d\n", bytes);
1049 return -EINVAL;
1050 }
1051
Joerg Roedel1d7f4492015-01-22 14:42:06 +01001052 dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001053
Laurent Pinchart286f6002014-03-08 00:44:38 +01001054 iotlb_init_entry(&e, da, pa, omap_pgsz);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001055
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001056 ret = omap_iopgtable_store_entry(oiommu, &e);
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001057 if (ret)
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001058 dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001059
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001060 return ret;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001061}
1062
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001063static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
Suman Anna5835b6a2015-07-20 17:33:32 -05001064 size_t size)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001065{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001066 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001067 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001068 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001069
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001070 dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001071
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001072 return iopgtable_clear_entry(oiommu, da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001073}
1074
1075static int
1076omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1077{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001078 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001079 struct omap_iommu *oiommu;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001080 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001081 int ret = 0;
1082
Suman Annae3f595b2014-09-04 17:27:29 -05001083 if (!arch_data || !arch_data->name) {
1084 dev_err(dev, "device doesn't have an associated iommu\n");
1085 return -EINVAL;
1086 }
1087
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001088 spin_lock(&omap_domain->lock);
1089
1090 /* only a single device is supported per domain for now */
1091 if (omap_domain->iommu_dev) {
1092 dev_err(dev, "iommu domain is already attached\n");
1093 ret = -EBUSY;
1094 goto out;
1095 }
1096
1097 /* get a handle to and enable the omap iommu */
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001098 oiommu = omap_iommu_attach(arch_data->name, omap_domain->pgtable);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001099 if (IS_ERR(oiommu)) {
1100 ret = PTR_ERR(oiommu);
1101 dev_err(dev, "can't get omap iommu: %d\n", ret);
1102 goto out;
1103 }
1104
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001105 omap_domain->iommu_dev = arch_data->iommu_dev = oiommu;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001106 omap_domain->dev = dev;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -04001107 oiommu->domain = domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001108
1109out:
1110 spin_unlock(&omap_domain->lock);
1111 return ret;
1112}
1113
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001114static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
Suman Anna5835b6a2015-07-20 17:33:32 -05001115 struct device *dev)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001116{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001117 struct omap_iommu *oiommu = dev_to_omap_iommu(dev);
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001118 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001119
1120 /* only a single device is supported per domain for now */
1121 if (omap_domain->iommu_dev != oiommu) {
1122 dev_err(dev, "invalid iommu device\n");
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001123 return;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001124 }
1125
1126 iopgtable_clear_entry_all(oiommu);
1127
1128 omap_iommu_detach(oiommu);
1129
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001130 omap_domain->iommu_dev = arch_data->iommu_dev = NULL;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001131 omap_domain->dev = NULL;
Suman Annaf24d9ad2014-10-22 17:22:33 -05001132 oiommu->domain = NULL;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001133}
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001134
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001135static void omap_iommu_detach_dev(struct iommu_domain *domain,
Suman Anna5835b6a2015-07-20 17:33:32 -05001136 struct device *dev)
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001137{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001138 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001139
1140 spin_lock(&omap_domain->lock);
1141 _omap_iommu_detach_dev(omap_domain, dev);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001142 spin_unlock(&omap_domain->lock);
1143}
1144
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001145static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001146{
1147 struct omap_iommu_domain *omap_domain;
1148
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001149 if (type != IOMMU_DOMAIN_UNMANAGED)
1150 return NULL;
1151
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001152 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
Suman Anna99ee98d2015-07-20 17:33:29 -05001153 if (!omap_domain)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001154 goto out;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001155
1156 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL);
Suman Anna99ee98d2015-07-20 17:33:29 -05001157 if (!omap_domain->pgtable)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001158 goto fail_nomem;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001159
1160 /*
1161 * should never fail, but please keep this around to ensure
1162 * we keep the hardware happy
1163 */
1164 BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE));
1165
1166 clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
1167 spin_lock_init(&omap_domain->lock);
1168
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001169 omap_domain->domain.geometry.aperture_start = 0;
1170 omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1;
1171 omap_domain->domain.geometry.force_aperture = true;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001172
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001173 return &omap_domain->domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001174
1175fail_nomem:
1176 kfree(omap_domain);
1177out:
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001178 return NULL;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001179}
1180
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001181static void omap_iommu_domain_free(struct iommu_domain *domain)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001182{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001183 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001184
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001185 /*
1186 * An iommu device is still attached
1187 * (currently, only one device can be attached) ?
1188 */
1189 if (omap_domain->iommu_dev)
1190 _omap_iommu_detach_dev(omap_domain, omap_domain->dev);
1191
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001192 kfree(omap_domain->pgtable);
1193 kfree(omap_domain);
1194}
1195
1196static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
Suman Anna5835b6a2015-07-20 17:33:32 -05001197 dma_addr_t da)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001198{
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001199 struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001200 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001201 struct device *dev = oiommu->dev;
1202 u32 *pgd, *pte;
1203 phys_addr_t ret = 0;
1204
1205 iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
1206
1207 if (pte) {
1208 if (iopte_is_small(*pte))
1209 ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
1210 else if (iopte_is_large(*pte))
1211 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
1212 else
Suman Anna2abfcfb2013-05-30 18:10:38 -05001213 dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte,
Suman Anna5835b6a2015-07-20 17:33:32 -05001214 (unsigned long long)da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001215 } else {
1216 if (iopgd_is_section(*pgd))
1217 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
1218 else if (iopgd_is_super(*pgd))
1219 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
1220 else
Suman Anna2abfcfb2013-05-30 18:10:38 -05001221 dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd,
Suman Anna5835b6a2015-07-20 17:33:32 -05001222 (unsigned long long)da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001223 }
1224
1225 return ret;
1226}
1227
Laurent Pinchart07a02032014-02-28 14:42:38 -06001228static int omap_iommu_add_device(struct device *dev)
1229{
1230 struct omap_iommu_arch_data *arch_data;
1231 struct device_node *np;
Suman Anna7d682772014-09-04 17:27:30 -05001232 struct platform_device *pdev;
Laurent Pinchart07a02032014-02-28 14:42:38 -06001233
1234 /*
1235 * Allocate the archdata iommu structure for DT-based devices.
1236 *
1237 * TODO: Simplify this when removing non-DT support completely from the
1238 * IOMMU users.
1239 */
1240 if (!dev->of_node)
1241 return 0;
1242
1243 np = of_parse_phandle(dev->of_node, "iommus", 0);
1244 if (!np)
1245 return 0;
1246
Suman Anna7d682772014-09-04 17:27:30 -05001247 pdev = of_find_device_by_node(np);
1248 if (WARN_ON(!pdev)) {
1249 of_node_put(np);
1250 return -EINVAL;
1251 }
1252
Laurent Pinchart07a02032014-02-28 14:42:38 -06001253 arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL);
1254 if (!arch_data) {
1255 of_node_put(np);
1256 return -ENOMEM;
1257 }
1258
Suman Anna7d682772014-09-04 17:27:30 -05001259 arch_data->name = kstrdup(dev_name(&pdev->dev), GFP_KERNEL);
Laurent Pinchart07a02032014-02-28 14:42:38 -06001260 dev->archdata.iommu = arch_data;
1261
1262 of_node_put(np);
1263
1264 return 0;
1265}
1266
1267static void omap_iommu_remove_device(struct device *dev)
1268{
1269 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
1270
1271 if (!dev->of_node || !arch_data)
1272 return;
1273
1274 kfree(arch_data->name);
1275 kfree(arch_data);
1276}
1277
Thierry Redingb22f6432014-06-27 09:03:12 +02001278static const struct iommu_ops omap_iommu_ops = {
Joerg Roedel8cf851e2015-03-26 13:43:09 +01001279 .domain_alloc = omap_iommu_domain_alloc,
1280 .domain_free = omap_iommu_domain_free,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001281 .attach_dev = omap_iommu_attach_dev,
1282 .detach_dev = omap_iommu_detach_dev,
1283 .map = omap_iommu_map,
1284 .unmap = omap_iommu_unmap,
Olav Haugan315786e2014-10-25 09:55:16 -07001285 .map_sg = default_iommu_map_sg,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001286 .iova_to_phys = omap_iommu_iova_to_phys,
Laurent Pinchart07a02032014-02-28 14:42:38 -06001287 .add_device = omap_iommu_add_device,
1288 .remove_device = omap_iommu_remove_device,
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +02001289 .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001290};
1291
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001292static int __init omap_iommu_init(void)
1293{
1294 struct kmem_cache *p;
1295 const unsigned long flags = SLAB_HWCACHE_ALIGN;
1296 size_t align = 1 << 10; /* L2 pagetable alignement */
Thierry Redingf938aab2015-02-06 11:44:06 +01001297 struct device_node *np;
1298
1299 np = of_find_matching_node(NULL, omap_iommu_of_match);
1300 if (!np)
1301 return 0;
1302
1303 of_node_put(np);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001304
1305 p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
1306 iopte_cachep_ctor);
1307 if (!p)
1308 return -ENOMEM;
1309 iopte_cachep = p;
1310
Joerg Roedela65bc642011-09-06 17:56:07 +02001311 bus_set_iommu(&platform_bus_type, &omap_iommu_ops);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001312
Suman Anna61c75352014-10-22 17:22:30 -05001313 omap_iommu_debugfs_init();
1314
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001315 return platform_driver_register(&omap_iommu_driver);
1316}
Ohad Ben-Cohen435792d2012-02-26 12:14:14 +02001317subsys_initcall(omap_iommu_init);
Suman Anna0cdbf722015-07-20 17:33:24 -05001318/* must be ready before omap3isp is probed */