blob: 36278870e84a7b1d7350772beb2c8e2aee11fa40 [file] [log] [blame]
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001/*
2 * omap iommu: tlb and pagetable primitives
3 *
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -08004 * Copyright (C) 2008-2010 Nokia Corporation
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02005 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
7 * Paul Mundt and Toshihiro Kobayashi
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/err.h>
15#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090016#include <linux/slab.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020017#include <linux/interrupt.h>
18#include <linux/ioport.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020019#include <linux/platform_device.h>
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030020#include <linux/iommu.h>
Tony Lindgrenc8d35c82012-11-02 12:24:03 -070021#include <linux/omap-iommu.h>
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030022#include <linux/mutex.h>
23#include <linux/spinlock.h>
Tony Lindgrened1c7de2012-11-02 12:24:06 -070024#include <linux/io.h>
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -060025#include <linux/pm_runtime.h>
Florian Vaussard3c927482014-02-28 14:42:36 -060026#include <linux/of.h>
27#include <linux/of_iommu.h>
28#include <linux/of_irq.h>
Suman Anna7d682772014-09-04 17:27:30 -050029#include <linux/of_platform.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020030
31#include <asm/cacheflush.h>
32
Tony Lindgren2ab7c842012-11-02 12:24:14 -070033#include <linux/platform_data/iommu-omap.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020034
Ido Yariv2f7702a2012-11-02 12:24:00 -070035#include "omap-iopgtable.h"
Tony Lindgrened1c7de2012-11-02 12:24:06 -070036#include "omap-iommu.h"
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020037
Suman Anna5acc97d2014-03-17 20:31:34 -050038#define to_iommu(dev) \
39 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)))
40
Hiroshi DOYU37c28362010-04-27 05:37:12 +000041#define for_each_iotlb_cr(obj, n, __i, cr) \
42 for (__i = 0; \
43 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
44 __i++)
45
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +020046/* bitmap of the page sizes currently supported */
47#define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
48
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030049/**
50 * struct omap_iommu_domain - omap iommu domain
51 * @pgtable: the page table
52 * @iommu_dev: an omap iommu device attached to this domain. only a single
53 * iommu device can be attached for now.
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050054 * @dev: Device using this domain.
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030055 * @lock: domain lock, should be taken when attaching/detaching
56 */
57struct omap_iommu_domain {
58 u32 *pgtable;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030059 struct omap_iommu *iommu_dev;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050060 struct device *dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030061 spinlock_t lock;
62};
63
Ido Yariv7bd9e252012-11-02 12:24:09 -070064#define MMU_LOCK_BASE_SHIFT 10
65#define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT)
66#define MMU_LOCK_BASE(x) \
67 ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT)
68
69#define MMU_LOCK_VICT_SHIFT 4
70#define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT)
71#define MMU_LOCK_VICT(x) \
72 ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT)
73
74struct iotlb_lock {
75 short base;
76 short vict;
77};
78
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020079/* accommodate the difference between omap1 and omap2/3 */
80static const struct iommu_functions *arch_iommu;
81
82static struct platform_driver omap_iommu_driver;
83static struct kmem_cache *iopte_cachep;
84
85/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030086 * omap_install_iommu_arch - Install archtecure specific iommu functions
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020087 * @ops: a pointer to architecture specific iommu functions
88 *
89 * There are several kind of iommu algorithm(tlb, pagetable) among
90 * omap series. This interface installs such an iommu algorighm.
91 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030092int omap_install_iommu_arch(const struct iommu_functions *ops)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020093{
94 if (arch_iommu)
95 return -EBUSY;
96
97 arch_iommu = ops;
98 return 0;
99}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300100EXPORT_SYMBOL_GPL(omap_install_iommu_arch);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200101
102/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300103 * omap_uninstall_iommu_arch - Uninstall archtecure specific iommu functions
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200104 * @ops: a pointer to architecture specific iommu functions
105 *
106 * This interface uninstalls the iommu algorighm installed previously.
107 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300108void omap_uninstall_iommu_arch(const struct iommu_functions *ops)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200109{
110 if (arch_iommu != ops)
111 pr_err("%s: not your arch\n", __func__);
112
113 arch_iommu = NULL;
114}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300115EXPORT_SYMBOL_GPL(omap_uninstall_iommu_arch);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200116
117/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300118 * omap_iommu_save_ctx - Save registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200119 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200120 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200121void omap_iommu_save_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200122{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200123 struct omap_iommu *obj = dev_to_omap_iommu(dev);
124
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200125 arch_iommu->save_ctx(obj);
126}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300127EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200128
129/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300130 * omap_iommu_restore_ctx - Restore registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200131 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200132 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200133void omap_iommu_restore_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200134{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200135 struct omap_iommu *obj = dev_to_omap_iommu(dev);
136
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200137 arch_iommu->restore_ctx(obj);
138}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300139EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200140
141/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300142 * omap_iommu_arch_version - Return running iommu arch version
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200143 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300144u32 omap_iommu_arch_version(void)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200145{
146 return arch_iommu->version;
147}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300148EXPORT_SYMBOL_GPL(omap_iommu_arch_version);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200149
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300150static int iommu_enable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200151{
152 int err;
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600153 struct platform_device *pdev = to_platform_device(obj->dev);
154 struct iommu_platform_data *pdata = pdev->dev.platform_data;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200155
Martin Hostettleref4815a2011-02-24 12:51:31 -0800156 if (!arch_iommu)
157 return -ENODEV;
158
Florian Vaussard90e569c2014-02-28 14:42:34 -0600159 if (pdata && pdata->deassert_reset) {
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600160 err = pdata->deassert_reset(pdev, pdata->reset_name);
161 if (err) {
162 dev_err(obj->dev, "deassert_reset failed: %d\n", err);
163 return err;
164 }
165 }
166
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600167 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200168
169 err = arch_iommu->enable(obj);
170
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200171 return err;
172}
173
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300174static void iommu_disable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200175{
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600176 struct platform_device *pdev = to_platform_device(obj->dev);
177 struct iommu_platform_data *pdata = pdev->dev.platform_data;
178
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200179 arch_iommu->disable(obj);
180
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600181 pm_runtime_put_sync(obj->dev);
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600182
Florian Vaussard90e569c2014-02-28 14:42:34 -0600183 if (pdata && pdata->assert_reset)
Omar Ramirez Luna72b15b62012-11-19 19:05:50 -0600184 pdata->assert_reset(pdev, pdata->reset_name);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200185}
186
187/*
188 * TLB operations
189 */
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300190void omap_iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200191{
192 BUG_ON(!cr || !e);
193
194 arch_iommu->cr_to_e(cr, e);
195}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300196EXPORT_SYMBOL_GPL(omap_iotlb_cr_to_e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200197
198static inline int iotlb_cr_valid(struct cr_regs *cr)
199{
200 if (!cr)
201 return -EINVAL;
202
203 return arch_iommu->cr_valid(cr);
204}
205
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300206static inline struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200207 struct iotlb_entry *e)
208{
209 if (!e)
210 return NULL;
211
212 return arch_iommu->alloc_cr(obj, e);
213}
214
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300215static u32 iotlb_cr_to_virt(struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200216{
217 return arch_iommu->cr_to_virt(cr);
218}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200219
220static u32 get_iopte_attr(struct iotlb_entry *e)
221{
222 return arch_iommu->get_pte_attr(e);
223}
224
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300225static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200226{
227 return arch_iommu->fault_isr(obj, da);
228}
229
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300230static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200231{
232 u32 val;
233
234 val = iommu_read_reg(obj, MMU_LOCK);
235
236 l->base = MMU_LOCK_BASE(val);
237 l->vict = MMU_LOCK_VICT(val);
238
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200239}
240
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300241static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200242{
243 u32 val;
244
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200245 val = (l->base << MMU_LOCK_BASE_SHIFT);
246 val |= (l->vict << MMU_LOCK_VICT_SHIFT);
247
248 iommu_write_reg(obj, val, MMU_LOCK);
249}
250
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300251static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200252{
253 arch_iommu->tlb_read_cr(obj, cr);
254}
255
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300256static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200257{
258 arch_iommu->tlb_load_cr(obj, cr);
259
260 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
261 iommu_write_reg(obj, 1, MMU_LD_TLB);
262}
263
264/**
265 * iotlb_dump_cr - Dump an iommu tlb entry into buf
266 * @obj: target iommu
267 * @cr: contents of cam and ram register
268 * @buf: output buffer
269 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300270static inline ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr,
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200271 char *buf)
272{
273 BUG_ON(!cr || !buf);
274
275 return arch_iommu->dump_cr(obj, cr, buf);
276}
277
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000278/* only used in iotlb iteration for-loop */
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300279static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000280{
281 struct cr_regs cr;
282 struct iotlb_lock l;
283
284 iotlb_lock_get(obj, &l);
285 l.vict = n;
286 iotlb_lock_set(obj, &l);
287 iotlb_read_cr(obj, &cr);
288
289 return cr;
290}
291
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200292/**
293 * load_iotlb_entry - Set an iommu tlb entry
294 * @obj: target iommu
295 * @e: an iommu tlb entry info
296 **/
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300297#ifdef PREFETCH_IOTLB
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300298static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200299{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200300 int err = 0;
301 struct iotlb_lock l;
302 struct cr_regs *cr;
303
304 if (!obj || !obj->nr_tlb_entries || !e)
305 return -EINVAL;
306
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600307 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200308
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000309 iotlb_lock_get(obj, &l);
310 if (l.base == obj->nr_tlb_entries) {
311 dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200312 err = -EBUSY;
313 goto out;
314 }
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000315 if (!e->prsvd) {
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000316 int i;
317 struct cr_regs tmp;
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000318
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000319 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000320 if (!iotlb_cr_valid(&tmp))
321 break;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000322
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000323 if (i == obj->nr_tlb_entries) {
324 dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
325 err = -EBUSY;
326 goto out;
327 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000328
329 iotlb_lock_get(obj, &l);
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000330 } else {
331 l.vict = l.base;
332 iotlb_lock_set(obj, &l);
333 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200334
335 cr = iotlb_alloc_cr(obj, e);
336 if (IS_ERR(cr)) {
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600337 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200338 return PTR_ERR(cr);
339 }
340
341 iotlb_load_cr(obj, cr);
342 kfree(cr);
343
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000344 if (e->prsvd)
345 l.base++;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200346 /* increment victim for next tlb load */
347 if (++l.vict == obj->nr_tlb_entries)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000348 l.vict = l.base;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200349 iotlb_lock_set(obj, &l);
350out:
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600351 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200352 return err;
353}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200354
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300355#else /* !PREFETCH_IOTLB */
356
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300357static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300358{
359 return 0;
360}
361
362#endif /* !PREFETCH_IOTLB */
363
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300364static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300365{
366 return load_iotlb_entry(obj, e);
367}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200368
369/**
370 * flush_iotlb_page - Clear an iommu tlb entry
371 * @obj: target iommu
372 * @da: iommu device virtual address
373 *
374 * Clear an iommu tlb entry which includes 'da' address.
375 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300376static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200377{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200378 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000379 struct cr_regs cr;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200380
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600381 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200382
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000383 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200384 u32 start;
385 size_t bytes;
386
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200387 if (!iotlb_cr_valid(&cr))
388 continue;
389
390 start = iotlb_cr_to_virt(&cr);
391 bytes = iopgsz_to_bytes(cr.cam & 3);
392
393 if ((start <= da) && (da < start + bytes)) {
394 dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",
395 __func__, start, da, bytes);
Hari Kanigeri0fa035e2010-08-20 13:50:18 +0000396 iotlb_load_cr(obj, &cr);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200397 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
Laurent Pinchartf7129a02014-03-07 23:47:03 +0100398 break;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200399 }
400 }
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600401 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200402
403 if (i == obj->nr_tlb_entries)
404 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
405}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200406
407/**
408 * flush_iotlb_all - Clear all iommu tlb entries
409 * @obj: target iommu
410 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300411static void flush_iotlb_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200412{
413 struct iotlb_lock l;
414
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600415 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200416
417 l.base = 0;
418 l.vict = 0;
419 iotlb_lock_set(obj, &l);
420
421 iommu_write_reg(obj, 1, MMU_GFLUSH);
422
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600423 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200424}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200425
Arnd Bergmanne4efd942011-10-02 14:34:05 -0400426#if defined(CONFIG_OMAP_IOMMU_DEBUG) || defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE)
Kanigeri, Hariddfa9752010-05-24 02:01:51 +0000427
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300428ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200429{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200430 if (!obj || !buf)
431 return -EINVAL;
432
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600433 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200434
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700435 bytes = arch_iommu->dump_ctx(obj, buf, bytes);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200436
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600437 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200438
439 return bytes;
440}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300441EXPORT_SYMBOL_GPL(omap_iommu_dump_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200442
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300443static int
444__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200445{
446 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000447 struct iotlb_lock saved;
448 struct cr_regs tmp;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200449 struct cr_regs *p = crs;
450
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600451 pm_runtime_get_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200452 iotlb_lock_get(obj, &saved);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200453
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000454 for_each_iotlb_cr(obj, num, i, tmp) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200455 if (!iotlb_cr_valid(&tmp))
456 continue;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200457 *p++ = tmp;
458 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000459
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200460 iotlb_lock_set(obj, &saved);
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600461 pm_runtime_put_sync(obj->dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200462
463 return p - crs;
464}
465
466/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300467 * omap_dump_tlb_entries - dump cr arrays to given buffer
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200468 * @obj: target iommu
469 * @buf: output buffer
470 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300471size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200472{
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700473 int i, num;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200474 struct cr_regs *cr;
475 char *p = buf;
476
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700477 num = bytes / sizeof(*cr);
478 num = min(obj->nr_tlb_entries, num);
479
480 cr = kcalloc(num, sizeof(*cr), GFP_KERNEL);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200481 if (!cr)
482 return 0;
483
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700484 num = __dump_tlb_entries(obj, cr, num);
485 for (i = 0; i < num; i++)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200486 p += iotlb_dump_cr(obj, cr + i, p);
487 kfree(cr);
488
489 return p - buf;
490}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300491EXPORT_SYMBOL_GPL(omap_dump_tlb_entries);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200492
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300493int omap_foreach_iommu_device(void *data, int (*fn)(struct device *, void *))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200494{
495 return driver_for_each_device(&omap_iommu_driver.driver,
496 NULL, data, fn);
497}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300498EXPORT_SYMBOL_GPL(omap_foreach_iommu_device);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200499
500#endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */
501
502/*
503 * H/W pagetable operations
504 */
505static void flush_iopgd_range(u32 *first, u32 *last)
506{
507 /* FIXME: L2 cache should be taken care of if it exists */
508 do {
509 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
510 : : "r" (first));
511 first += L1_CACHE_BYTES / sizeof(*first);
512 } while (first <= last);
513}
514
515static void flush_iopte_range(u32 *first, u32 *last)
516{
517 /* FIXME: L2 cache should be taken care of if it exists */
518 do {
519 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
520 : : "r" (first));
521 first += L1_CACHE_BYTES / sizeof(*first);
522 } while (first <= last);
523}
524
525static void iopte_free(u32 *iopte)
526{
527 /* Note: freed iopte's must be clean ready for re-use */
Zhouyi Zhoue28045a2014-03-05 18:20:19 +0800528 if (iopte)
529 kmem_cache_free(iopte_cachep, iopte);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200530}
531
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300532static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200533{
534 u32 *iopte;
535
536 /* a table has already existed */
537 if (*iopgd)
538 goto pte_ready;
539
540 /*
541 * do the allocation outside the page table lock
542 */
543 spin_unlock(&obj->page_table_lock);
544 iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
545 spin_lock(&obj->page_table_lock);
546
547 if (!*iopgd) {
548 if (!iopte)
549 return ERR_PTR(-ENOMEM);
550
551 *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
552 flush_iopgd_range(iopgd, iopgd);
553
554 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
555 } else {
556 /* We raced, free the reduniovant table */
557 iopte_free(iopte);
558 }
559
560pte_ready:
561 iopte = iopte_offset(iopgd, da);
562
563 dev_vdbg(obj->dev,
564 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
565 __func__, da, iopgd, *iopgd, iopte, *iopte);
566
567 return iopte;
568}
569
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300570static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200571{
572 u32 *iopgd = iopgd_offset(obj, da);
573
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300574 if ((da | pa) & ~IOSECTION_MASK) {
575 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
576 __func__, da, pa, IOSECTION_SIZE);
577 return -EINVAL;
578 }
579
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200580 *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
581 flush_iopgd_range(iopgd, iopgd);
582 return 0;
583}
584
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300585static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200586{
587 u32 *iopgd = iopgd_offset(obj, da);
588 int i;
589
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300590 if ((da | pa) & ~IOSUPER_MASK) {
591 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
592 __func__, da, pa, IOSUPER_SIZE);
593 return -EINVAL;
594 }
595
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200596 for (i = 0; i < 16; i++)
597 *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
598 flush_iopgd_range(iopgd, iopgd + 15);
599 return 0;
600}
601
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300602static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200603{
604 u32 *iopgd = iopgd_offset(obj, da);
605 u32 *iopte = iopte_alloc(obj, iopgd, da);
606
607 if (IS_ERR(iopte))
608 return PTR_ERR(iopte);
609
610 *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
611 flush_iopte_range(iopte, iopte);
612
613 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
614 __func__, da, pa, iopte, *iopte);
615
616 return 0;
617}
618
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300619static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200620{
621 u32 *iopgd = iopgd_offset(obj, da);
622 u32 *iopte = iopte_alloc(obj, iopgd, da);
623 int i;
624
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300625 if ((da | pa) & ~IOLARGE_MASK) {
626 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
627 __func__, da, pa, IOLARGE_SIZE);
628 return -EINVAL;
629 }
630
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200631 if (IS_ERR(iopte))
632 return PTR_ERR(iopte);
633
634 for (i = 0; i < 16; i++)
635 *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
636 flush_iopte_range(iopte, iopte + 15);
637 return 0;
638}
639
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300640static int
641iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200642{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300643 int (*fn)(struct omap_iommu *, u32, u32, u32);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200644 u32 prot;
645 int err;
646
647 if (!obj || !e)
648 return -EINVAL;
649
650 switch (e->pgsz) {
651 case MMU_CAM_PGSZ_16M:
652 fn = iopgd_alloc_super;
653 break;
654 case MMU_CAM_PGSZ_1M:
655 fn = iopgd_alloc_section;
656 break;
657 case MMU_CAM_PGSZ_64K:
658 fn = iopte_alloc_large;
659 break;
660 case MMU_CAM_PGSZ_4K:
661 fn = iopte_alloc_page;
662 break;
663 default:
664 fn = NULL;
665 BUG();
666 break;
667 }
668
669 prot = get_iopte_attr(e);
670
671 spin_lock(&obj->page_table_lock);
672 err = fn(obj, e->da, e->pa, prot);
673 spin_unlock(&obj->page_table_lock);
674
675 return err;
676}
677
678/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300679 * omap_iopgtable_store_entry - Make an iommu pte entry
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200680 * @obj: target iommu
681 * @e: an iommu tlb entry info
682 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300683int omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200684{
685 int err;
686
687 flush_iotlb_page(obj, e->da);
688 err = iopgtable_store_entry_core(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200689 if (!err)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300690 prefetch_iotlb_entry(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200691 return err;
692}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300693EXPORT_SYMBOL_GPL(omap_iopgtable_store_entry);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200694
695/**
696 * iopgtable_lookup_entry - Lookup an iommu pte entry
697 * @obj: target iommu
698 * @da: iommu device virtual address
699 * @ppgd: iommu pgd entry pointer to be returned
700 * @ppte: iommu pte entry pointer to be returned
701 **/
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300702static void
703iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200704{
705 u32 *iopgd, *iopte = NULL;
706
707 iopgd = iopgd_offset(obj, da);
708 if (!*iopgd)
709 goto out;
710
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300711 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200712 iopte = iopte_offset(iopgd, da);
713out:
714 *ppgd = iopgd;
715 *ppte = iopte;
716}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200717
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300718static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200719{
720 size_t bytes;
721 u32 *iopgd = iopgd_offset(obj, da);
722 int nent = 1;
723
724 if (!*iopgd)
725 return 0;
726
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300727 if (iopgd_is_table(*iopgd)) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200728 int i;
729 u32 *iopte = iopte_offset(iopgd, da);
730
731 bytes = IOPTE_SIZE;
732 if (*iopte & IOPTE_LARGE) {
733 nent *= 16;
734 /* rewind to the 1st entry */
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -0800735 iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200736 }
737 bytes *= nent;
738 memset(iopte, 0, nent * sizeof(*iopte));
739 flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
740
741 /*
742 * do table walk to check if this table is necessary or not
743 */
744 iopte = iopte_offset(iopgd, 0);
745 for (i = 0; i < PTRS_PER_IOPTE; i++)
746 if (iopte[i])
747 goto out;
748
749 iopte_free(iopte);
750 nent = 1; /* for the next L1 entry */
751 } else {
752 bytes = IOPGD_SIZE;
Hiroshi DOYUdcc730d2009-10-22 14:46:32 -0700753 if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200754 nent *= 16;
755 /* rewind to the 1st entry */
Hiroshi DOYU8d33ea52010-02-15 10:03:32 -0800756 iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200757 }
758 bytes *= nent;
759 }
760 memset(iopgd, 0, nent * sizeof(*iopgd));
761 flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
762out:
763 return bytes;
764}
765
766/**
767 * iopgtable_clear_entry - Remove an iommu pte entry
768 * @obj: target iommu
769 * @da: iommu device virtual address
770 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300771static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200772{
773 size_t bytes;
774
775 spin_lock(&obj->page_table_lock);
776
777 bytes = iopgtable_clear_entry_core(obj, da);
778 flush_iotlb_page(obj, da);
779
780 spin_unlock(&obj->page_table_lock);
781
782 return bytes;
783}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200784
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300785static void iopgtable_clear_entry_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200786{
787 int i;
788
789 spin_lock(&obj->page_table_lock);
790
791 for (i = 0; i < PTRS_PER_IOPGD; i++) {
792 u32 da;
793 u32 *iopgd;
794
795 da = i << IOPGD_SHIFT;
796 iopgd = iopgd_offset(obj, da);
797
798 if (!*iopgd)
799 continue;
800
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300801 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200802 iopte_free(iopte_offset(iopgd, 0));
803
804 *iopgd = 0;
805 flush_iopgd_range(iopgd, iopgd);
806 }
807
808 flush_iotlb_all(obj);
809
810 spin_unlock(&obj->page_table_lock);
811}
812
813/*
814 * Device IOMMU generic operations
815 */
816static irqreturn_t iommu_fault_handler(int irq, void *data)
817{
David Cohend594f1f2011-02-16 19:35:51 +0000818 u32 da, errs;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200819 u32 *iopgd, *iopte;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300820 struct omap_iommu *obj = data;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400821 struct iommu_domain *domain = obj->domain;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200822
823 if (!obj->refcount)
824 return IRQ_NONE;
825
David Cohend594f1f2011-02-16 19:35:51 +0000826 errs = iommu_report_fault(obj, &da);
Laurent Pinchartc56b2dd2011-05-10 16:56:46 +0200827 if (errs == 0)
828 return IRQ_HANDLED;
David Cohend594f1f2011-02-16 19:35:51 +0000829
830 /* Fault callback or TLB/PTE Dynamic loading */
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400831 if (!report_iommu_fault(domain, obj->dev, da, 0))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200832 return IRQ_HANDLED;
833
Hiroshi DOYU37b29812010-05-24 02:01:52 +0000834 iommu_disable(obj);
835
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200836 iopgd = iopgd_offset(obj, da);
837
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300838 if (!iopgd_is_table(*iopgd)) {
Suman Annab6c2e092013-05-30 18:10:59 -0500839 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",
840 obj->name, errs, da, iopgd, *iopgd);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200841 return IRQ_NONE;
842 }
843
844 iopte = iopte_offset(iopgd, da);
845
Suman Annab6c2e092013-05-30 18:10:59 -0500846 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
847 obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200848
849 return IRQ_NONE;
850}
851
852static int device_match_by_alias(struct device *dev, void *data)
853{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300854 struct omap_iommu *obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200855 const char *name = data;
856
857 pr_debug("%s: %s %s\n", __func__, obj->name, name);
858
859 return strcmp(obj->name, name) == 0;
860}
861
862/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300863 * omap_iommu_attach() - attach iommu device to an iommu domain
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200864 * @name: name of target omap iommu device
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300865 * @iopgd: page table
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200866 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200867static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200868{
Suman Anna7ee08b9e2014-02-28 14:42:33 -0600869 int err;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200870 struct device *dev;
871 struct omap_iommu *obj;
872
873 dev = driver_find_device(&omap_iommu_driver.driver, NULL,
874 (void *)name,
875 device_match_by_alias);
876 if (!dev)
Suman Anna7ee08b9e2014-02-28 14:42:33 -0600877 return ERR_PTR(-ENODEV);
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200878
879 obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200880
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300881 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200882
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300883 /* an iommu device can only be attached once */
884 if (++obj->refcount > 1) {
885 dev_err(dev, "%s: already attached!\n", obj->name);
886 err = -EBUSY;
887 goto err_enable;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200888 }
889
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300890 obj->iopgd = iopgd;
891 err = iommu_enable(obj);
892 if (err)
893 goto err_enable;
894 flush_iotlb_all(obj);
895
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300896 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200897
898 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
899 return obj;
900
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200901err_enable:
902 obj->refcount--;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300903 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200904 return ERR_PTR(err);
905}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200906
907/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300908 * omap_iommu_detach - release iommu device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200909 * @obj: target iommu
910 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300911static void omap_iommu_detach(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200912{
Roel Kluinacf9d462010-01-08 10:29:05 -0800913 if (!obj || IS_ERR(obj))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200914 return;
915
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300916 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200917
918 if (--obj->refcount == 0)
919 iommu_disable(obj);
920
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300921 obj->iopgd = NULL;
922
923 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200924
925 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
926}
David Cohend594f1f2011-02-16 19:35:51 +0000927
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200928/*
929 * OMAP Device MMU(IOMMU) detection
930 */
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -0800931static int omap_iommu_probe(struct platform_device *pdev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200932{
933 int err = -ENODEV;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200934 int irq;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300935 struct omap_iommu *obj;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200936 struct resource *res;
937 struct iommu_platform_data *pdata = pdev->dev.platform_data;
Florian Vaussard3c927482014-02-28 14:42:36 -0600938 struct device_node *of = pdev->dev.of_node;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200939
Suman Annaf129b3d2014-02-28 14:42:32 -0600940 obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200941 if (!obj)
942 return -ENOMEM;
943
Florian Vaussard3c927482014-02-28 14:42:36 -0600944 if (of) {
945 obj->name = dev_name(&pdev->dev);
946 obj->nr_tlb_entries = 32;
947 err = of_property_read_u32(of, "ti,#tlb-entries",
948 &obj->nr_tlb_entries);
949 if (err && err != -EINVAL)
950 return err;
951 if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
952 return -EINVAL;
Suman Annab148d5f2014-02-28 14:42:37 -0600953 if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
954 obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
Florian Vaussard3c927482014-02-28 14:42:36 -0600955 } else {
956 obj->nr_tlb_entries = pdata->nr_tlb_entries;
957 obj->name = pdata->name;
Florian Vaussard3c927482014-02-28 14:42:36 -0600958 }
Florian Vaussard3c927482014-02-28 14:42:36 -0600959
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200960 obj->dev = &pdev->dev;
961 obj->ctx = (void *)obj + sizeof(*obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200962
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300963 spin_lock_init(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200964 spin_lock_init(&obj->page_table_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200965
966 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
Suman Annaf129b3d2014-02-28 14:42:32 -0600967 obj->regbase = devm_ioremap_resource(obj->dev, res);
968 if (IS_ERR(obj->regbase))
969 return PTR_ERR(obj->regbase);
Aaro Koskinenda4a0f72011-03-14 12:28:32 +0000970
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200971 irq = platform_get_irq(pdev, 0);
Suman Annaf129b3d2014-02-28 14:42:32 -0600972 if (irq < 0)
973 return -ENODEV;
974
975 err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,
976 dev_name(obj->dev), obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200977 if (err < 0)
Suman Annaf129b3d2014-02-28 14:42:32 -0600978 return err;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200979 platform_set_drvdata(pdev, obj);
980
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600981 pm_runtime_irq_safe(obj->dev);
982 pm_runtime_enable(obj->dev);
983
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200984 dev_info(&pdev->dev, "%s registered\n", obj->name);
985 return 0;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200986}
987
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -0800988static int omap_iommu_remove(struct platform_device *pdev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200989{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300990 struct omap_iommu *obj = platform_get_drvdata(pdev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200991
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200992 iopgtable_clear_entry_all(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200993
Omar Ramirez Lunaebf7cda2012-11-19 19:05:51 -0600994 pm_runtime_disable(obj->dev);
995
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200996 dev_info(&pdev->dev, "%s removed\n", obj->name);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200997 return 0;
998}
999
Kiran Padwald943b0f2014-09-11 19:07:36 +05301000static const struct of_device_id omap_iommu_of_match[] = {
Florian Vaussard3c927482014-02-28 14:42:36 -06001001 { .compatible = "ti,omap2-iommu" },
1002 { .compatible = "ti,omap4-iommu" },
1003 { .compatible = "ti,dra7-iommu" },
1004 {},
1005};
1006MODULE_DEVICE_TABLE(of, omap_iommu_of_match);
1007
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001008static struct platform_driver omap_iommu_driver = {
1009 .probe = omap_iommu_probe,
Greg Kroah-Hartmand34d6512012-12-21 15:05:21 -08001010 .remove = omap_iommu_remove,
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001011 .driver = {
1012 .name = "omap-iommu",
Florian Vaussard3c927482014-02-28 14:42:36 -06001013 .of_match_table = of_match_ptr(omap_iommu_of_match),
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001014 },
1015};
1016
1017static void iopte_cachep_ctor(void *iopte)
1018{
1019 clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
1020}
1021
Laurent Pinchart286f6002014-03-08 00:44:38 +01001022static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
Tony Lindgrened1c7de2012-11-02 12:24:06 -07001023{
1024 memset(e, 0, sizeof(*e));
1025
1026 e->da = da;
1027 e->pa = pa;
Suman Annad760e3e2014-03-17 20:31:32 -05001028 e->valid = MMU_CAM_V;
Tony Lindgrened1c7de2012-11-02 12:24:06 -07001029 /* FIXME: add OMAP1 support */
Laurent Pinchart286f6002014-03-08 00:44:38 +01001030 e->pgsz = pgsz;
1031 e->endian = MMU_RAM_ENDIAN_LITTLE;
1032 e->elsz = MMU_RAM_ELSZ_8;
1033 e->mixed = 0;
Tony Lindgrened1c7de2012-11-02 12:24:06 -07001034
1035 return iopgsz_to_bytes(e->pgsz);
1036}
1037
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001038static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001039 phys_addr_t pa, size_t bytes, int prot)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001040{
1041 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001042 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001043 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001044 struct iotlb_entry e;
1045 int omap_pgsz;
Laurent Pinchart286f6002014-03-08 00:44:38 +01001046 u32 ret;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001047
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001048 omap_pgsz = bytes_to_iopgsz(bytes);
1049 if (omap_pgsz < 0) {
1050 dev_err(dev, "invalid size to map: %d\n", bytes);
1051 return -EINVAL;
1052 }
1053
1054 dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes);
1055
Laurent Pinchart286f6002014-03-08 00:44:38 +01001056 iotlb_init_entry(&e, da, pa, omap_pgsz);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001057
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001058 ret = omap_iopgtable_store_entry(oiommu, &e);
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001059 if (ret)
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001060 dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001061
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001062 return ret;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001063}
1064
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001065static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
1066 size_t size)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001067{
1068 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001069 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001070 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001071
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001072 dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001073
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001074 return iopgtable_clear_entry(oiommu, da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001075}
1076
1077static int
1078omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1079{
1080 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001081 struct omap_iommu *oiommu;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001082 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001083 int ret = 0;
1084
Suman Annae3f595b2014-09-04 17:27:29 -05001085 if (!arch_data || !arch_data->name) {
1086 dev_err(dev, "device doesn't have an associated iommu\n");
1087 return -EINVAL;
1088 }
1089
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001090 spin_lock(&omap_domain->lock);
1091
1092 /* only a single device is supported per domain for now */
1093 if (omap_domain->iommu_dev) {
1094 dev_err(dev, "iommu domain is already attached\n");
1095 ret = -EBUSY;
1096 goto out;
1097 }
1098
1099 /* get a handle to and enable the omap iommu */
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001100 oiommu = omap_iommu_attach(arch_data->name, omap_domain->pgtable);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001101 if (IS_ERR(oiommu)) {
1102 ret = PTR_ERR(oiommu);
1103 dev_err(dev, "can't get omap iommu: %d\n", ret);
1104 goto out;
1105 }
1106
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001107 omap_domain->iommu_dev = arch_data->iommu_dev = oiommu;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001108 omap_domain->dev = dev;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -04001109 oiommu->domain = domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001110
1111out:
1112 spin_unlock(&omap_domain->lock);
1113 return ret;
1114}
1115
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001116static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
1117 struct device *dev)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001118{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001119 struct omap_iommu *oiommu = dev_to_omap_iommu(dev);
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001120 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001121
1122 /* only a single device is supported per domain for now */
1123 if (omap_domain->iommu_dev != oiommu) {
1124 dev_err(dev, "invalid iommu device\n");
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001125 return;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001126 }
1127
1128 iopgtable_clear_entry_all(oiommu);
1129
1130 omap_iommu_detach(oiommu);
1131
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001132 omap_domain->iommu_dev = arch_data->iommu_dev = NULL;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001133 omap_domain->dev = NULL;
1134}
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001135
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001136static void omap_iommu_detach_dev(struct iommu_domain *domain,
1137 struct device *dev)
1138{
1139 struct omap_iommu_domain *omap_domain = domain->priv;
1140
1141 spin_lock(&omap_domain->lock);
1142 _omap_iommu_detach_dev(omap_domain, dev);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001143 spin_unlock(&omap_domain->lock);
1144}
1145
1146static int omap_iommu_domain_init(struct iommu_domain *domain)
1147{
1148 struct omap_iommu_domain *omap_domain;
1149
1150 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
1151 if (!omap_domain) {
1152 pr_err("kzalloc failed\n");
1153 goto out;
1154 }
1155
1156 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL);
1157 if (!omap_domain->pgtable) {
1158 pr_err("kzalloc failed\n");
1159 goto fail_nomem;
1160 }
1161
1162 /*
1163 * should never fail, but please keep this around to ensure
1164 * we keep the hardware happy
1165 */
1166 BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE));
1167
1168 clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
1169 spin_lock_init(&omap_domain->lock);
1170
1171 domain->priv = omap_domain;
1172
Joerg Roedel2c6edb02012-01-26 19:40:55 +01001173 domain->geometry.aperture_start = 0;
1174 domain->geometry.aperture_end = (1ULL << 32) - 1;
1175 domain->geometry.force_aperture = true;
1176
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001177 return 0;
1178
1179fail_nomem:
1180 kfree(omap_domain);
1181out:
1182 return -ENOMEM;
1183}
1184
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001185static void omap_iommu_domain_destroy(struct iommu_domain *domain)
1186{
1187 struct omap_iommu_domain *omap_domain = domain->priv;
1188
1189 domain->priv = NULL;
1190
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001191 /*
1192 * An iommu device is still attached
1193 * (currently, only one device can be attached) ?
1194 */
1195 if (omap_domain->iommu_dev)
1196 _omap_iommu_detach_dev(omap_domain, omap_domain->dev);
1197
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001198 kfree(omap_domain->pgtable);
1199 kfree(omap_domain);
1200}
1201
1202static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
Varun Sethibb5547ac2013-03-29 01:23:58 +05301203 dma_addr_t da)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001204{
1205 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001206 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001207 struct device *dev = oiommu->dev;
1208 u32 *pgd, *pte;
1209 phys_addr_t ret = 0;
1210
1211 iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
1212
1213 if (pte) {
1214 if (iopte_is_small(*pte))
1215 ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
1216 else if (iopte_is_large(*pte))
1217 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
1218 else
Suman Anna2abfcfb2013-05-30 18:10:38 -05001219 dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte,
1220 (unsigned long long)da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001221 } else {
1222 if (iopgd_is_section(*pgd))
1223 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
1224 else if (iopgd_is_super(*pgd))
1225 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
1226 else
Suman Anna2abfcfb2013-05-30 18:10:38 -05001227 dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd,
1228 (unsigned long long)da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001229 }
1230
1231 return ret;
1232}
1233
Laurent Pinchart07a02032014-02-28 14:42:38 -06001234static int omap_iommu_add_device(struct device *dev)
1235{
1236 struct omap_iommu_arch_data *arch_data;
1237 struct device_node *np;
Suman Anna7d682772014-09-04 17:27:30 -05001238 struct platform_device *pdev;
Laurent Pinchart07a02032014-02-28 14:42:38 -06001239
1240 /*
1241 * Allocate the archdata iommu structure for DT-based devices.
1242 *
1243 * TODO: Simplify this when removing non-DT support completely from the
1244 * IOMMU users.
1245 */
1246 if (!dev->of_node)
1247 return 0;
1248
1249 np = of_parse_phandle(dev->of_node, "iommus", 0);
1250 if (!np)
1251 return 0;
1252
Suman Anna7d682772014-09-04 17:27:30 -05001253 pdev = of_find_device_by_node(np);
1254 if (WARN_ON(!pdev)) {
1255 of_node_put(np);
1256 return -EINVAL;
1257 }
1258
Laurent Pinchart07a02032014-02-28 14:42:38 -06001259 arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL);
1260 if (!arch_data) {
1261 of_node_put(np);
1262 return -ENOMEM;
1263 }
1264
Suman Anna7d682772014-09-04 17:27:30 -05001265 arch_data->name = kstrdup(dev_name(&pdev->dev), GFP_KERNEL);
Laurent Pinchart07a02032014-02-28 14:42:38 -06001266 dev->archdata.iommu = arch_data;
1267
1268 of_node_put(np);
1269
1270 return 0;
1271}
1272
1273static void omap_iommu_remove_device(struct device *dev)
1274{
1275 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
1276
1277 if (!dev->of_node || !arch_data)
1278 return;
1279
1280 kfree(arch_data->name);
1281 kfree(arch_data);
1282}
1283
Thierry Redingb22f6432014-06-27 09:03:12 +02001284static const struct iommu_ops omap_iommu_ops = {
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001285 .domain_init = omap_iommu_domain_init,
1286 .domain_destroy = omap_iommu_domain_destroy,
1287 .attach_dev = omap_iommu_attach_dev,
1288 .detach_dev = omap_iommu_detach_dev,
1289 .map = omap_iommu_map,
1290 .unmap = omap_iommu_unmap,
1291 .iova_to_phys = omap_iommu_iova_to_phys,
Laurent Pinchart07a02032014-02-28 14:42:38 -06001292 .add_device = omap_iommu_add_device,
1293 .remove_device = omap_iommu_remove_device,
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +02001294 .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001295};
1296
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001297static int __init omap_iommu_init(void)
1298{
1299 struct kmem_cache *p;
1300 const unsigned long flags = SLAB_HWCACHE_ALIGN;
1301 size_t align = 1 << 10; /* L2 pagetable alignement */
1302
1303 p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
1304 iopte_cachep_ctor);
1305 if (!p)
1306 return -ENOMEM;
1307 iopte_cachep = p;
1308
Joerg Roedela65bc642011-09-06 17:56:07 +02001309 bus_set_iommu(&platform_bus_type, &omap_iommu_ops);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001310
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001311 return platform_driver_register(&omap_iommu_driver);
1312}
Ohad Ben-Cohen435792d2012-02-26 12:14:14 +02001313/* must be ready before omap3isp is probed */
1314subsys_initcall(omap_iommu_init);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001315
1316static void __exit omap_iommu_exit(void)
1317{
1318 kmem_cache_destroy(iopte_cachep);
1319
1320 platform_driver_unregister(&omap_iommu_driver);
1321}
1322module_exit(omap_iommu_exit);
1323
1324MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives");
1325MODULE_ALIAS("platform:omap-iommu");
1326MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi");
1327MODULE_LICENSE("GPL v2");