blob: d0b1234581be17da531c7c39f18c6a0c48f68d75 [file] [log] [blame]
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001/*
2 * omap iommu: tlb and pagetable primitives
3 *
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -08004 * Copyright (C) 2008-2010 Nokia Corporation
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02005 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
7 * Paul Mundt and Toshihiro Kobayashi
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/err.h>
15#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090016#include <linux/slab.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020017#include <linux/interrupt.h>
18#include <linux/ioport.h>
19#include <linux/clk.h>
20#include <linux/platform_device.h>
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030021#include <linux/iommu.h>
22#include <linux/mutex.h>
23#include <linux/spinlock.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020024
25#include <asm/cacheflush.h>
26
Tony Lindgrence491cf2009-10-20 09:40:47 -070027#include <plat/iommu.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020028
Ohad Ben-Cohenfcf3a6e2011-08-15 23:21:41 +030029#include <plat/iopgtable.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020030
Hiroshi DOYU37c28362010-04-27 05:37:12 +000031#define for_each_iotlb_cr(obj, n, __i, cr) \
32 for (__i = 0; \
33 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
34 __i++)
35
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +020036/* bitmap of the page sizes currently supported */
37#define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
38
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030039/**
40 * struct omap_iommu_domain - omap iommu domain
41 * @pgtable: the page table
42 * @iommu_dev: an omap iommu device attached to this domain. only a single
43 * iommu device can be attached for now.
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050044 * @dev: Device using this domain.
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030045 * @lock: domain lock, should be taken when attaching/detaching
46 */
47struct omap_iommu_domain {
48 u32 *pgtable;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030049 struct omap_iommu *iommu_dev;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -050050 struct device *dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +030051 spinlock_t lock;
52};
53
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020054/* accommodate the difference between omap1 and omap2/3 */
55static const struct iommu_functions *arch_iommu;
56
57static struct platform_driver omap_iommu_driver;
58static struct kmem_cache *iopte_cachep;
59
60/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030061 * omap_install_iommu_arch - Install archtecure specific iommu functions
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020062 * @ops: a pointer to architecture specific iommu functions
63 *
64 * There are several kind of iommu algorithm(tlb, pagetable) among
65 * omap series. This interface installs such an iommu algorighm.
66 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030067int omap_install_iommu_arch(const struct iommu_functions *ops)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020068{
69 if (arch_iommu)
70 return -EBUSY;
71
72 arch_iommu = ops;
73 return 0;
74}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030075EXPORT_SYMBOL_GPL(omap_install_iommu_arch);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020076
77/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030078 * omap_uninstall_iommu_arch - Uninstall archtecure specific iommu functions
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020079 * @ops: a pointer to architecture specific iommu functions
80 *
81 * This interface uninstalls the iommu algorighm installed previously.
82 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030083void omap_uninstall_iommu_arch(const struct iommu_functions *ops)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020084{
85 if (arch_iommu != ops)
86 pr_err("%s: not your arch\n", __func__);
87
88 arch_iommu = NULL;
89}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030090EXPORT_SYMBOL_GPL(omap_uninstall_iommu_arch);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020091
92/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +030093 * omap_iommu_save_ctx - Save registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020094 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020095 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020096void omap_iommu_save_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020097{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +020098 struct omap_iommu *obj = dev_to_omap_iommu(dev);
99
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200100 arch_iommu->save_ctx(obj);
101}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300102EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200103
104/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300105 * omap_iommu_restore_ctx - Restore registers for pm off-mode support
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200106 * @dev: client device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200107 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200108void omap_iommu_restore_ctx(struct device *dev)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200109{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200110 struct omap_iommu *obj = dev_to_omap_iommu(dev);
111
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200112 arch_iommu->restore_ctx(obj);
113}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300114EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200115
116/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300117 * omap_iommu_arch_version - Return running iommu arch version
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200118 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300119u32 omap_iommu_arch_version(void)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200120{
121 return arch_iommu->version;
122}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300123EXPORT_SYMBOL_GPL(omap_iommu_arch_version);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200124
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300125static int iommu_enable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200126{
127 int err;
128
129 if (!obj)
130 return -EINVAL;
131
Martin Hostettleref4815a2011-02-24 12:51:31 -0800132 if (!arch_iommu)
133 return -ENODEV;
134
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200135 clk_enable(obj->clk);
136
137 err = arch_iommu->enable(obj);
138
139 clk_disable(obj->clk);
140 return err;
141}
142
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300143static void iommu_disable(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200144{
145 if (!obj)
146 return;
147
148 clk_enable(obj->clk);
149
150 arch_iommu->disable(obj);
151
152 clk_disable(obj->clk);
153}
154
155/*
156 * TLB operations
157 */
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300158void omap_iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200159{
160 BUG_ON(!cr || !e);
161
162 arch_iommu->cr_to_e(cr, e);
163}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300164EXPORT_SYMBOL_GPL(omap_iotlb_cr_to_e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200165
166static inline int iotlb_cr_valid(struct cr_regs *cr)
167{
168 if (!cr)
169 return -EINVAL;
170
171 return arch_iommu->cr_valid(cr);
172}
173
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300174static inline struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200175 struct iotlb_entry *e)
176{
177 if (!e)
178 return NULL;
179
180 return arch_iommu->alloc_cr(obj, e);
181}
182
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300183static u32 iotlb_cr_to_virt(struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200184{
185 return arch_iommu->cr_to_virt(cr);
186}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200187
188static u32 get_iopte_attr(struct iotlb_entry *e)
189{
190 return arch_iommu->get_pte_attr(e);
191}
192
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300193static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200194{
195 return arch_iommu->fault_isr(obj, da);
196}
197
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300198static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200199{
200 u32 val;
201
202 val = iommu_read_reg(obj, MMU_LOCK);
203
204 l->base = MMU_LOCK_BASE(val);
205 l->vict = MMU_LOCK_VICT(val);
206
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200207}
208
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300209static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200210{
211 u32 val;
212
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200213 val = (l->base << MMU_LOCK_BASE_SHIFT);
214 val |= (l->vict << MMU_LOCK_VICT_SHIFT);
215
216 iommu_write_reg(obj, val, MMU_LOCK);
217}
218
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300219static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200220{
221 arch_iommu->tlb_read_cr(obj, cr);
222}
223
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300224static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200225{
226 arch_iommu->tlb_load_cr(obj, cr);
227
228 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
229 iommu_write_reg(obj, 1, MMU_LD_TLB);
230}
231
232/**
233 * iotlb_dump_cr - Dump an iommu tlb entry into buf
234 * @obj: target iommu
235 * @cr: contents of cam and ram register
236 * @buf: output buffer
237 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300238static inline ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr,
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200239 char *buf)
240{
241 BUG_ON(!cr || !buf);
242
243 return arch_iommu->dump_cr(obj, cr, buf);
244}
245
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000246/* only used in iotlb iteration for-loop */
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300247static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000248{
249 struct cr_regs cr;
250 struct iotlb_lock l;
251
252 iotlb_lock_get(obj, &l);
253 l.vict = n;
254 iotlb_lock_set(obj, &l);
255 iotlb_read_cr(obj, &cr);
256
257 return cr;
258}
259
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200260/**
261 * load_iotlb_entry - Set an iommu tlb entry
262 * @obj: target iommu
263 * @e: an iommu tlb entry info
264 **/
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300265#ifdef PREFETCH_IOTLB
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300266static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200267{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200268 int err = 0;
269 struct iotlb_lock l;
270 struct cr_regs *cr;
271
272 if (!obj || !obj->nr_tlb_entries || !e)
273 return -EINVAL;
274
275 clk_enable(obj->clk);
276
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000277 iotlb_lock_get(obj, &l);
278 if (l.base == obj->nr_tlb_entries) {
279 dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200280 err = -EBUSY;
281 goto out;
282 }
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000283 if (!e->prsvd) {
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000284 int i;
285 struct cr_regs tmp;
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000286
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000287 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000288 if (!iotlb_cr_valid(&tmp))
289 break;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000290
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000291 if (i == obj->nr_tlb_entries) {
292 dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
293 err = -EBUSY;
294 goto out;
295 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000296
297 iotlb_lock_get(obj, &l);
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000298 } else {
299 l.vict = l.base;
300 iotlb_lock_set(obj, &l);
301 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200302
303 cr = iotlb_alloc_cr(obj, e);
304 if (IS_ERR(cr)) {
305 clk_disable(obj->clk);
306 return PTR_ERR(cr);
307 }
308
309 iotlb_load_cr(obj, cr);
310 kfree(cr);
311
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000312 if (e->prsvd)
313 l.base++;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200314 /* increment victim for next tlb load */
315 if (++l.vict == obj->nr_tlb_entries)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000316 l.vict = l.base;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200317 iotlb_lock_set(obj, &l);
318out:
319 clk_disable(obj->clk);
320 return err;
321}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200322
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300323#else /* !PREFETCH_IOTLB */
324
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300325static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300326{
327 return 0;
328}
329
330#endif /* !PREFETCH_IOTLB */
331
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300332static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300333{
334 return load_iotlb_entry(obj, e);
335}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200336
337/**
338 * flush_iotlb_page - Clear an iommu tlb entry
339 * @obj: target iommu
340 * @da: iommu device virtual address
341 *
342 * Clear an iommu tlb entry which includes 'da' address.
343 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300344static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200345{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200346 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000347 struct cr_regs cr;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200348
349 clk_enable(obj->clk);
350
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000351 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200352 u32 start;
353 size_t bytes;
354
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200355 if (!iotlb_cr_valid(&cr))
356 continue;
357
358 start = iotlb_cr_to_virt(&cr);
359 bytes = iopgsz_to_bytes(cr.cam & 3);
360
361 if ((start <= da) && (da < start + bytes)) {
362 dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",
363 __func__, start, da, bytes);
Hari Kanigeri0fa035e2010-08-20 13:50:18 +0000364 iotlb_load_cr(obj, &cr);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200365 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
366 }
367 }
368 clk_disable(obj->clk);
369
370 if (i == obj->nr_tlb_entries)
371 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
372}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200373
374/**
375 * flush_iotlb_all - Clear all iommu tlb entries
376 * @obj: target iommu
377 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300378static void flush_iotlb_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200379{
380 struct iotlb_lock l;
381
382 clk_enable(obj->clk);
383
384 l.base = 0;
385 l.vict = 0;
386 iotlb_lock_set(obj, &l);
387
388 iommu_write_reg(obj, 1, MMU_GFLUSH);
389
390 clk_disable(obj->clk);
391}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200392
Arnd Bergmanne4efd942011-10-02 14:34:05 -0400393#if defined(CONFIG_OMAP_IOMMU_DEBUG) || defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE)
Kanigeri, Hariddfa9752010-05-24 02:01:51 +0000394
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300395ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200396{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200397 if (!obj || !buf)
398 return -EINVAL;
399
400 clk_enable(obj->clk);
401
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700402 bytes = arch_iommu->dump_ctx(obj, buf, bytes);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200403
404 clk_disable(obj->clk);
405
406 return bytes;
407}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300408EXPORT_SYMBOL_GPL(omap_iommu_dump_ctx);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200409
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300410static int
411__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200412{
413 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000414 struct iotlb_lock saved;
415 struct cr_regs tmp;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200416 struct cr_regs *p = crs;
417
418 clk_enable(obj->clk);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200419 iotlb_lock_get(obj, &saved);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200420
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000421 for_each_iotlb_cr(obj, num, i, tmp) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200422 if (!iotlb_cr_valid(&tmp))
423 continue;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200424 *p++ = tmp;
425 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000426
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200427 iotlb_lock_set(obj, &saved);
428 clk_disable(obj->clk);
429
430 return p - crs;
431}
432
433/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300434 * omap_dump_tlb_entries - dump cr arrays to given buffer
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200435 * @obj: target iommu
436 * @buf: output buffer
437 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300438size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200439{
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700440 int i, num;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200441 struct cr_regs *cr;
442 char *p = buf;
443
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700444 num = bytes / sizeof(*cr);
445 num = min(obj->nr_tlb_entries, num);
446
447 cr = kcalloc(num, sizeof(*cr), GFP_KERNEL);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200448 if (!cr)
449 return 0;
450
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700451 num = __dump_tlb_entries(obj, cr, num);
452 for (i = 0; i < num; i++)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200453 p += iotlb_dump_cr(obj, cr + i, p);
454 kfree(cr);
455
456 return p - buf;
457}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300458EXPORT_SYMBOL_GPL(omap_dump_tlb_entries);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200459
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300460int omap_foreach_iommu_device(void *data, int (*fn)(struct device *, void *))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200461{
462 return driver_for_each_device(&omap_iommu_driver.driver,
463 NULL, data, fn);
464}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300465EXPORT_SYMBOL_GPL(omap_foreach_iommu_device);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200466
467#endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */
468
469/*
470 * H/W pagetable operations
471 */
472static void flush_iopgd_range(u32 *first, u32 *last)
473{
474 /* FIXME: L2 cache should be taken care of if it exists */
475 do {
476 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
477 : : "r" (first));
478 first += L1_CACHE_BYTES / sizeof(*first);
479 } while (first <= last);
480}
481
482static void flush_iopte_range(u32 *first, u32 *last)
483{
484 /* FIXME: L2 cache should be taken care of if it exists */
485 do {
486 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
487 : : "r" (first));
488 first += L1_CACHE_BYTES / sizeof(*first);
489 } while (first <= last);
490}
491
492static void iopte_free(u32 *iopte)
493{
494 /* Note: freed iopte's must be clean ready for re-use */
495 kmem_cache_free(iopte_cachep, iopte);
496}
497
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300498static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200499{
500 u32 *iopte;
501
502 /* a table has already existed */
503 if (*iopgd)
504 goto pte_ready;
505
506 /*
507 * do the allocation outside the page table lock
508 */
509 spin_unlock(&obj->page_table_lock);
510 iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
511 spin_lock(&obj->page_table_lock);
512
513 if (!*iopgd) {
514 if (!iopte)
515 return ERR_PTR(-ENOMEM);
516
517 *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
518 flush_iopgd_range(iopgd, iopgd);
519
520 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
521 } else {
522 /* We raced, free the reduniovant table */
523 iopte_free(iopte);
524 }
525
526pte_ready:
527 iopte = iopte_offset(iopgd, da);
528
529 dev_vdbg(obj->dev,
530 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
531 __func__, da, iopgd, *iopgd, iopte, *iopte);
532
533 return iopte;
534}
535
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300536static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200537{
538 u32 *iopgd = iopgd_offset(obj, da);
539
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300540 if ((da | pa) & ~IOSECTION_MASK) {
541 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
542 __func__, da, pa, IOSECTION_SIZE);
543 return -EINVAL;
544 }
545
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200546 *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
547 flush_iopgd_range(iopgd, iopgd);
548 return 0;
549}
550
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300551static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200552{
553 u32 *iopgd = iopgd_offset(obj, da);
554 int i;
555
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300556 if ((da | pa) & ~IOSUPER_MASK) {
557 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
558 __func__, da, pa, IOSUPER_SIZE);
559 return -EINVAL;
560 }
561
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200562 for (i = 0; i < 16; i++)
563 *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
564 flush_iopgd_range(iopgd, iopgd + 15);
565 return 0;
566}
567
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300568static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200569{
570 u32 *iopgd = iopgd_offset(obj, da);
571 u32 *iopte = iopte_alloc(obj, iopgd, da);
572
573 if (IS_ERR(iopte))
574 return PTR_ERR(iopte);
575
576 *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
577 flush_iopte_range(iopte, iopte);
578
579 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
580 __func__, da, pa, iopte, *iopte);
581
582 return 0;
583}
584
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300585static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200586{
587 u32 *iopgd = iopgd_offset(obj, da);
588 u32 *iopte = iopte_alloc(obj, iopgd, da);
589 int i;
590
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300591 if ((da | pa) & ~IOLARGE_MASK) {
592 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
593 __func__, da, pa, IOLARGE_SIZE);
594 return -EINVAL;
595 }
596
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200597 if (IS_ERR(iopte))
598 return PTR_ERR(iopte);
599
600 for (i = 0; i < 16; i++)
601 *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
602 flush_iopte_range(iopte, iopte + 15);
603 return 0;
604}
605
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300606static int
607iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200608{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300609 int (*fn)(struct omap_iommu *, u32, u32, u32);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200610 u32 prot;
611 int err;
612
613 if (!obj || !e)
614 return -EINVAL;
615
616 switch (e->pgsz) {
617 case MMU_CAM_PGSZ_16M:
618 fn = iopgd_alloc_super;
619 break;
620 case MMU_CAM_PGSZ_1M:
621 fn = iopgd_alloc_section;
622 break;
623 case MMU_CAM_PGSZ_64K:
624 fn = iopte_alloc_large;
625 break;
626 case MMU_CAM_PGSZ_4K:
627 fn = iopte_alloc_page;
628 break;
629 default:
630 fn = NULL;
631 BUG();
632 break;
633 }
634
635 prot = get_iopte_attr(e);
636
637 spin_lock(&obj->page_table_lock);
638 err = fn(obj, e->da, e->pa, prot);
639 spin_unlock(&obj->page_table_lock);
640
641 return err;
642}
643
644/**
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300645 * omap_iopgtable_store_entry - Make an iommu pte entry
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200646 * @obj: target iommu
647 * @e: an iommu tlb entry info
648 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300649int omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200650{
651 int err;
652
653 flush_iotlb_page(obj, e->da);
654 err = iopgtable_store_entry_core(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200655 if (!err)
Ohad Ben-Cohen5da14a42011-08-16 15:19:10 +0300656 prefetch_iotlb_entry(obj, e);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200657 return err;
658}
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300659EXPORT_SYMBOL_GPL(omap_iopgtable_store_entry);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200660
661/**
662 * iopgtable_lookup_entry - Lookup an iommu pte entry
663 * @obj: target iommu
664 * @da: iommu device virtual address
665 * @ppgd: iommu pgd entry pointer to be returned
666 * @ppte: iommu pte entry pointer to be returned
667 **/
Ohad Ben-Cohene1f23812011-08-16 14:58:14 +0300668static void
669iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200670{
671 u32 *iopgd, *iopte = NULL;
672
673 iopgd = iopgd_offset(obj, da);
674 if (!*iopgd)
675 goto out;
676
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300677 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200678 iopte = iopte_offset(iopgd, da);
679out:
680 *ppgd = iopgd;
681 *ppte = iopte;
682}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200683
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300684static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200685{
686 size_t bytes;
687 u32 *iopgd = iopgd_offset(obj, da);
688 int nent = 1;
689
690 if (!*iopgd)
691 return 0;
692
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300693 if (iopgd_is_table(*iopgd)) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200694 int i;
695 u32 *iopte = iopte_offset(iopgd, da);
696
697 bytes = IOPTE_SIZE;
698 if (*iopte & IOPTE_LARGE) {
699 nent *= 16;
700 /* rewind to the 1st entry */
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -0800701 iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200702 }
703 bytes *= nent;
704 memset(iopte, 0, nent * sizeof(*iopte));
705 flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
706
707 /*
708 * do table walk to check if this table is necessary or not
709 */
710 iopte = iopte_offset(iopgd, 0);
711 for (i = 0; i < PTRS_PER_IOPTE; i++)
712 if (iopte[i])
713 goto out;
714
715 iopte_free(iopte);
716 nent = 1; /* for the next L1 entry */
717 } else {
718 bytes = IOPGD_SIZE;
Hiroshi DOYUdcc730d2009-10-22 14:46:32 -0700719 if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200720 nent *= 16;
721 /* rewind to the 1st entry */
Hiroshi DOYU8d33ea52010-02-15 10:03:32 -0800722 iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200723 }
724 bytes *= nent;
725 }
726 memset(iopgd, 0, nent * sizeof(*iopgd));
727 flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
728out:
729 return bytes;
730}
731
732/**
733 * iopgtable_clear_entry - Remove an iommu pte entry
734 * @obj: target iommu
735 * @da: iommu device virtual address
736 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300737static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200738{
739 size_t bytes;
740
741 spin_lock(&obj->page_table_lock);
742
743 bytes = iopgtable_clear_entry_core(obj, da);
744 flush_iotlb_page(obj, da);
745
746 spin_unlock(&obj->page_table_lock);
747
748 return bytes;
749}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200750
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300751static void iopgtable_clear_entry_all(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200752{
753 int i;
754
755 spin_lock(&obj->page_table_lock);
756
757 for (i = 0; i < PTRS_PER_IOPGD; i++) {
758 u32 da;
759 u32 *iopgd;
760
761 da = i << IOPGD_SHIFT;
762 iopgd = iopgd_offset(obj, da);
763
764 if (!*iopgd)
765 continue;
766
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300767 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200768 iopte_free(iopte_offset(iopgd, 0));
769
770 *iopgd = 0;
771 flush_iopgd_range(iopgd, iopgd);
772 }
773
774 flush_iotlb_all(obj);
775
776 spin_unlock(&obj->page_table_lock);
777}
778
779/*
780 * Device IOMMU generic operations
781 */
782static irqreturn_t iommu_fault_handler(int irq, void *data)
783{
David Cohend594f1f2011-02-16 19:35:51 +0000784 u32 da, errs;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200785 u32 *iopgd, *iopte;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300786 struct omap_iommu *obj = data;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400787 struct iommu_domain *domain = obj->domain;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200788
789 if (!obj->refcount)
790 return IRQ_NONE;
791
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200792 clk_enable(obj->clk);
David Cohend594f1f2011-02-16 19:35:51 +0000793 errs = iommu_report_fault(obj, &da);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200794 clk_disable(obj->clk);
Laurent Pinchartc56b2dd2011-05-10 16:56:46 +0200795 if (errs == 0)
796 return IRQ_HANDLED;
David Cohend594f1f2011-02-16 19:35:51 +0000797
798 /* Fault callback or TLB/PTE Dynamic loading */
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -0400799 if (!report_iommu_fault(domain, obj->dev, da, 0))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200800 return IRQ_HANDLED;
801
Hiroshi DOYU37b29812010-05-24 02:01:52 +0000802 iommu_disable(obj);
803
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200804 iopgd = iopgd_offset(obj, da);
805
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300806 if (!iopgd_is_table(*iopgd)) {
David Cohend594f1f2011-02-16 19:35:51 +0000807 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p "
808 "*pgd:px%08x\n", obj->name, errs, da, iopgd, *iopgd);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200809 return IRQ_NONE;
810 }
811
812 iopte = iopte_offset(iopgd, da);
813
David Cohend594f1f2011-02-16 19:35:51 +0000814 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x "
815 "pte:0x%p *pte:0x%08x\n", obj->name, errs, da, iopgd, *iopgd,
816 iopte, *iopte);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200817
818 return IRQ_NONE;
819}
820
821static int device_match_by_alias(struct device *dev, void *data)
822{
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300823 struct omap_iommu *obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200824 const char *name = data;
825
826 pr_debug("%s: %s %s\n", __func__, obj->name, name);
827
828 return strcmp(obj->name, name) == 0;
829}
830
831/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300832 * omap_iommu_attach() - attach iommu device to an iommu domain
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200833 * @name: name of target omap iommu device
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300834 * @iopgd: page table
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200835 **/
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200836static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200837{
838 int err = -ENOMEM;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +0200839 struct device *dev;
840 struct omap_iommu *obj;
841
842 dev = driver_find_device(&omap_iommu_driver.driver, NULL,
843 (void *)name,
844 device_match_by_alias);
845 if (!dev)
846 return NULL;
847
848 obj = to_iommu(dev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200849
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300850 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200851
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300852 /* an iommu device can only be attached once */
853 if (++obj->refcount > 1) {
854 dev_err(dev, "%s: already attached!\n", obj->name);
855 err = -EBUSY;
856 goto err_enable;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200857 }
858
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300859 obj->iopgd = iopgd;
860 err = iommu_enable(obj);
861 if (err)
862 goto err_enable;
863 flush_iotlb_all(obj);
864
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200865 if (!try_module_get(obj->owner))
866 goto err_module;
867
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300868 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200869
870 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
871 return obj;
872
873err_module:
874 if (obj->refcount == 1)
875 iommu_disable(obj);
876err_enable:
877 obj->refcount--;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300878 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200879 return ERR_PTR(err);
880}
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200881
882/**
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300883 * omap_iommu_detach - release iommu device
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200884 * @obj: target iommu
885 **/
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300886static void omap_iommu_detach(struct omap_iommu *obj)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200887{
Roel Kluinacf9d462010-01-08 10:29:05 -0800888 if (!obj || IS_ERR(obj))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200889 return;
890
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300891 spin_lock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200892
893 if (--obj->refcount == 0)
894 iommu_disable(obj);
895
896 module_put(obj->owner);
897
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300898 obj->iopgd = NULL;
899
900 spin_unlock(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200901
902 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
903}
David Cohend594f1f2011-02-16 19:35:51 +0000904
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200905/*
906 * OMAP Device MMU(IOMMU) detection
907 */
908static int __devinit omap_iommu_probe(struct platform_device *pdev)
909{
910 int err = -ENODEV;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200911 int irq;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300912 struct omap_iommu *obj;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200913 struct resource *res;
914 struct iommu_platform_data *pdata = pdev->dev.platform_data;
915
916 if (pdev->num_resources != 2)
917 return -EINVAL;
918
919 obj = kzalloc(sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
920 if (!obj)
921 return -ENOMEM;
922
923 obj->clk = clk_get(&pdev->dev, pdata->clk_name);
924 if (IS_ERR(obj->clk))
925 goto err_clk;
926
927 obj->nr_tlb_entries = pdata->nr_tlb_entries;
928 obj->name = pdata->name;
929 obj->dev = &pdev->dev;
930 obj->ctx = (void *)obj + sizeof(*obj);
Guzman Lugo, Fernandoc7f4ab22010-12-15 00:54:03 +0000931 obj->da_start = pdata->da_start;
932 obj->da_end = pdata->da_end;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200933
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +0300934 spin_lock_init(&obj->iommu_lock);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200935 mutex_init(&obj->mmap_lock);
936 spin_lock_init(&obj->page_table_lock);
937 INIT_LIST_HEAD(&obj->mmap);
938
939 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
940 if (!res) {
941 err = -ENODEV;
942 goto err_mem;
943 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200944
945 res = request_mem_region(res->start, resource_size(res),
946 dev_name(&pdev->dev));
947 if (!res) {
948 err = -EIO;
949 goto err_mem;
950 }
951
Aaro Koskinenda4a0f72011-03-14 12:28:32 +0000952 obj->regbase = ioremap(res->start, resource_size(res));
953 if (!obj->regbase) {
954 err = -ENOMEM;
955 goto err_ioremap;
956 }
957
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200958 irq = platform_get_irq(pdev, 0);
959 if (irq < 0) {
960 err = -ENODEV;
961 goto err_irq;
962 }
963 err = request_irq(irq, iommu_fault_handler, IRQF_SHARED,
964 dev_name(&pdev->dev), obj);
965 if (err < 0)
966 goto err_irq;
967 platform_set_drvdata(pdev, obj);
968
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200969 dev_info(&pdev->dev, "%s registered\n", obj->name);
970 return 0;
971
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200972err_irq:
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200973 iounmap(obj->regbase);
Aaro Koskinenda4a0f72011-03-14 12:28:32 +0000974err_ioremap:
975 release_mem_region(res->start, resource_size(res));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200976err_mem:
977 clk_put(obj->clk);
978err_clk:
979 kfree(obj);
980 return err;
981}
982
983static int __devexit omap_iommu_remove(struct platform_device *pdev)
984{
985 int irq;
986 struct resource *res;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +0300987 struct omap_iommu *obj = platform_get_drvdata(pdev);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200988
989 platform_set_drvdata(pdev, NULL);
990
991 iopgtable_clear_entry_all(obj);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200992
993 irq = platform_get_irq(pdev, 0);
994 free_irq(irq, obj);
995 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
996 release_mem_region(res->start, resource_size(res));
997 iounmap(obj->regbase);
998
999 clk_put(obj->clk);
1000 dev_info(&pdev->dev, "%s removed\n", obj->name);
1001 kfree(obj);
1002 return 0;
1003}
1004
1005static struct platform_driver omap_iommu_driver = {
1006 .probe = omap_iommu_probe,
1007 .remove = __devexit_p(omap_iommu_remove),
1008 .driver = {
1009 .name = "omap-iommu",
1010 },
1011};
1012
1013static void iopte_cachep_ctor(void *iopte)
1014{
1015 clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
1016}
1017
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001018static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001019 phys_addr_t pa, size_t bytes, int prot)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001020{
1021 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001022 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001023 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001024 struct iotlb_entry e;
1025 int omap_pgsz;
1026 u32 ret, flags;
1027
1028 /* we only support mapping a single iommu page for now */
1029 omap_pgsz = bytes_to_iopgsz(bytes);
1030 if (omap_pgsz < 0) {
1031 dev_err(dev, "invalid size to map: %d\n", bytes);
1032 return -EINVAL;
1033 }
1034
1035 dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes);
1036
1037 flags = omap_pgsz | prot;
1038
1039 iotlb_init_entry(&e, da, pa, flags);
1040
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001041 ret = omap_iopgtable_store_entry(oiommu, &e);
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001042 if (ret)
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001043 dev_err(dev, "omap_iopgtable_store_entry failed: %d\n", ret);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001044
Ohad Ben-Cohenb4550d42011-09-02 13:32:31 -04001045 return ret;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001046}
1047
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001048static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
1049 size_t size)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001050{
1051 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001052 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001053 struct device *dev = oiommu->dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001054
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001055 dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001056
Ohad Ben-Cohen50090652011-11-10 11:32:25 +02001057 return iopgtable_clear_entry(oiommu, da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001058}
1059
1060static int
1061omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1062{
1063 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001064 struct omap_iommu *oiommu;
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001065 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001066 int ret = 0;
1067
1068 spin_lock(&omap_domain->lock);
1069
1070 /* only a single device is supported per domain for now */
1071 if (omap_domain->iommu_dev) {
1072 dev_err(dev, "iommu domain is already attached\n");
1073 ret = -EBUSY;
1074 goto out;
1075 }
1076
1077 /* get a handle to and enable the omap iommu */
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001078 oiommu = omap_iommu_attach(arch_data->name, omap_domain->pgtable);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001079 if (IS_ERR(oiommu)) {
1080 ret = PTR_ERR(oiommu);
1081 dev_err(dev, "can't get omap iommu: %d\n", ret);
1082 goto out;
1083 }
1084
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001085 omap_domain->iommu_dev = arch_data->iommu_dev = oiommu;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001086 omap_domain->dev = dev;
Ohad Ben-Cohene7f10f02011-09-13 15:26:29 -04001087 oiommu->domain = domain;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001088
1089out:
1090 spin_unlock(&omap_domain->lock);
1091 return ret;
1092}
1093
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001094static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
1095 struct device *dev)
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001096{
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001097 struct omap_iommu *oiommu = dev_to_omap_iommu(dev);
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001098 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001099
1100 /* only a single device is supported per domain for now */
1101 if (omap_domain->iommu_dev != oiommu) {
1102 dev_err(dev, "invalid iommu device\n");
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001103 return;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001104 }
1105
1106 iopgtable_clear_entry_all(oiommu);
1107
1108 omap_iommu_detach(oiommu);
1109
Ohad Ben-Cohenfabdbca2011-10-11 00:18:33 +02001110 omap_domain->iommu_dev = arch_data->iommu_dev = NULL;
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001111 omap_domain->dev = NULL;
1112}
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001113
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001114static void omap_iommu_detach_dev(struct iommu_domain *domain,
1115 struct device *dev)
1116{
1117 struct omap_iommu_domain *omap_domain = domain->priv;
1118
1119 spin_lock(&omap_domain->lock);
1120 _omap_iommu_detach_dev(omap_domain, dev);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001121 spin_unlock(&omap_domain->lock);
1122}
1123
1124static int omap_iommu_domain_init(struct iommu_domain *domain)
1125{
1126 struct omap_iommu_domain *omap_domain;
1127
1128 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
1129 if (!omap_domain) {
1130 pr_err("kzalloc failed\n");
1131 goto out;
1132 }
1133
1134 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL);
1135 if (!omap_domain->pgtable) {
1136 pr_err("kzalloc failed\n");
1137 goto fail_nomem;
1138 }
1139
1140 /*
1141 * should never fail, but please keep this around to ensure
1142 * we keep the hardware happy
1143 */
1144 BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE));
1145
1146 clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
1147 spin_lock_init(&omap_domain->lock);
1148
1149 domain->priv = omap_domain;
1150
Joerg Roedel2c6edb02012-01-26 19:40:55 +01001151 domain->geometry.aperture_start = 0;
1152 domain->geometry.aperture_end = (1ULL << 32) - 1;
1153 domain->geometry.force_aperture = true;
1154
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001155 return 0;
1156
1157fail_nomem:
1158 kfree(omap_domain);
1159out:
1160 return -ENOMEM;
1161}
1162
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001163static void omap_iommu_domain_destroy(struct iommu_domain *domain)
1164{
1165 struct omap_iommu_domain *omap_domain = domain->priv;
1166
1167 domain->priv = NULL;
1168
Omar Ramirez Luna803b5272012-04-18 13:09:41 -05001169 /*
1170 * An iommu device is still attached
1171 * (currently, only one device can be attached) ?
1172 */
1173 if (omap_domain->iommu_dev)
1174 _omap_iommu_detach_dev(omap_domain, omap_domain->dev);
1175
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001176 kfree(omap_domain->pgtable);
1177 kfree(omap_domain);
1178}
1179
1180static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
1181 unsigned long da)
1182{
1183 struct omap_iommu_domain *omap_domain = domain->priv;
Ohad Ben-Cohen6c32df42011-08-17 22:57:56 +03001184 struct omap_iommu *oiommu = omap_domain->iommu_dev;
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001185 struct device *dev = oiommu->dev;
1186 u32 *pgd, *pte;
1187 phys_addr_t ret = 0;
1188
1189 iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
1190
1191 if (pte) {
1192 if (iopte_is_small(*pte))
1193 ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
1194 else if (iopte_is_large(*pte))
1195 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
1196 else
Ohad Ben-Cohen1a36ea82011-12-06 15:22:10 +02001197 dev_err(dev, "bogus pte 0x%x, da 0x%lx", *pte, da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001198 } else {
1199 if (iopgd_is_section(*pgd))
1200 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
1201 else if (iopgd_is_super(*pgd))
1202 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
1203 else
Ohad Ben-Cohen1a36ea82011-12-06 15:22:10 +02001204 dev_err(dev, "bogus pgd 0x%x, da 0x%lx", *pgd, da);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001205 }
1206
1207 return ret;
1208}
1209
1210static int omap_iommu_domain_has_cap(struct iommu_domain *domain,
1211 unsigned long cap)
1212{
1213 return 0;
1214}
1215
1216static struct iommu_ops omap_iommu_ops = {
1217 .domain_init = omap_iommu_domain_init,
1218 .domain_destroy = omap_iommu_domain_destroy,
1219 .attach_dev = omap_iommu_attach_dev,
1220 .detach_dev = omap_iommu_detach_dev,
1221 .map = omap_iommu_map,
1222 .unmap = omap_iommu_unmap,
1223 .iova_to_phys = omap_iommu_iova_to_phys,
1224 .domain_has_cap = omap_iommu_domain_has_cap,
Ohad Ben-Cohen66bc8cf2011-11-10 11:32:27 +02001225 .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001226};
1227
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001228static int __init omap_iommu_init(void)
1229{
1230 struct kmem_cache *p;
1231 const unsigned long flags = SLAB_HWCACHE_ALIGN;
1232 size_t align = 1 << 10; /* L2 pagetable alignement */
1233
1234 p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
1235 iopte_cachep_ctor);
1236 if (!p)
1237 return -ENOMEM;
1238 iopte_cachep = p;
1239
Joerg Roedela65bc642011-09-06 17:56:07 +02001240 bus_set_iommu(&platform_bus_type, &omap_iommu_ops);
Ohad Ben-Cohenf626b522011-06-02 01:46:12 +03001241
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001242 return platform_driver_register(&omap_iommu_driver);
1243}
Ohad Ben-Cohen435792d2012-02-26 12:14:14 +02001244/* must be ready before omap3isp is probed */
1245subsys_initcall(omap_iommu_init);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001246
1247static void __exit omap_iommu_exit(void)
1248{
1249 kmem_cache_destroy(iopte_cachep);
1250
1251 platform_driver_unregister(&omap_iommu_driver);
1252}
1253module_exit(omap_iommu_exit);
1254
1255MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives");
1256MODULE_ALIAS("platform:omap-iommu");
1257MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi");
1258MODULE_LICENSE("GPL v2");