blob: 341c48179ee01720ae7cae76768c91e3decddb33 [file] [log] [blame]
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02001/*
2 * omap iommu: tlb and pagetable primitives
3 *
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -08004 * Copyright (C) 2008-2010 Nokia Corporation
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +02005 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
7 * Paul Mundt and Toshihiro Kobayashi
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/err.h>
15#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090016#include <linux/slab.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020017#include <linux/interrupt.h>
18#include <linux/ioport.h>
19#include <linux/clk.h>
20#include <linux/platform_device.h>
21
22#include <asm/cacheflush.h>
23
Tony Lindgrence491cf2009-10-20 09:40:47 -070024#include <plat/iommu.h>
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020025
26#include "iopgtable.h"
27
Hiroshi DOYU37c28362010-04-27 05:37:12 +000028#define for_each_iotlb_cr(obj, n, __i, cr) \
29 for (__i = 0; \
30 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
31 __i++)
32
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +020033/* accommodate the difference between omap1 and omap2/3 */
34static const struct iommu_functions *arch_iommu;
35
36static struct platform_driver omap_iommu_driver;
37static struct kmem_cache *iopte_cachep;
38
39/**
40 * install_iommu_arch - Install archtecure specific iommu functions
41 * @ops: a pointer to architecture specific iommu functions
42 *
43 * There are several kind of iommu algorithm(tlb, pagetable) among
44 * omap series. This interface installs such an iommu algorighm.
45 **/
46int install_iommu_arch(const struct iommu_functions *ops)
47{
48 if (arch_iommu)
49 return -EBUSY;
50
51 arch_iommu = ops;
52 return 0;
53}
54EXPORT_SYMBOL_GPL(install_iommu_arch);
55
56/**
57 * uninstall_iommu_arch - Uninstall archtecure specific iommu functions
58 * @ops: a pointer to architecture specific iommu functions
59 *
60 * This interface uninstalls the iommu algorighm installed previously.
61 **/
62void uninstall_iommu_arch(const struct iommu_functions *ops)
63{
64 if (arch_iommu != ops)
65 pr_err("%s: not your arch\n", __func__);
66
67 arch_iommu = NULL;
68}
69EXPORT_SYMBOL_GPL(uninstall_iommu_arch);
70
71/**
72 * iommu_save_ctx - Save registers for pm off-mode support
73 * @obj: target iommu
74 **/
75void iommu_save_ctx(struct iommu *obj)
76{
77 arch_iommu->save_ctx(obj);
78}
79EXPORT_SYMBOL_GPL(iommu_save_ctx);
80
81/**
82 * iommu_restore_ctx - Restore registers for pm off-mode support
83 * @obj: target iommu
84 **/
85void iommu_restore_ctx(struct iommu *obj)
86{
87 arch_iommu->restore_ctx(obj);
88}
89EXPORT_SYMBOL_GPL(iommu_restore_ctx);
90
91/**
92 * iommu_arch_version - Return running iommu arch version
93 **/
94u32 iommu_arch_version(void)
95{
96 return arch_iommu->version;
97}
98EXPORT_SYMBOL_GPL(iommu_arch_version);
99
100static int iommu_enable(struct iommu *obj)
101{
102 int err;
103
104 if (!obj)
105 return -EINVAL;
106
107 clk_enable(obj->clk);
108
109 err = arch_iommu->enable(obj);
110
111 clk_disable(obj->clk);
112 return err;
113}
114
115static void iommu_disable(struct iommu *obj)
116{
117 if (!obj)
118 return;
119
120 clk_enable(obj->clk);
121
122 arch_iommu->disable(obj);
123
124 clk_disable(obj->clk);
125}
126
127/*
128 * TLB operations
129 */
130void iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e)
131{
132 BUG_ON(!cr || !e);
133
134 arch_iommu->cr_to_e(cr, e);
135}
136EXPORT_SYMBOL_GPL(iotlb_cr_to_e);
137
138static inline int iotlb_cr_valid(struct cr_regs *cr)
139{
140 if (!cr)
141 return -EINVAL;
142
143 return arch_iommu->cr_valid(cr);
144}
145
146static inline struct cr_regs *iotlb_alloc_cr(struct iommu *obj,
147 struct iotlb_entry *e)
148{
149 if (!e)
150 return NULL;
151
152 return arch_iommu->alloc_cr(obj, e);
153}
154
155u32 iotlb_cr_to_virt(struct cr_regs *cr)
156{
157 return arch_iommu->cr_to_virt(cr);
158}
159EXPORT_SYMBOL_GPL(iotlb_cr_to_virt);
160
161static u32 get_iopte_attr(struct iotlb_entry *e)
162{
163 return arch_iommu->get_pte_attr(e);
164}
165
166static u32 iommu_report_fault(struct iommu *obj, u32 *da)
167{
168 return arch_iommu->fault_isr(obj, da);
169}
170
171static void iotlb_lock_get(struct iommu *obj, struct iotlb_lock *l)
172{
173 u32 val;
174
175 val = iommu_read_reg(obj, MMU_LOCK);
176
177 l->base = MMU_LOCK_BASE(val);
178 l->vict = MMU_LOCK_VICT(val);
179
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200180}
181
182static void iotlb_lock_set(struct iommu *obj, struct iotlb_lock *l)
183{
184 u32 val;
185
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200186 val = (l->base << MMU_LOCK_BASE_SHIFT);
187 val |= (l->vict << MMU_LOCK_VICT_SHIFT);
188
189 iommu_write_reg(obj, val, MMU_LOCK);
190}
191
192static void iotlb_read_cr(struct iommu *obj, struct cr_regs *cr)
193{
194 arch_iommu->tlb_read_cr(obj, cr);
195}
196
197static void iotlb_load_cr(struct iommu *obj, struct cr_regs *cr)
198{
199 arch_iommu->tlb_load_cr(obj, cr);
200
201 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
202 iommu_write_reg(obj, 1, MMU_LD_TLB);
203}
204
205/**
206 * iotlb_dump_cr - Dump an iommu tlb entry into buf
207 * @obj: target iommu
208 * @cr: contents of cam and ram register
209 * @buf: output buffer
210 **/
211static inline ssize_t iotlb_dump_cr(struct iommu *obj, struct cr_regs *cr,
212 char *buf)
213{
214 BUG_ON(!cr || !buf);
215
216 return arch_iommu->dump_cr(obj, cr, buf);
217}
218
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000219/* only used in iotlb iteration for-loop */
220static struct cr_regs __iotlb_read_cr(struct iommu *obj, int n)
221{
222 struct cr_regs cr;
223 struct iotlb_lock l;
224
225 iotlb_lock_get(obj, &l);
226 l.vict = n;
227 iotlb_lock_set(obj, &l);
228 iotlb_read_cr(obj, &cr);
229
230 return cr;
231}
232
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200233/**
234 * load_iotlb_entry - Set an iommu tlb entry
235 * @obj: target iommu
236 * @e: an iommu tlb entry info
237 **/
238int load_iotlb_entry(struct iommu *obj, struct iotlb_entry *e)
239{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200240 int err = 0;
241 struct iotlb_lock l;
242 struct cr_regs *cr;
243
244 if (!obj || !obj->nr_tlb_entries || !e)
245 return -EINVAL;
246
247 clk_enable(obj->clk);
248
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000249 iotlb_lock_get(obj, &l);
250 if (l.base == obj->nr_tlb_entries) {
251 dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200252 err = -EBUSY;
253 goto out;
254 }
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000255 if (!e->prsvd) {
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000256 int i;
257 struct cr_regs tmp;
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000258
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000259 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000260 if (!iotlb_cr_valid(&tmp))
261 break;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000262
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000263 if (i == obj->nr_tlb_entries) {
264 dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
265 err = -EBUSY;
266 goto out;
267 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000268
269 iotlb_lock_get(obj, &l);
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000270 } else {
271 l.vict = l.base;
272 iotlb_lock_set(obj, &l);
273 }
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200274
275 cr = iotlb_alloc_cr(obj, e);
276 if (IS_ERR(cr)) {
277 clk_disable(obj->clk);
278 return PTR_ERR(cr);
279 }
280
281 iotlb_load_cr(obj, cr);
282 kfree(cr);
283
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000284 if (e->prsvd)
285 l.base++;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200286 /* increment victim for next tlb load */
287 if (++l.vict == obj->nr_tlb_entries)
Kanigeri, Haribe6d8022010-04-22 23:26:11 +0000288 l.vict = l.base;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200289 iotlb_lock_set(obj, &l);
290out:
291 clk_disable(obj->clk);
292 return err;
293}
294EXPORT_SYMBOL_GPL(load_iotlb_entry);
295
296/**
297 * flush_iotlb_page - Clear an iommu tlb entry
298 * @obj: target iommu
299 * @da: iommu device virtual address
300 *
301 * Clear an iommu tlb entry which includes 'da' address.
302 **/
303void flush_iotlb_page(struct iommu *obj, u32 da)
304{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200305 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000306 struct cr_regs cr;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200307
308 clk_enable(obj->clk);
309
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000310 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200311 u32 start;
312 size_t bytes;
313
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200314 if (!iotlb_cr_valid(&cr))
315 continue;
316
317 start = iotlb_cr_to_virt(&cr);
318 bytes = iopgsz_to_bytes(cr.cam & 3);
319
320 if ((start <= da) && (da < start + bytes)) {
321 dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",
322 __func__, start, da, bytes);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200323 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
324 }
325 }
326 clk_disable(obj->clk);
327
328 if (i == obj->nr_tlb_entries)
329 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
330}
331EXPORT_SYMBOL_GPL(flush_iotlb_page);
332
333/**
334 * flush_iotlb_range - Clear an iommu tlb entries
335 * @obj: target iommu
336 * @start: iommu device virtual address(start)
337 * @end: iommu device virtual address(end)
338 *
339 * Clear an iommu tlb entry which includes 'da' address.
340 **/
341void flush_iotlb_range(struct iommu *obj, u32 start, u32 end)
342{
343 u32 da = start;
344
345 while (da < end) {
346 flush_iotlb_page(obj, da);
347 /* FIXME: Optimize for multiple page size */
348 da += IOPTE_SIZE;
349 }
350}
351EXPORT_SYMBOL_GPL(flush_iotlb_range);
352
353/**
354 * flush_iotlb_all - Clear all iommu tlb entries
355 * @obj: target iommu
356 **/
357void flush_iotlb_all(struct iommu *obj)
358{
359 struct iotlb_lock l;
360
361 clk_enable(obj->clk);
362
363 l.base = 0;
364 l.vict = 0;
365 iotlb_lock_set(obj, &l);
366
367 iommu_write_reg(obj, 1, MMU_GFLUSH);
368
369 clk_disable(obj->clk);
370}
371EXPORT_SYMBOL_GPL(flush_iotlb_all);
372
373#if defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE)
374
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700375ssize_t iommu_dump_ctx(struct iommu *obj, char *buf, ssize_t bytes)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200376{
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200377 if (!obj || !buf)
378 return -EINVAL;
379
380 clk_enable(obj->clk);
381
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700382 bytes = arch_iommu->dump_ctx(obj, buf, bytes);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200383
384 clk_disable(obj->clk);
385
386 return bytes;
387}
388EXPORT_SYMBOL_GPL(iommu_dump_ctx);
389
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700390static int __dump_tlb_entries(struct iommu *obj, struct cr_regs *crs, int num)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200391{
392 int i;
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000393 struct iotlb_lock saved;
394 struct cr_regs tmp;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200395 struct cr_regs *p = crs;
396
397 clk_enable(obj->clk);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200398 iotlb_lock_get(obj, &saved);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200399
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000400 for_each_iotlb_cr(obj, num, i, tmp) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200401 if (!iotlb_cr_valid(&tmp))
402 continue;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200403 *p++ = tmp;
404 }
Hiroshi DOYU37c28362010-04-27 05:37:12 +0000405
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200406 iotlb_lock_set(obj, &saved);
407 clk_disable(obj->clk);
408
409 return p - crs;
410}
411
412/**
413 * dump_tlb_entries - dump cr arrays to given buffer
414 * @obj: target iommu
415 * @buf: output buffer
416 **/
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700417size_t dump_tlb_entries(struct iommu *obj, char *buf, ssize_t bytes)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200418{
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700419 int i, num;
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200420 struct cr_regs *cr;
421 char *p = buf;
422
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700423 num = bytes / sizeof(*cr);
424 num = min(obj->nr_tlb_entries, num);
425
426 cr = kcalloc(num, sizeof(*cr), GFP_KERNEL);
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200427 if (!cr)
428 return 0;
429
Hiroshi DOYU14e0e672009-08-28 10:54:41 -0700430 num = __dump_tlb_entries(obj, cr, num);
431 for (i = 0; i < num; i++)
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200432 p += iotlb_dump_cr(obj, cr + i, p);
433 kfree(cr);
434
435 return p - buf;
436}
437EXPORT_SYMBOL_GPL(dump_tlb_entries);
438
439int foreach_iommu_device(void *data, int (*fn)(struct device *, void *))
440{
441 return driver_for_each_device(&omap_iommu_driver.driver,
442 NULL, data, fn);
443}
444EXPORT_SYMBOL_GPL(foreach_iommu_device);
445
446#endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */
447
448/*
449 * H/W pagetable operations
450 */
451static void flush_iopgd_range(u32 *first, u32 *last)
452{
453 /* FIXME: L2 cache should be taken care of if it exists */
454 do {
455 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
456 : : "r" (first));
457 first += L1_CACHE_BYTES / sizeof(*first);
458 } while (first <= last);
459}
460
461static void flush_iopte_range(u32 *first, u32 *last)
462{
463 /* FIXME: L2 cache should be taken care of if it exists */
464 do {
465 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
466 : : "r" (first));
467 first += L1_CACHE_BYTES / sizeof(*first);
468 } while (first <= last);
469}
470
471static void iopte_free(u32 *iopte)
472{
473 /* Note: freed iopte's must be clean ready for re-use */
474 kmem_cache_free(iopte_cachep, iopte);
475}
476
477static u32 *iopte_alloc(struct iommu *obj, u32 *iopgd, u32 da)
478{
479 u32 *iopte;
480
481 /* a table has already existed */
482 if (*iopgd)
483 goto pte_ready;
484
485 /*
486 * do the allocation outside the page table lock
487 */
488 spin_unlock(&obj->page_table_lock);
489 iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
490 spin_lock(&obj->page_table_lock);
491
492 if (!*iopgd) {
493 if (!iopte)
494 return ERR_PTR(-ENOMEM);
495
496 *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
497 flush_iopgd_range(iopgd, iopgd);
498
499 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
500 } else {
501 /* We raced, free the reduniovant table */
502 iopte_free(iopte);
503 }
504
505pte_ready:
506 iopte = iopte_offset(iopgd, da);
507
508 dev_vdbg(obj->dev,
509 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
510 __func__, da, iopgd, *iopgd, iopte, *iopte);
511
512 return iopte;
513}
514
515static int iopgd_alloc_section(struct iommu *obj, u32 da, u32 pa, u32 prot)
516{
517 u32 *iopgd = iopgd_offset(obj, da);
518
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300519 if ((da | pa) & ~IOSECTION_MASK) {
520 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
521 __func__, da, pa, IOSECTION_SIZE);
522 return -EINVAL;
523 }
524
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200525 *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
526 flush_iopgd_range(iopgd, iopgd);
527 return 0;
528}
529
530static int iopgd_alloc_super(struct iommu *obj, u32 da, u32 pa, u32 prot)
531{
532 u32 *iopgd = iopgd_offset(obj, da);
533 int i;
534
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300535 if ((da | pa) & ~IOSUPER_MASK) {
536 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
537 __func__, da, pa, IOSUPER_SIZE);
538 return -EINVAL;
539 }
540
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200541 for (i = 0; i < 16; i++)
542 *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
543 flush_iopgd_range(iopgd, iopgd + 15);
544 return 0;
545}
546
547static int iopte_alloc_page(struct iommu *obj, u32 da, u32 pa, u32 prot)
548{
549 u32 *iopgd = iopgd_offset(obj, da);
550 u32 *iopte = iopte_alloc(obj, iopgd, da);
551
552 if (IS_ERR(iopte))
553 return PTR_ERR(iopte);
554
555 *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
556 flush_iopte_range(iopte, iopte);
557
558 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
559 __func__, da, pa, iopte, *iopte);
560
561 return 0;
562}
563
564static int iopte_alloc_large(struct iommu *obj, u32 da, u32 pa, u32 prot)
565{
566 u32 *iopgd = iopgd_offset(obj, da);
567 u32 *iopte = iopte_alloc(obj, iopgd, da);
568 int i;
569
Hiroshi DOYU4abb7612010-05-06 18:24:04 +0300570 if ((da | pa) & ~IOLARGE_MASK) {
571 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
572 __func__, da, pa, IOLARGE_SIZE);
573 return -EINVAL;
574 }
575
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200576 if (IS_ERR(iopte))
577 return PTR_ERR(iopte);
578
579 for (i = 0; i < 16; i++)
580 *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
581 flush_iopte_range(iopte, iopte + 15);
582 return 0;
583}
584
585static int iopgtable_store_entry_core(struct iommu *obj, struct iotlb_entry *e)
586{
587 int (*fn)(struct iommu *, u32, u32, u32);
588 u32 prot;
589 int err;
590
591 if (!obj || !e)
592 return -EINVAL;
593
594 switch (e->pgsz) {
595 case MMU_CAM_PGSZ_16M:
596 fn = iopgd_alloc_super;
597 break;
598 case MMU_CAM_PGSZ_1M:
599 fn = iopgd_alloc_section;
600 break;
601 case MMU_CAM_PGSZ_64K:
602 fn = iopte_alloc_large;
603 break;
604 case MMU_CAM_PGSZ_4K:
605 fn = iopte_alloc_page;
606 break;
607 default:
608 fn = NULL;
609 BUG();
610 break;
611 }
612
613 prot = get_iopte_attr(e);
614
615 spin_lock(&obj->page_table_lock);
616 err = fn(obj, e->da, e->pa, prot);
617 spin_unlock(&obj->page_table_lock);
618
619 return err;
620}
621
622/**
623 * iopgtable_store_entry - Make an iommu pte entry
624 * @obj: target iommu
625 * @e: an iommu tlb entry info
626 **/
627int iopgtable_store_entry(struct iommu *obj, struct iotlb_entry *e)
628{
629 int err;
630
631 flush_iotlb_page(obj, e->da);
632 err = iopgtable_store_entry_core(obj, e);
633#ifdef PREFETCH_IOTLB
634 if (!err)
635 load_iotlb_entry(obj, e);
636#endif
637 return err;
638}
639EXPORT_SYMBOL_GPL(iopgtable_store_entry);
640
641/**
642 * iopgtable_lookup_entry - Lookup an iommu pte entry
643 * @obj: target iommu
644 * @da: iommu device virtual address
645 * @ppgd: iommu pgd entry pointer to be returned
646 * @ppte: iommu pte entry pointer to be returned
647 **/
648void iopgtable_lookup_entry(struct iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
649{
650 u32 *iopgd, *iopte = NULL;
651
652 iopgd = iopgd_offset(obj, da);
653 if (!*iopgd)
654 goto out;
655
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300656 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200657 iopte = iopte_offset(iopgd, da);
658out:
659 *ppgd = iopgd;
660 *ppte = iopte;
661}
662EXPORT_SYMBOL_GPL(iopgtable_lookup_entry);
663
664static size_t iopgtable_clear_entry_core(struct iommu *obj, u32 da)
665{
666 size_t bytes;
667 u32 *iopgd = iopgd_offset(obj, da);
668 int nent = 1;
669
670 if (!*iopgd)
671 return 0;
672
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300673 if (iopgd_is_table(*iopgd)) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200674 int i;
675 u32 *iopte = iopte_offset(iopgd, da);
676
677 bytes = IOPTE_SIZE;
678 if (*iopte & IOPTE_LARGE) {
679 nent *= 16;
680 /* rewind to the 1st entry */
Hiroshi DOYUc127c7d2010-02-15 10:03:32 -0800681 iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200682 }
683 bytes *= nent;
684 memset(iopte, 0, nent * sizeof(*iopte));
685 flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
686
687 /*
688 * do table walk to check if this table is necessary or not
689 */
690 iopte = iopte_offset(iopgd, 0);
691 for (i = 0; i < PTRS_PER_IOPTE; i++)
692 if (iopte[i])
693 goto out;
694
695 iopte_free(iopte);
696 nent = 1; /* for the next L1 entry */
697 } else {
698 bytes = IOPGD_SIZE;
Hiroshi DOYUdcc730d2009-10-22 14:46:32 -0700699 if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200700 nent *= 16;
701 /* rewind to the 1st entry */
Hiroshi DOYU8d33ea52010-02-15 10:03:32 -0800702 iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200703 }
704 bytes *= nent;
705 }
706 memset(iopgd, 0, nent * sizeof(*iopgd));
707 flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
708out:
709 return bytes;
710}
711
712/**
713 * iopgtable_clear_entry - Remove an iommu pte entry
714 * @obj: target iommu
715 * @da: iommu device virtual address
716 **/
717size_t iopgtable_clear_entry(struct iommu *obj, u32 da)
718{
719 size_t bytes;
720
721 spin_lock(&obj->page_table_lock);
722
723 bytes = iopgtable_clear_entry_core(obj, da);
724 flush_iotlb_page(obj, da);
725
726 spin_unlock(&obj->page_table_lock);
727
728 return bytes;
729}
730EXPORT_SYMBOL_GPL(iopgtable_clear_entry);
731
732static void iopgtable_clear_entry_all(struct iommu *obj)
733{
734 int i;
735
736 spin_lock(&obj->page_table_lock);
737
738 for (i = 0; i < PTRS_PER_IOPGD; i++) {
739 u32 da;
740 u32 *iopgd;
741
742 da = i << IOPGD_SHIFT;
743 iopgd = iopgd_offset(obj, da);
744
745 if (!*iopgd)
746 continue;
747
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300748 if (iopgd_is_table(*iopgd))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200749 iopte_free(iopte_offset(iopgd, 0));
750
751 *iopgd = 0;
752 flush_iopgd_range(iopgd, iopgd);
753 }
754
755 flush_iotlb_all(obj);
756
757 spin_unlock(&obj->page_table_lock);
758}
759
760/*
761 * Device IOMMU generic operations
762 */
763static irqreturn_t iommu_fault_handler(int irq, void *data)
764{
765 u32 stat, da;
766 u32 *iopgd, *iopte;
767 int err = -EIO;
768 struct iommu *obj = data;
769
770 if (!obj->refcount)
771 return IRQ_NONE;
772
773 /* Dynamic loading TLB or PTE */
774 if (obj->isr)
775 err = obj->isr(obj);
776
777 if (!err)
778 return IRQ_HANDLED;
779
780 clk_enable(obj->clk);
781 stat = iommu_report_fault(obj, &da);
782 clk_disable(obj->clk);
783 if (!stat)
784 return IRQ_HANDLED;
785
786 iopgd = iopgd_offset(obj, da);
787
Hiroshi DOYUa1a54452010-05-13 09:45:35 +0300788 if (!iopgd_is_table(*iopgd)) {
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200789 dev_err(obj->dev, "%s: da:%08x pgd:%p *pgd:%08x\n", __func__,
790 da, iopgd, *iopgd);
791 return IRQ_NONE;
792 }
793
794 iopte = iopte_offset(iopgd, da);
795
796 dev_err(obj->dev, "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
797 __func__, da, iopgd, *iopgd, iopte, *iopte);
798
799 return IRQ_NONE;
800}
801
802static int device_match_by_alias(struct device *dev, void *data)
803{
804 struct iommu *obj = to_iommu(dev);
805 const char *name = data;
806
807 pr_debug("%s: %s %s\n", __func__, obj->name, name);
808
809 return strcmp(obj->name, name) == 0;
810}
811
812/**
813 * iommu_get - Get iommu handler
814 * @name: target iommu name
815 **/
816struct iommu *iommu_get(const char *name)
817{
818 int err = -ENOMEM;
819 struct device *dev;
820 struct iommu *obj;
821
822 dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name,
823 device_match_by_alias);
824 if (!dev)
825 return ERR_PTR(-ENODEV);
826
827 obj = to_iommu(dev);
828
829 mutex_lock(&obj->iommu_lock);
830
831 if (obj->refcount++ == 0) {
832 err = iommu_enable(obj);
833 if (err)
834 goto err_enable;
835 flush_iotlb_all(obj);
836 }
837
838 if (!try_module_get(obj->owner))
839 goto err_module;
840
841 mutex_unlock(&obj->iommu_lock);
842
843 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
844 return obj;
845
846err_module:
847 if (obj->refcount == 1)
848 iommu_disable(obj);
849err_enable:
850 obj->refcount--;
851 mutex_unlock(&obj->iommu_lock);
852 return ERR_PTR(err);
853}
854EXPORT_SYMBOL_GPL(iommu_get);
855
856/**
857 * iommu_put - Put back iommu handler
858 * @obj: target iommu
859 **/
860void iommu_put(struct iommu *obj)
861{
Roel Kluinacf9d462010-01-08 10:29:05 -0800862 if (!obj || IS_ERR(obj))
Hiroshi DOYUa9dcad52009-01-26 15:13:40 +0200863 return;
864
865 mutex_lock(&obj->iommu_lock);
866
867 if (--obj->refcount == 0)
868 iommu_disable(obj);
869
870 module_put(obj->owner);
871
872 mutex_unlock(&obj->iommu_lock);
873
874 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
875}
876EXPORT_SYMBOL_GPL(iommu_put);
877
878/*
879 * OMAP Device MMU(IOMMU) detection
880 */
881static int __devinit omap_iommu_probe(struct platform_device *pdev)
882{
883 int err = -ENODEV;
884 void *p;
885 int irq;
886 struct iommu *obj;
887 struct resource *res;
888 struct iommu_platform_data *pdata = pdev->dev.platform_data;
889
890 if (pdev->num_resources != 2)
891 return -EINVAL;
892
893 obj = kzalloc(sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
894 if (!obj)
895 return -ENOMEM;
896
897 obj->clk = clk_get(&pdev->dev, pdata->clk_name);
898 if (IS_ERR(obj->clk))
899 goto err_clk;
900
901 obj->nr_tlb_entries = pdata->nr_tlb_entries;
902 obj->name = pdata->name;
903 obj->dev = &pdev->dev;
904 obj->ctx = (void *)obj + sizeof(*obj);
905
906 mutex_init(&obj->iommu_lock);
907 mutex_init(&obj->mmap_lock);
908 spin_lock_init(&obj->page_table_lock);
909 INIT_LIST_HEAD(&obj->mmap);
910
911 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
912 if (!res) {
913 err = -ENODEV;
914 goto err_mem;
915 }
916 obj->regbase = ioremap(res->start, resource_size(res));
917 if (!obj->regbase) {
918 err = -ENOMEM;
919 goto err_mem;
920 }
921
922 res = request_mem_region(res->start, resource_size(res),
923 dev_name(&pdev->dev));
924 if (!res) {
925 err = -EIO;
926 goto err_mem;
927 }
928
929 irq = platform_get_irq(pdev, 0);
930 if (irq < 0) {
931 err = -ENODEV;
932 goto err_irq;
933 }
934 err = request_irq(irq, iommu_fault_handler, IRQF_SHARED,
935 dev_name(&pdev->dev), obj);
936 if (err < 0)
937 goto err_irq;
938 platform_set_drvdata(pdev, obj);
939
940 p = (void *)__get_free_pages(GFP_KERNEL, get_order(IOPGD_TABLE_SIZE));
941 if (!p) {
942 err = -ENOMEM;
943 goto err_pgd;
944 }
945 memset(p, 0, IOPGD_TABLE_SIZE);
946 clean_dcache_area(p, IOPGD_TABLE_SIZE);
947 obj->iopgd = p;
948
949 BUG_ON(!IS_ALIGNED((unsigned long)obj->iopgd, IOPGD_TABLE_SIZE));
950
951 dev_info(&pdev->dev, "%s registered\n", obj->name);
952 return 0;
953
954err_pgd:
955 free_irq(irq, obj);
956err_irq:
957 release_mem_region(res->start, resource_size(res));
958 iounmap(obj->regbase);
959err_mem:
960 clk_put(obj->clk);
961err_clk:
962 kfree(obj);
963 return err;
964}
965
966static int __devexit omap_iommu_remove(struct platform_device *pdev)
967{
968 int irq;
969 struct resource *res;
970 struct iommu *obj = platform_get_drvdata(pdev);
971
972 platform_set_drvdata(pdev, NULL);
973
974 iopgtable_clear_entry_all(obj);
975 free_pages((unsigned long)obj->iopgd, get_order(IOPGD_TABLE_SIZE));
976
977 irq = platform_get_irq(pdev, 0);
978 free_irq(irq, obj);
979 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
980 release_mem_region(res->start, resource_size(res));
981 iounmap(obj->regbase);
982
983 clk_put(obj->clk);
984 dev_info(&pdev->dev, "%s removed\n", obj->name);
985 kfree(obj);
986 return 0;
987}
988
989static struct platform_driver omap_iommu_driver = {
990 .probe = omap_iommu_probe,
991 .remove = __devexit_p(omap_iommu_remove),
992 .driver = {
993 .name = "omap-iommu",
994 },
995};
996
997static void iopte_cachep_ctor(void *iopte)
998{
999 clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
1000}
1001
1002static int __init omap_iommu_init(void)
1003{
1004 struct kmem_cache *p;
1005 const unsigned long flags = SLAB_HWCACHE_ALIGN;
1006 size_t align = 1 << 10; /* L2 pagetable alignement */
1007
1008 p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
1009 iopte_cachep_ctor);
1010 if (!p)
1011 return -ENOMEM;
1012 iopte_cachep = p;
1013
1014 return platform_driver_register(&omap_iommu_driver);
1015}
1016module_init(omap_iommu_init);
1017
1018static void __exit omap_iommu_exit(void)
1019{
1020 kmem_cache_destroy(iopte_cachep);
1021
1022 platform_driver_unregister(&omap_iommu_driver);
1023}
1024module_exit(omap_iommu_exit);
1025
1026MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives");
1027MODULE_ALIAS("platform:omap-iommu");
1028MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi");
1029MODULE_LICENSE("GPL v2");