blob: 90f70d0e114192a43d7cd7bac3119b0fc66b185c [file] [log] [blame]
Joerg Roedele3c495c2011-11-09 12:31:15 +01001/*
2 * Copyright (C) 2010-2012 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
Joerg Roedel8736b2c2011-11-24 16:21:52 +010019#include <linux/mmu_notifier.h>
Joerg Roedeled96f222011-11-23 17:30:39 +010020#include <linux/amd-iommu.h>
21#include <linux/mm_types.h>
Joerg Roedel8736b2c2011-11-24 16:21:52 +010022#include <linux/profile.h>
Joerg Roedele3c495c2011-11-09 12:31:15 +010023#include <linux/module.h>
Joerg Roedel2d5503b2011-11-24 10:41:57 +010024#include <linux/sched.h>
Joerg Roedeled96f222011-11-23 17:30:39 +010025#include <linux/iommu.h>
Joerg Roedel028eeac2011-11-24 12:48:13 +010026#include <linux/wait.h>
Joerg Roedeled96f222011-11-23 17:30:39 +010027#include <linux/pci.h>
28#include <linux/gfp.h>
29
Joerg Roedel028eeac2011-11-24 12:48:13 +010030#include "amd_iommu_types.h"
Joerg Roedeled96f222011-11-23 17:30:39 +010031#include "amd_iommu_proto.h"
Joerg Roedele3c495c2011-11-09 12:31:15 +010032
33MODULE_LICENSE("GPL v2");
34MODULE_AUTHOR("Joerg Roedel <joerg.roedel@amd.com>");
35
Joerg Roedeled96f222011-11-23 17:30:39 +010036#define MAX_DEVICES 0x10000
37#define PRI_QUEUE_SIZE 512
38
39struct pri_queue {
40 atomic_t inflight;
41 bool finish;
Joerg Roedel028eeac2011-11-24 12:48:13 +010042 int status;
Joerg Roedeled96f222011-11-23 17:30:39 +010043};
44
45struct pasid_state {
46 struct list_head list; /* For global state-list */
47 atomic_t count; /* Reference count */
Joerg Roedeld73a6d72014-06-20 16:14:22 +020048 unsigned mmu_notifier_count; /* Counting nested mmu_notifier
Joerg Roedele79df312014-05-20 23:18:26 +020049 calls */
Joerg Roedeled96f222011-11-23 17:30:39 +010050 struct mm_struct *mm; /* mm_struct for the faults */
Joerg Roedelff6d0cc2014-07-08 12:49:50 +020051 struct mmu_notifier mn; /* mmu_notifier handle */
Joerg Roedeled96f222011-11-23 17:30:39 +010052 struct pri_queue pri[PRI_QUEUE_SIZE]; /* PRI tag states */
53 struct device_state *device_state; /* Link to our device_state */
54 int pasid; /* PASID index */
Joerg Roedeld9e16112014-07-09 15:43:11 +020055 bool invalid; /* Used during setup and
56 teardown of the pasid */
Joerg Roedeld73a6d72014-06-20 16:14:22 +020057 spinlock_t lock; /* Protect pri_queues and
58 mmu_notifer_count */
Joerg Roedel028eeac2011-11-24 12:48:13 +010059 wait_queue_head_t wq; /* To wait for count == 0 */
Joerg Roedeled96f222011-11-23 17:30:39 +010060};
61
62struct device_state {
Joerg Roedel741669c2014-05-20 23:18:23 +020063 struct list_head list;
64 u16 devid;
Joerg Roedeled96f222011-11-23 17:30:39 +010065 atomic_t count;
66 struct pci_dev *pdev;
67 struct pasid_state **states;
68 struct iommu_domain *domain;
69 int pasid_levels;
70 int max_pasids;
Joerg Roedel175d6142011-11-28 14:36:36 +010071 amd_iommu_invalid_ppr_cb inv_ppr_cb;
Joerg Roedelbc216622011-12-07 12:24:42 +010072 amd_iommu_invalidate_ctx inv_ctx_cb;
Joerg Roedeled96f222011-11-23 17:30:39 +010073 spinlock_t lock;
Joerg Roedel028eeac2011-11-24 12:48:13 +010074 wait_queue_head_t wq;
75};
76
77struct fault {
78 struct work_struct work;
79 struct device_state *dev_state;
80 struct pasid_state *state;
81 struct mm_struct *mm;
82 u64 address;
83 u16 devid;
84 u16 pasid;
85 u16 tag;
86 u16 finish;
87 u16 flags;
Joerg Roedeled96f222011-11-23 17:30:39 +010088};
89
Joerg Roedel741669c2014-05-20 23:18:23 +020090static LIST_HEAD(state_list);
Joerg Roedeled96f222011-11-23 17:30:39 +010091static spinlock_t state_lock;
92
Joerg Roedel028eeac2011-11-24 12:48:13 +010093static struct workqueue_struct *iommu_wq;
94
Joerg Roedel2d5503b2011-11-24 10:41:57 +010095static void free_pasid_states(struct device_state *dev_state);
Joerg Roedeled96f222011-11-23 17:30:39 +010096
97static u16 device_id(struct pci_dev *pdev)
98{
99 u16 devid;
100
101 devid = pdev->bus->number;
102 devid = (devid << 8) | pdev->devfn;
103
104 return devid;
105}
106
Joerg Roedelb87d2d72014-05-20 23:18:22 +0200107static struct device_state *__get_device_state(u16 devid)
108{
Joerg Roedel741669c2014-05-20 23:18:23 +0200109 struct device_state *dev_state;
110
111 list_for_each_entry(dev_state, &state_list, list) {
112 if (dev_state->devid == devid)
113 return dev_state;
114 }
115
116 return NULL;
Joerg Roedelb87d2d72014-05-20 23:18:22 +0200117}
118
Joerg Roedeled96f222011-11-23 17:30:39 +0100119static struct device_state *get_device_state(u16 devid)
120{
121 struct device_state *dev_state;
122 unsigned long flags;
123
124 spin_lock_irqsave(&state_lock, flags);
Joerg Roedelb87d2d72014-05-20 23:18:22 +0200125 dev_state = __get_device_state(devid);
Joerg Roedeled96f222011-11-23 17:30:39 +0100126 if (dev_state != NULL)
127 atomic_inc(&dev_state->count);
128 spin_unlock_irqrestore(&state_lock, flags);
129
130 return dev_state;
131}
132
133static void free_device_state(struct device_state *dev_state)
134{
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100135 /*
136 * First detach device from domain - No more PRI requests will arrive
137 * from that device after it is unbound from the IOMMUv2 domain.
138 */
Joerg Roedeled96f222011-11-23 17:30:39 +0100139 iommu_detach_device(dev_state->domain, &dev_state->pdev->dev);
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100140
141 /* Everything is down now, free the IOMMUv2 domain */
Joerg Roedeled96f222011-11-23 17:30:39 +0100142 iommu_domain_free(dev_state->domain);
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100143
144 /* Finally get rid of the device-state */
Joerg Roedeled96f222011-11-23 17:30:39 +0100145 kfree(dev_state);
146}
147
148static void put_device_state(struct device_state *dev_state)
149{
150 if (atomic_dec_and_test(&dev_state->count))
Joerg Roedel028eeac2011-11-24 12:48:13 +0100151 wake_up(&dev_state->wq);
Joerg Roedeled96f222011-11-23 17:30:39 +0100152}
153
Joerg Roedel028eeac2011-11-24 12:48:13 +0100154static void put_device_state_wait(struct device_state *dev_state)
155{
156 DEFINE_WAIT(wait);
157
158 prepare_to_wait(&dev_state->wq, &wait, TASK_UNINTERRUPTIBLE);
159 if (!atomic_dec_and_test(&dev_state->count))
160 schedule();
161 finish_wait(&dev_state->wq, &wait);
162
163 free_device_state(dev_state);
164}
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100165
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100166/* Must be called under dev_state->lock */
167static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
168 int pasid, bool alloc)
169{
170 struct pasid_state **root, **ptr;
171 int level, index;
172
173 level = dev_state->pasid_levels;
174 root = dev_state->states;
175
176 while (true) {
177
178 index = (pasid >> (9 * level)) & 0x1ff;
179 ptr = &root[index];
180
181 if (level == 0)
182 break;
183
184 if (*ptr == NULL) {
185 if (!alloc)
186 return NULL;
187
188 *ptr = (void *)get_zeroed_page(GFP_ATOMIC);
189 if (*ptr == NULL)
190 return NULL;
191 }
192
193 root = (struct pasid_state **)*ptr;
194 level -= 1;
195 }
196
197 return ptr;
198}
199
200static int set_pasid_state(struct device_state *dev_state,
201 struct pasid_state *pasid_state,
202 int pasid)
203{
204 struct pasid_state **ptr;
205 unsigned long flags;
206 int ret;
207
208 spin_lock_irqsave(&dev_state->lock, flags);
209 ptr = __get_pasid_state_ptr(dev_state, pasid, true);
210
211 ret = -ENOMEM;
212 if (ptr == NULL)
213 goto out_unlock;
214
215 ret = -ENOMEM;
216 if (*ptr != NULL)
217 goto out_unlock;
218
219 *ptr = pasid_state;
220
221 ret = 0;
222
223out_unlock:
224 spin_unlock_irqrestore(&dev_state->lock, flags);
225
226 return ret;
227}
228
229static void clear_pasid_state(struct device_state *dev_state, int pasid)
230{
231 struct pasid_state **ptr;
232 unsigned long flags;
233
234 spin_lock_irqsave(&dev_state->lock, flags);
235 ptr = __get_pasid_state_ptr(dev_state, pasid, true);
236
237 if (ptr == NULL)
238 goto out_unlock;
239
240 *ptr = NULL;
241
242out_unlock:
243 spin_unlock_irqrestore(&dev_state->lock, flags);
244}
245
246static struct pasid_state *get_pasid_state(struct device_state *dev_state,
247 int pasid)
248{
249 struct pasid_state **ptr, *ret = NULL;
250 unsigned long flags;
251
252 spin_lock_irqsave(&dev_state->lock, flags);
253 ptr = __get_pasid_state_ptr(dev_state, pasid, false);
254
255 if (ptr == NULL)
256 goto out_unlock;
257
258 ret = *ptr;
259 if (ret)
260 atomic_inc(&ret->count);
261
262out_unlock:
263 spin_unlock_irqrestore(&dev_state->lock, flags);
264
265 return ret;
266}
267
268static void free_pasid_state(struct pasid_state *pasid_state)
269{
270 kfree(pasid_state);
271}
272
273static void put_pasid_state(struct pasid_state *pasid_state)
274{
Oded Gabbay1c510992014-11-10 12:21:39 +0200275 if (atomic_dec_and_test(&pasid_state->count))
Joerg Roedel028eeac2011-11-24 12:48:13 +0100276 wake_up(&pasid_state->wq);
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100277}
278
Joerg Roedel028eeac2011-11-24 12:48:13 +0100279static void put_pasid_state_wait(struct pasid_state *pasid_state)
280{
281 DEFINE_WAIT(wait);
282
283 prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE);
284
Oded Gabbay1c510992014-11-10 12:21:39 +0200285 if (!atomic_dec_and_test(&pasid_state->count))
Joerg Roedel028eeac2011-11-24 12:48:13 +0100286 schedule();
287
288 finish_wait(&pasid_state->wq, &wait);
Joerg Roedel028eeac2011-11-24 12:48:13 +0100289 free_pasid_state(pasid_state);
290}
291
Joerg Roedel61feb432014-07-08 14:19:35 +0200292static void unbind_pasid(struct pasid_state *pasid_state)
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100293{
294 struct iommu_domain *domain;
295
296 domain = pasid_state->device_state->domain;
297
Joerg Roedel53d340e2014-07-08 15:01:43 +0200298 /*
299 * Mark pasid_state as invalid, no more faults will we added to the
300 * work queue after this is visible everywhere.
301 */
302 pasid_state->invalid = true;
303
304 /* Make sure this is visible */
305 smp_wmb();
306
307 /* After this the device/pasid can't access the mm anymore */
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100308 amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100309
310 /* Make sure no more pending faults are in the queue */
311 flush_workqueue(iommu_wq);
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100312}
313
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100314static void free_pasid_states_level1(struct pasid_state **tbl)
315{
316 int i;
317
318 for (i = 0; i < 512; ++i) {
319 if (tbl[i] == NULL)
320 continue;
321
322 free_page((unsigned long)tbl[i]);
323 }
324}
325
326static void free_pasid_states_level2(struct pasid_state **tbl)
327{
328 struct pasid_state **ptr;
329 int i;
330
331 for (i = 0; i < 512; ++i) {
332 if (tbl[i] == NULL)
333 continue;
334
335 ptr = (struct pasid_state **)tbl[i];
336 free_pasid_states_level1(ptr);
337 }
338}
339
340static void free_pasid_states(struct device_state *dev_state)
341{
342 struct pasid_state *pasid_state;
343 int i;
344
345 for (i = 0; i < dev_state->max_pasids; ++i) {
346 pasid_state = get_pasid_state(dev_state, i);
347 if (pasid_state == NULL)
348 continue;
349
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100350 put_pasid_state(pasid_state);
Joerg Roedela40d4c62014-05-20 23:18:24 +0200351
352 /*
353 * This will call the mn_release function and
354 * unbind the PASID
355 */
356 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
Joerg Roedelc5db16a2014-07-08 14:15:45 +0200357
358 put_pasid_state_wait(pasid_state); /* Reference taken in
Joerg Roedeldaff2f92014-07-30 16:04:40 +0200359 amd_iommu_bind_pasid */
Joerg Roedel75058a32014-07-30 16:04:39 +0200360
361 /* Drop reference taken in amd_iommu_bind_pasid */
362 put_device_state(dev_state);
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100363 }
364
365 if (dev_state->pasid_levels == 2)
366 free_pasid_states_level2(dev_state->states);
367 else if (dev_state->pasid_levels == 1)
368 free_pasid_states_level1(dev_state->states);
369 else if (dev_state->pasid_levels != 0)
370 BUG();
371
372 free_page((unsigned long)dev_state->states);
373}
374
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100375static struct pasid_state *mn_to_state(struct mmu_notifier *mn)
376{
377 return container_of(mn, struct pasid_state, mn);
378}
379
380static void __mn_flush_page(struct mmu_notifier *mn,
381 unsigned long address)
382{
383 struct pasid_state *pasid_state;
384 struct device_state *dev_state;
385
386 pasid_state = mn_to_state(mn);
387 dev_state = pasid_state->device_state;
388
389 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address);
390}
391
392static int mn_clear_flush_young(struct mmu_notifier *mn,
393 struct mm_struct *mm,
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -0700394 unsigned long start,
395 unsigned long end)
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100396{
Andres Lagar-Cavilla57128462014-09-22 14:54:42 -0700397 for (; start < end; start += PAGE_SIZE)
398 __mn_flush_page(mn, start);
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100399
400 return 0;
401}
402
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100403static void mn_invalidate_page(struct mmu_notifier *mn,
404 struct mm_struct *mm,
405 unsigned long address)
406{
407 __mn_flush_page(mn, address);
408}
409
Joerg Roedele7cc3dd2014-11-13 13:46:09 +1100410static void mn_invalidate_range(struct mmu_notifier *mn,
411 struct mm_struct *mm,
412 unsigned long start, unsigned long end)
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100413{
414 struct pasid_state *pasid_state;
415 struct device_state *dev_state;
416
417 pasid_state = mn_to_state(mn);
418 dev_state = pasid_state->device_state;
419
Joerg Roedele7cc3dd2014-11-13 13:46:09 +1100420 if ((start ^ (end - 1)) < PAGE_SIZE)
421 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,
422 start);
423 else
424 amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100425}
426
Joerg Roedela40d4c62014-05-20 23:18:24 +0200427static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
428{
429 struct pasid_state *pasid_state;
430 struct device_state *dev_state;
Joerg Roedeld9e16112014-07-09 15:43:11 +0200431 bool run_inv_ctx_cb;
Joerg Roedela40d4c62014-05-20 23:18:24 +0200432
433 might_sleep();
434
Joerg Roedeld9e16112014-07-09 15:43:11 +0200435 pasid_state = mn_to_state(mn);
436 dev_state = pasid_state->device_state;
437 run_inv_ctx_cb = !pasid_state->invalid;
Joerg Roedela40d4c62014-05-20 23:18:24 +0200438
Joerg Roedeld9e16112014-07-09 15:43:11 +0200439 if (run_inv_ctx_cb && pasid_state->device_state->inv_ctx_cb)
Joerg Roedela40d4c62014-05-20 23:18:24 +0200440 dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);
441
Joerg Roedel61feb432014-07-08 14:19:35 +0200442 unbind_pasid(pasid_state);
Joerg Roedela40d4c62014-05-20 23:18:24 +0200443}
444
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100445static struct mmu_notifier_ops iommu_mn = {
Joerg Roedela40d4c62014-05-20 23:18:24 +0200446 .release = mn_release,
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100447 .clear_flush_young = mn_clear_flush_young,
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100448 .invalidate_page = mn_invalidate_page,
Joerg Roedele7cc3dd2014-11-13 13:46:09 +1100449 .invalidate_range = mn_invalidate_range,
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100450};
451
Joerg Roedel028eeac2011-11-24 12:48:13 +0100452static void set_pri_tag_status(struct pasid_state *pasid_state,
453 u16 tag, int status)
454{
455 unsigned long flags;
456
457 spin_lock_irqsave(&pasid_state->lock, flags);
458 pasid_state->pri[tag].status = status;
459 spin_unlock_irqrestore(&pasid_state->lock, flags);
460}
461
462static void finish_pri_tag(struct device_state *dev_state,
463 struct pasid_state *pasid_state,
464 u16 tag)
465{
466 unsigned long flags;
467
468 spin_lock_irqsave(&pasid_state->lock, flags);
469 if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&
470 pasid_state->pri[tag].finish) {
471 amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,
472 pasid_state->pri[tag].status, tag);
473 pasid_state->pri[tag].finish = false;
474 pasid_state->pri[tag].status = PPR_SUCCESS;
475 }
476 spin_unlock_irqrestore(&pasid_state->lock, flags);
477}
478
Jesse Barnes9dc00f42014-12-12 16:55:30 -0800479static void handle_fault_error(struct fault *fault)
480{
481 int status;
482
483 if (!fault->dev_state->inv_ppr_cb) {
484 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
485 return;
486 }
487
488 status = fault->dev_state->inv_ppr_cb(fault->dev_state->pdev,
489 fault->pasid,
490 fault->address,
491 fault->flags);
492 switch (status) {
493 case AMD_IOMMU_INV_PRI_RSP_SUCCESS:
494 set_pri_tag_status(fault->state, fault->tag, PPR_SUCCESS);
495 break;
496 case AMD_IOMMU_INV_PRI_RSP_INVALID:
497 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
498 break;
499 case AMD_IOMMU_INV_PRI_RSP_FAIL:
500 set_pri_tag_status(fault->state, fault->tag, PPR_FAILURE);
501 break;
502 default:
503 BUG();
504 }
505}
506
Joerg Roedel028eeac2011-11-24 12:48:13 +0100507static void do_fault(struct work_struct *work)
508{
509 struct fault *fault = container_of(work, struct fault, work);
Jesse Barnes9dc00f42014-12-12 16:55:30 -0800510 struct mm_struct *mm;
511 struct vm_area_struct *vma;
512 u64 address;
513 int ret, write;
Joerg Roedel028eeac2011-11-24 12:48:13 +0100514
515 write = !!(fault->flags & PPR_FAULT_WRITE);
516
Jesse Barnes9dc00f42014-12-12 16:55:30 -0800517 mm = fault->state->mm;
518 address = fault->address;
Joerg Roedel028eeac2011-11-24 12:48:13 +0100519
Jesse Barnes9dc00f42014-12-12 16:55:30 -0800520 down_read(&mm->mmap_sem);
521 vma = find_extend_vma(mm, address);
522 if (!vma || address < vma->vm_start) {
523 /* failed to get a vma in the right range */
524 up_read(&mm->mmap_sem);
525 handle_fault_error(fault);
526 goto out;
Joerg Roedel175d6142011-11-28 14:36:36 +0100527 }
Joerg Roedel028eeac2011-11-24 12:48:13 +0100528
Jesse Barnes9dc00f42014-12-12 16:55:30 -0800529 ret = handle_mm_fault(mm, vma, address, write);
530 if (ret & VM_FAULT_ERROR) {
531 /* failed to service fault */
532 up_read(&mm->mmap_sem);
533 handle_fault_error(fault);
534 goto out;
535 }
536
537 up_read(&mm->mmap_sem);
538
539out:
Joerg Roedel028eeac2011-11-24 12:48:13 +0100540 finish_pri_tag(fault->dev_state, fault->state, fault->tag);
541
542 put_pasid_state(fault->state);
543
544 kfree(fault);
545}
546
547static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
548{
549 struct amd_iommu_fault *iommu_fault;
550 struct pasid_state *pasid_state;
551 struct device_state *dev_state;
552 unsigned long flags;
553 struct fault *fault;
554 bool finish;
555 u16 tag;
556 int ret;
557
558 iommu_fault = data;
559 tag = iommu_fault->tag & 0x1ff;
560 finish = (iommu_fault->tag >> 9) & 1;
561
562 ret = NOTIFY_DONE;
563 dev_state = get_device_state(iommu_fault->device_id);
564 if (dev_state == NULL)
565 goto out;
566
567 pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);
Joerg Roedel53d340e2014-07-08 15:01:43 +0200568 if (pasid_state == NULL || pasid_state->invalid) {
Joerg Roedel028eeac2011-11-24 12:48:13 +0100569 /* We know the device but not the PASID -> send INVALID */
570 amd_iommu_complete_ppr(dev_state->pdev, iommu_fault->pasid,
571 PPR_INVALID, tag);
572 goto out_drop_state;
573 }
574
575 spin_lock_irqsave(&pasid_state->lock, flags);
576 atomic_inc(&pasid_state->pri[tag].inflight);
577 if (finish)
578 pasid_state->pri[tag].finish = true;
579 spin_unlock_irqrestore(&pasid_state->lock, flags);
580
581 fault = kzalloc(sizeof(*fault), GFP_ATOMIC);
582 if (fault == NULL) {
583 /* We are OOM - send success and let the device re-fault */
584 finish_pri_tag(dev_state, pasid_state, tag);
585 goto out_drop_state;
586 }
587
588 fault->dev_state = dev_state;
589 fault->address = iommu_fault->address;
590 fault->state = pasid_state;
591 fault->tag = tag;
592 fault->finish = finish;
Alexey Skidanovb00675b2014-07-08 17:30:16 +0300593 fault->pasid = iommu_fault->pasid;
Joerg Roedel028eeac2011-11-24 12:48:13 +0100594 fault->flags = iommu_fault->flags;
595 INIT_WORK(&fault->work, do_fault);
596
597 queue_work(iommu_wq, &fault->work);
598
599 ret = NOTIFY_OK;
600
601out_drop_state:
Joerg Roedeldc88db72014-07-08 14:55:10 +0200602
603 if (ret != NOTIFY_OK && pasid_state)
604 put_pasid_state(pasid_state);
605
Joerg Roedel028eeac2011-11-24 12:48:13 +0100606 put_device_state(dev_state);
607
608out:
609 return ret;
610}
611
612static struct notifier_block ppr_nb = {
613 .notifier_call = ppr_notifier,
614};
615
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100616int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
617 struct task_struct *task)
618{
619 struct pasid_state *pasid_state;
620 struct device_state *dev_state;
Joerg Roedelf0aac632014-07-08 15:15:07 +0200621 struct mm_struct *mm;
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100622 u16 devid;
623 int ret;
624
625 might_sleep();
626
627 if (!amd_iommu_v2_supported())
628 return -ENODEV;
629
630 devid = device_id(pdev);
631 dev_state = get_device_state(devid);
632
633 if (dev_state == NULL)
634 return -EINVAL;
635
636 ret = -EINVAL;
637 if (pasid < 0 || pasid >= dev_state->max_pasids)
638 goto out;
639
640 ret = -ENOMEM;
641 pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);
642 if (pasid_state == NULL)
643 goto out;
644
Joerg Roedelf0aac632014-07-08 15:15:07 +0200645
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100646 atomic_set(&pasid_state->count, 1);
Joerg Roedel028eeac2011-11-24 12:48:13 +0100647 init_waitqueue_head(&pasid_state->wq);
Joerg Roedel2c13d472012-07-19 10:56:10 +0200648 spin_lock_init(&pasid_state->lock);
649
Joerg Roedelf0aac632014-07-08 15:15:07 +0200650 mm = get_task_mm(task);
Joerg Roedelf0aac632014-07-08 15:15:07 +0200651 pasid_state->mm = mm;
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100652 pasid_state->device_state = dev_state;
653 pasid_state->pasid = pasid;
Joerg Roedeld9e16112014-07-09 15:43:11 +0200654 pasid_state->invalid = true; /* Mark as valid only if we are
655 done with setting up the pasid */
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100656 pasid_state->mn.ops = &iommu_mn;
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100657
658 if (pasid_state->mm == NULL)
659 goto out_free;
660
Joerg Roedelf0aac632014-07-08 15:15:07 +0200661 mmu_notifier_register(&pasid_state->mn, mm);
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100662
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100663 ret = set_pasid_state(dev_state, pasid_state, pasid);
664 if (ret)
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100665 goto out_unregister;
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100666
667 ret = amd_iommu_domain_set_gcr3(dev_state->domain, pasid,
668 __pa(pasid_state->mm->pgd));
669 if (ret)
670 goto out_clear_state;
671
Joerg Roedeld9e16112014-07-09 15:43:11 +0200672 /* Now we are ready to handle faults */
673 pasid_state->invalid = false;
674
Joerg Roedelf0aac632014-07-08 15:15:07 +0200675 /*
676 * Drop the reference to the mm_struct here. We rely on the
677 * mmu_notifier release call-back to inform us when the mm
678 * is going away.
679 */
680 mmput(mm);
681
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100682 return 0;
683
684out_clear_state:
685 clear_pasid_state(dev_state, pasid);
686
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100687out_unregister:
Joerg Roedelf0aac632014-07-08 15:15:07 +0200688 mmu_notifier_unregister(&pasid_state->mn, mm);
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100689
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100690out_free:
Joerg Roedelf0aac632014-07-08 15:15:07 +0200691 mmput(mm);
Joerg Roedel028eeac2011-11-24 12:48:13 +0100692 free_pasid_state(pasid_state);
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100693
694out:
695 put_device_state(dev_state);
696
697 return ret;
698}
699EXPORT_SYMBOL(amd_iommu_bind_pasid);
700
701void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
702{
Joerg Roedela40d4c62014-05-20 23:18:24 +0200703 struct pasid_state *pasid_state;
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100704 struct device_state *dev_state;
705 u16 devid;
706
707 might_sleep();
708
709 if (!amd_iommu_v2_supported())
710 return;
711
712 devid = device_id(pdev);
713 dev_state = get_device_state(devid);
714 if (dev_state == NULL)
715 return;
716
717 if (pasid < 0 || pasid >= dev_state->max_pasids)
718 goto out;
719
Joerg Roedela40d4c62014-05-20 23:18:24 +0200720 pasid_state = get_pasid_state(dev_state, pasid);
721 if (pasid_state == NULL)
722 goto out;
723 /*
724 * Drop reference taken here. We are safe because we still hold
725 * the reference taken in the amd_iommu_bind_pasid function.
726 */
727 put_pasid_state(pasid_state);
728
Joerg Roedel53d340e2014-07-08 15:01:43 +0200729 /* Clear the pasid state so that the pasid can be re-used */
730 clear_pasid_state(dev_state, pasid_state->pasid);
731
Joerg Roedelf0aac632014-07-08 15:15:07 +0200732 /*
Joerg Roedelfcaa9602014-07-30 16:04:37 +0200733 * Call mmu_notifier_unregister to drop our reference
734 * to pasid_state->mm
Joerg Roedelf0aac632014-07-08 15:15:07 +0200735 */
Joerg Roedelfcaa9602014-07-30 16:04:37 +0200736 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100737
Joerg Roedelc5db16a2014-07-08 14:15:45 +0200738 put_pasid_state_wait(pasid_state); /* Reference taken in
Joerg Roedeldaff2f92014-07-30 16:04:40 +0200739 amd_iommu_bind_pasid */
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100740out:
Joerg Roedel75058a32014-07-30 16:04:39 +0200741 /* Drop reference taken in this function */
742 put_device_state(dev_state);
743
744 /* Drop reference taken in amd_iommu_bind_pasid */
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100745 put_device_state(dev_state);
746}
747EXPORT_SYMBOL(amd_iommu_unbind_pasid);
748
Joerg Roedeled96f222011-11-23 17:30:39 +0100749int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
750{
751 struct device_state *dev_state;
752 unsigned long flags;
753 int ret, tmp;
754 u16 devid;
755
756 might_sleep();
757
758 if (!amd_iommu_v2_supported())
759 return -ENODEV;
760
761 if (pasids <= 0 || pasids > (PASID_MASK + 1))
762 return -EINVAL;
763
764 devid = device_id(pdev);
765
766 dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL);
767 if (dev_state == NULL)
768 return -ENOMEM;
769
770 spin_lock_init(&dev_state->lock);
Joerg Roedel028eeac2011-11-24 12:48:13 +0100771 init_waitqueue_head(&dev_state->wq);
Joerg Roedel741669c2014-05-20 23:18:23 +0200772 dev_state->pdev = pdev;
773 dev_state->devid = devid;
Joerg Roedeled96f222011-11-23 17:30:39 +0100774
775 tmp = pasids;
776 for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
777 dev_state->pasid_levels += 1;
778
779 atomic_set(&dev_state->count, 1);
780 dev_state->max_pasids = pasids;
781
782 ret = -ENOMEM;
783 dev_state->states = (void *)get_zeroed_page(GFP_KERNEL);
784 if (dev_state->states == NULL)
785 goto out_free_dev_state;
786
787 dev_state->domain = iommu_domain_alloc(&pci_bus_type);
788 if (dev_state->domain == NULL)
789 goto out_free_states;
790
791 amd_iommu_domain_direct_map(dev_state->domain);
792
793 ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids);
794 if (ret)
795 goto out_free_domain;
796
797 ret = iommu_attach_device(dev_state->domain, &pdev->dev);
798 if (ret != 0)
799 goto out_free_domain;
800
801 spin_lock_irqsave(&state_lock, flags);
802
Joerg Roedel741669c2014-05-20 23:18:23 +0200803 if (__get_device_state(devid) != NULL) {
Joerg Roedeled96f222011-11-23 17:30:39 +0100804 spin_unlock_irqrestore(&state_lock, flags);
805 ret = -EBUSY;
806 goto out_free_domain;
807 }
808
Joerg Roedel741669c2014-05-20 23:18:23 +0200809 list_add_tail(&dev_state->list, &state_list);
Joerg Roedeled96f222011-11-23 17:30:39 +0100810
811 spin_unlock_irqrestore(&state_lock, flags);
812
813 return 0;
814
815out_free_domain:
816 iommu_domain_free(dev_state->domain);
817
818out_free_states:
819 free_page((unsigned long)dev_state->states);
820
821out_free_dev_state:
822 kfree(dev_state);
823
824 return ret;
825}
826EXPORT_SYMBOL(amd_iommu_init_device);
827
828void amd_iommu_free_device(struct pci_dev *pdev)
829{
830 struct device_state *dev_state;
831 unsigned long flags;
832 u16 devid;
833
834 if (!amd_iommu_v2_supported())
835 return;
836
837 devid = device_id(pdev);
838
839 spin_lock_irqsave(&state_lock, flags);
840
Joerg Roedelb87d2d72014-05-20 23:18:22 +0200841 dev_state = __get_device_state(devid);
Joerg Roedeled96f222011-11-23 17:30:39 +0100842 if (dev_state == NULL) {
843 spin_unlock_irqrestore(&state_lock, flags);
844 return;
845 }
846
Joerg Roedel741669c2014-05-20 23:18:23 +0200847 list_del(&dev_state->list);
Joerg Roedeled96f222011-11-23 17:30:39 +0100848
849 spin_unlock_irqrestore(&state_lock, flags);
850
Joerg Roedel2d5503b2011-11-24 10:41:57 +0100851 /* Get rid of any remaining pasid states */
852 free_pasid_states(dev_state);
853
Joerg Roedel028eeac2011-11-24 12:48:13 +0100854 put_device_state_wait(dev_state);
Joerg Roedeled96f222011-11-23 17:30:39 +0100855}
856EXPORT_SYMBOL(amd_iommu_free_device);
857
Joerg Roedel175d6142011-11-28 14:36:36 +0100858int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
859 amd_iommu_invalid_ppr_cb cb)
860{
861 struct device_state *dev_state;
862 unsigned long flags;
863 u16 devid;
864 int ret;
865
866 if (!amd_iommu_v2_supported())
867 return -ENODEV;
868
869 devid = device_id(pdev);
870
871 spin_lock_irqsave(&state_lock, flags);
872
873 ret = -EINVAL;
Joerg Roedelb87d2d72014-05-20 23:18:22 +0200874 dev_state = __get_device_state(devid);
Joerg Roedel175d6142011-11-28 14:36:36 +0100875 if (dev_state == NULL)
876 goto out_unlock;
877
878 dev_state->inv_ppr_cb = cb;
879
880 ret = 0;
881
882out_unlock:
883 spin_unlock_irqrestore(&state_lock, flags);
884
885 return ret;
886}
887EXPORT_SYMBOL(amd_iommu_set_invalid_ppr_cb);
888
Joerg Roedelbc216622011-12-07 12:24:42 +0100889int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
890 amd_iommu_invalidate_ctx cb)
891{
892 struct device_state *dev_state;
893 unsigned long flags;
894 u16 devid;
895 int ret;
896
897 if (!amd_iommu_v2_supported())
898 return -ENODEV;
899
900 devid = device_id(pdev);
901
902 spin_lock_irqsave(&state_lock, flags);
903
904 ret = -EINVAL;
Joerg Roedelb87d2d72014-05-20 23:18:22 +0200905 dev_state = __get_device_state(devid);
Joerg Roedelbc216622011-12-07 12:24:42 +0100906 if (dev_state == NULL)
907 goto out_unlock;
908
909 dev_state->inv_ctx_cb = cb;
910
911 ret = 0;
912
913out_unlock:
914 spin_unlock_irqrestore(&state_lock, flags);
915
916 return ret;
917}
918EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
919
Joerg Roedele3c495c2011-11-09 12:31:15 +0100920static int __init amd_iommu_v2_init(void)
921{
Joerg Roedel028eeac2011-11-24 12:48:13 +0100922 int ret;
Joerg Roedeled96f222011-11-23 17:30:39 +0100923
Joerg Roedel474d567d2012-03-15 12:46:40 +0100924 pr_info("AMD IOMMUv2 driver by Joerg Roedel <joerg.roedel@amd.com>\n");
925
926 if (!amd_iommu_v2_supported()) {
Masanari Iida07db0402012-07-22 02:21:32 +0900927 pr_info("AMD IOMMUv2 functionality not available on this system\n");
Joerg Roedel474d567d2012-03-15 12:46:40 +0100928 /*
929 * Load anyway to provide the symbols to other modules
930 * which may use AMD IOMMUv2 optionally.
931 */
932 return 0;
933 }
Joerg Roedele3c495c2011-11-09 12:31:15 +0100934
Joerg Roedeled96f222011-11-23 17:30:39 +0100935 spin_lock_init(&state_lock);
936
Joerg Roedel028eeac2011-11-24 12:48:13 +0100937 ret = -ENOMEM;
938 iommu_wq = create_workqueue("amd_iommu_v2");
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100939 if (iommu_wq == NULL)
Joerg Roedel741669c2014-05-20 23:18:23 +0200940 goto out;
Joerg Roedel8736b2c2011-11-24 16:21:52 +0100941
Joerg Roedel028eeac2011-11-24 12:48:13 +0100942 amd_iommu_register_ppr_notifier(&ppr_nb);
943
Joerg Roedele3c495c2011-11-09 12:31:15 +0100944 return 0;
Joerg Roedel028eeac2011-11-24 12:48:13 +0100945
Joerg Roedel741669c2014-05-20 23:18:23 +0200946out:
Joerg Roedel028eeac2011-11-24 12:48:13 +0100947 return ret;
Joerg Roedele3c495c2011-11-09 12:31:15 +0100948}
949
950static void __exit amd_iommu_v2_exit(void)
951{
Joerg Roedeled96f222011-11-23 17:30:39 +0100952 struct device_state *dev_state;
Joerg Roedeled96f222011-11-23 17:30:39 +0100953 int i;
954
Joerg Roedel474d567d2012-03-15 12:46:40 +0100955 if (!amd_iommu_v2_supported())
956 return;
957
Joerg Roedel028eeac2011-11-24 12:48:13 +0100958 amd_iommu_unregister_ppr_notifier(&ppr_nb);
959
960 flush_workqueue(iommu_wq);
961
962 /*
963 * The loop below might call flush_workqueue(), so call
964 * destroy_workqueue() after it
965 */
Joerg Roedeled96f222011-11-23 17:30:39 +0100966 for (i = 0; i < MAX_DEVICES; ++i) {
967 dev_state = get_device_state(i);
968
969 if (dev_state == NULL)
970 continue;
971
972 WARN_ON_ONCE(1);
973
Joerg Roedeled96f222011-11-23 17:30:39 +0100974 put_device_state(dev_state);
Joerg Roedel028eeac2011-11-24 12:48:13 +0100975 amd_iommu_free_device(dev_state->pdev);
Joerg Roedeled96f222011-11-23 17:30:39 +0100976 }
977
Joerg Roedel028eeac2011-11-24 12:48:13 +0100978 destroy_workqueue(iommu_wq);
Joerg Roedele3c495c2011-11-09 12:31:15 +0100979}
980
981module_init(amd_iommu_v2_init);
982module_exit(amd_iommu_v2_exit);