blob: 4ae0929c6e387b769dda8287580ad1b8b51e784a [file] [log] [blame]
Jens Axboe86db1e22008-01-29 14:53:40 +01001/*
2 * Functions related to io context handling
3 */
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/init.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>
9#include <linux/bootmem.h> /* for max_pfn/max_low_pfn */
10
11#include "blk.h"
12
13/*
14 * For io context allocations
15 */
16static struct kmem_cache *iocontext_cachep;
17
18static void cfq_dtor(struct io_context *ioc)
19{
20 struct cfq_io_context *cic[1];
21 int r;
22
23 /*
24 * We don't have a specific key to lookup with, so use the gang
25 * lookup to just retrieve the first item stored. The cfq exit
26 * function will iterate the full tree, so any member will do.
27 */
28 r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
29 if (r > 0)
30 cic[0]->dtor(ioc);
31}
32
33/*
34 * IO Context helper functions. put_io_context() returns 1 if there are no
35 * more users of this io context, 0 otherwise.
36 */
37int put_io_context(struct io_context *ioc)
38{
39 if (ioc == NULL)
40 return 1;
41
42 BUG_ON(atomic_read(&ioc->refcount) == 0);
43
44 if (atomic_dec_and_test(&ioc->refcount)) {
45 rcu_read_lock();
46 if (ioc->aic && ioc->aic->dtor)
47 ioc->aic->dtor(ioc->aic);
48 rcu_read_unlock();
49 cfq_dtor(ioc);
50
51 kmem_cache_free(iocontext_cachep, ioc);
52 return 1;
53 }
54 return 0;
55}
56EXPORT_SYMBOL(put_io_context);
57
58static void cfq_exit(struct io_context *ioc)
59{
60 struct cfq_io_context *cic[1];
61 int r;
62
63 rcu_read_lock();
64 /*
65 * See comment for cfq_dtor()
66 */
67 r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
68 rcu_read_unlock();
69
70 if (r > 0)
71 cic[0]->exit(ioc);
72}
73
74/* Called by the exitting task */
75void exit_io_context(void)
76{
77 struct io_context *ioc;
78
79 task_lock(current);
80 ioc = current->io_context;
81 current->io_context = NULL;
82 task_unlock(current);
83
84 if (atomic_dec_and_test(&ioc->nr_tasks)) {
85 if (ioc->aic && ioc->aic->exit)
86 ioc->aic->exit(ioc->aic);
87 cfq_exit(ioc);
88
89 put_io_context(ioc);
90 }
91}
92
93struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
94{
95 struct io_context *ret;
96
97 ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
98 if (ret) {
99 atomic_set(&ret->refcount, 1);
100 atomic_set(&ret->nr_tasks, 1);
101 spin_lock_init(&ret->lock);
102 ret->ioprio_changed = 0;
103 ret->ioprio = 0;
104 ret->last_waited = jiffies; /* doesn't matter... */
105 ret->nr_batch_requests = 0; /* because this is 0 */
106 ret->aic = NULL;
107 INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
108 ret->ioc_data = NULL;
109 }
110
111 return ret;
112}
113
114/*
115 * If the current task has no IO context then create one and initialise it.
116 * Otherwise, return its existing IO context.
117 *
118 * This returned IO context doesn't have a specifically elevated refcount,
119 * but since the current task itself holds a reference, the context can be
120 * used in general code, so long as it stays within `current` context.
121 */
122struct io_context *current_io_context(gfp_t gfp_flags, int node)
123{
124 struct task_struct *tsk = current;
125 struct io_context *ret;
126
127 ret = tsk->io_context;
128 if (likely(ret))
129 return ret;
130
131 ret = alloc_io_context(gfp_flags, node);
132 if (ret) {
133 /* make sure set_task_ioprio() sees the settings above */
134 smp_wmb();
135 tsk->io_context = ret;
136 }
137
138 return ret;
139}
140
141/*
142 * If the current task has no IO context then create one and initialise it.
143 * If it does have a context, take a ref on it.
144 *
145 * This is always called in the context of the task which submitted the I/O.
146 */
147struct io_context *get_io_context(gfp_t gfp_flags, int node)
148{
149 struct io_context *ret = NULL;
150
151 /*
152 * Check for unlikely race with exiting task. ioc ref count is
153 * zero when ioc is being detached.
154 */
155 do {
156 ret = current_io_context(gfp_flags, node);
157 if (unlikely(!ret))
158 break;
159 } while (!atomic_inc_not_zero(&ret->refcount));
160
161 return ret;
162}
163EXPORT_SYMBOL(get_io_context);
164
165void copy_io_context(struct io_context **pdst, struct io_context **psrc)
166{
167 struct io_context *src = *psrc;
168 struct io_context *dst = *pdst;
169
170 if (src) {
171 BUG_ON(atomic_read(&src->refcount) == 0);
172 atomic_inc(&src->refcount);
173 put_io_context(dst);
174 *pdst = src;
175 }
176}
177EXPORT_SYMBOL(copy_io_context);
178
Adrian Bunk13341592008-02-18 13:45:53 +0100179static int __init blk_ioc_init(void)
Jens Axboe86db1e22008-01-29 14:53:40 +0100180{
181 iocontext_cachep = kmem_cache_create("blkdev_ioc",
182 sizeof(struct io_context), 0, SLAB_PANIC, NULL);
183 return 0;
184}
185subsys_initcall(blk_ioc_init);