blob: 5cc4b871cb1155690b03454f6d772858c12b68fd [file] [log] [blame]
Jens Axboe75bb4622014-05-28 10:15:41 -06001/*
2 * Block multiqueue core code
3 *
4 * Copyright (C) 2013-2014 Jens Axboe
5 * Copyright (C) 2013-2014 Christoph Hellwig
6 */
Jens Axboe320ae512013-10-24 09:20:05 +01007#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/backing-dev.h>
10#include <linux/bio.h>
11#include <linux/blkdev.h>
12#include <linux/mm.h>
13#include <linux/init.h>
14#include <linux/slab.h>
15#include <linux/workqueue.h>
16#include <linux/smp.h>
17#include <linux/llist.h>
18#include <linux/list_sort.h>
19#include <linux/cpu.h>
20#include <linux/cache.h>
21#include <linux/sched/sysctl.h>
22#include <linux/delay.h>
23
24#include <trace/events/block.h>
25
26#include <linux/blk-mq.h>
27#include "blk.h"
28#include "blk-mq.h"
29#include "blk-mq-tag.h"
30
31static DEFINE_MUTEX(all_q_mutex);
32static LIST_HEAD(all_q_list);
33
34static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
35
Jens Axboe320ae512013-10-24 09:20:05 +010036static struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
37 unsigned int cpu)
38{
39 return per_cpu_ptr(q->queue_ctx, cpu);
40}
41
42/*
43 * This assumes per-cpu software queueing queues. They could be per-node
44 * as well, for instance. For now this is hardcoded as-is. Note that we don't
45 * care about preemption, since we know the ctx's are persistent. This does
46 * mean that we can't rely on ctx always matching the currently running CPU.
47 */
48static struct blk_mq_ctx *blk_mq_get_ctx(struct request_queue *q)
49{
50 return __blk_mq_get_ctx(q, get_cpu());
51}
52
53static void blk_mq_put_ctx(struct blk_mq_ctx *ctx)
54{
55 put_cpu();
56}
57
58/*
59 * Check if any of the ctx's have pending work in this hardware queue
60 */
61static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
62{
63 unsigned int i;
64
Jens Axboe1429d7c2014-05-19 09:23:55 -060065 for (i = 0; i < hctx->ctx_map.map_size; i++)
66 if (hctx->ctx_map.map[i].word)
Jens Axboe320ae512013-10-24 09:20:05 +010067 return true;
68
69 return false;
70}
71
Jens Axboe1429d7c2014-05-19 09:23:55 -060072static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
73 struct blk_mq_ctx *ctx)
74{
75 return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
76}
77
78#define CTX_TO_BIT(hctx, ctx) \
79 ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
80
Jens Axboe320ae512013-10-24 09:20:05 +010081/*
82 * Mark this ctx as having pending work in this hardware queue
83 */
84static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
85 struct blk_mq_ctx *ctx)
86{
Jens Axboe1429d7c2014-05-19 09:23:55 -060087 struct blk_align_bitmap *bm = get_bm(hctx, ctx);
88
89 if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
90 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
91}
92
93static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
94 struct blk_mq_ctx *ctx)
95{
96 struct blk_align_bitmap *bm = get_bm(hctx, ctx);
97
98 clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
Jens Axboe320ae512013-10-24 09:20:05 +010099}
100
Jens Axboe320ae512013-10-24 09:20:05 +0100101static int blk_mq_queue_enter(struct request_queue *q)
102{
103 int ret;
104
105 __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
106 smp_wmb();
107 /* we have problems to freeze the queue if it's initializing */
108 if (!blk_queue_bypass(q) || !blk_queue_init_done(q))
109 return 0;
110
111 __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
112
113 spin_lock_irq(q->queue_lock);
114 ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
Ming Lei43a5e4e2013-12-26 21:31:35 +0800115 !blk_queue_bypass(q) || blk_queue_dying(q),
116 *q->queue_lock);
Jens Axboe320ae512013-10-24 09:20:05 +0100117 /* inc usage with lock hold to avoid freeze_queue runs here */
Ming Lei43a5e4e2013-12-26 21:31:35 +0800118 if (!ret && !blk_queue_dying(q))
Jens Axboe320ae512013-10-24 09:20:05 +0100119 __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
Ming Lei43a5e4e2013-12-26 21:31:35 +0800120 else if (blk_queue_dying(q))
121 ret = -ENODEV;
Jens Axboe320ae512013-10-24 09:20:05 +0100122 spin_unlock_irq(q->queue_lock);
123
124 return ret;
125}
126
127static void blk_mq_queue_exit(struct request_queue *q)
128{
129 __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
130}
131
Ming Lei43a5e4e2013-12-26 21:31:35 +0800132static void __blk_mq_drain_queue(struct request_queue *q)
133{
134 while (true) {
135 s64 count;
136
137 spin_lock_irq(q->queue_lock);
138 count = percpu_counter_sum(&q->mq_usage_counter);
139 spin_unlock_irq(q->queue_lock);
140
141 if (count == 0)
142 break;
143 blk_mq_run_queues(q, false);
144 msleep(10);
145 }
146}
147
Jens Axboe320ae512013-10-24 09:20:05 +0100148/*
149 * Guarantee no request is in use, so we can change any data structure of
150 * the queue afterward.
151 */
152static void blk_mq_freeze_queue(struct request_queue *q)
153{
154 bool drain;
155
156 spin_lock_irq(q->queue_lock);
157 drain = !q->bypass_depth++;
158 queue_flag_set(QUEUE_FLAG_BYPASS, q);
159 spin_unlock_irq(q->queue_lock);
160
Ming Lei43a5e4e2013-12-26 21:31:35 +0800161 if (drain)
162 __blk_mq_drain_queue(q);
163}
Jens Axboe320ae512013-10-24 09:20:05 +0100164
Ming Lei43a5e4e2013-12-26 21:31:35 +0800165void blk_mq_drain_queue(struct request_queue *q)
166{
167 __blk_mq_drain_queue(q);
Jens Axboe320ae512013-10-24 09:20:05 +0100168}
169
170static void blk_mq_unfreeze_queue(struct request_queue *q)
171{
172 bool wake = false;
173
174 spin_lock_irq(q->queue_lock);
175 if (!--q->bypass_depth) {
176 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
177 wake = true;
178 }
179 WARN_ON_ONCE(q->bypass_depth < 0);
180 spin_unlock_irq(q->queue_lock);
181 if (wake)
182 wake_up_all(&q->mq_freeze_wq);
183}
184
185bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
186{
187 return blk_mq_has_free_tags(hctx->tags);
188}
189EXPORT_SYMBOL(blk_mq_can_queue);
190
Jens Axboe94eddfb2013-11-19 09:25:07 -0700191static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
192 struct request *rq, unsigned int rw_flags)
Jens Axboe320ae512013-10-24 09:20:05 +0100193{
Jens Axboe94eddfb2013-11-19 09:25:07 -0700194 if (blk_queue_io_stat(q))
195 rw_flags |= REQ_IO_STAT;
196
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200197 INIT_LIST_HEAD(&rq->queuelist);
198 /* csd/requeue_work/fifo_time is initialized before use */
199 rq->q = q;
Jens Axboe320ae512013-10-24 09:20:05 +0100200 rq->mq_ctx = ctx;
Jens Axboe0d2602c2014-05-13 15:10:52 -0600201 rq->cmd_flags |= rw_flags;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200202 rq->cmd_type = 0;
203 /* do not touch atomic flags, it needs atomic ops against the timer */
204 rq->cpu = -1;
205 rq->__data_len = 0;
206 rq->__sector = (sector_t) -1;
207 rq->bio = NULL;
208 rq->biotail = NULL;
209 INIT_HLIST_NODE(&rq->hash);
210 RB_CLEAR_NODE(&rq->rb_node);
211 memset(&rq->flush, 0, max(sizeof(rq->flush), sizeof(rq->elv)));
212 rq->rq_disk = NULL;
213 rq->part = NULL;
Ming Lei0fec08b2014-01-03 10:00:08 -0700214 rq->start_time = jiffies;
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200215#ifdef CONFIG_BLK_CGROUP
216 rq->rl = NULL;
Ming Lei0fec08b2014-01-03 10:00:08 -0700217 set_start_time_ns(rq);
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200218 rq->io_start_time_ns = 0;
219#endif
220 rq->nr_phys_segments = 0;
221#if defined(CONFIG_BLK_DEV_INTEGRITY)
222 rq->nr_integrity_segments = 0;
223#endif
224 rq->ioprio = 0;
225 rq->special = NULL;
226 /* tag was already set */
227 rq->errors = 0;
228 memset(rq->__cmd, 0, sizeof(rq->__cmd));
229 rq->cmd = rq->__cmd;
230 rq->cmd_len = BLK_MAX_CDB;
231
232 rq->extra_len = 0;
233 rq->sense_len = 0;
234 rq->resid_len = 0;
235 rq->sense = NULL;
236
237 rq->deadline = 0;
238 INIT_LIST_HEAD(&rq->timeout_list);
239 rq->timeout = 0;
240 rq->retries = 0;
241 rq->end_io = NULL;
242 rq->end_io_data = NULL;
243 rq->next_rq = NULL;
244
Jens Axboe320ae512013-10-24 09:20:05 +0100245 ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
246}
247
Christoph Hellwig5dee8572014-05-27 20:59:47 +0200248static struct request *
249__blk_mq_alloc_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
250 struct blk_mq_ctx *ctx, int rw, gfp_t gfp, bool reserved)
251{
252 struct request *rq;
253 unsigned int tag;
254
255 tag = blk_mq_get_tag(hctx, &ctx->last_tag, gfp, reserved);
256 if (tag != BLK_MQ_TAG_FAIL) {
257 rq = hctx->tags->rqs[tag];
258
259 rq->cmd_flags = 0;
260 if (blk_mq_tag_busy(hctx)) {
261 rq->cmd_flags = REQ_MQ_INFLIGHT;
262 atomic_inc(&hctx->nr_active);
263 }
264
265 rq->tag = tag;
266 blk_mq_rq_ctx_init(q, ctx, rq, rw);
267 return rq;
268 }
269
270 return NULL;
271}
272
Christoph Hellwig4ce01dd2014-05-27 20:59:46 +0200273struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
274 bool reserved)
Jens Axboe320ae512013-10-24 09:20:05 +0100275{
Christoph Hellwigd8525642014-05-27 20:59:50 +0200276 struct blk_mq_ctx *ctx;
277 struct blk_mq_hw_ctx *hctx;
Jens Axboe320ae512013-10-24 09:20:05 +0100278 struct request *rq;
279
280 if (blk_mq_queue_enter(q))
281 return NULL;
282
Christoph Hellwigd8525642014-05-27 20:59:50 +0200283 ctx = blk_mq_get_ctx(q);
284 hctx = q->mq_ops->map_queue(q, ctx->cpu);
285
286 rq = __blk_mq_alloc_request(q, hctx, ctx, rw, gfp & ~__GFP_WAIT,
287 reserved);
288 if (!rq && (gfp & __GFP_WAIT)) {
289 __blk_mq_run_hw_queue(hctx);
290 blk_mq_put_ctx(ctx);
291
292 ctx = blk_mq_get_ctx(q);
293 hctx = q->mq_ops->map_queue(q, ctx->cpu);
294 rq = __blk_mq_alloc_request(q, hctx, ctx, rw, gfp, reserved);
295 }
296 blk_mq_put_ctx(ctx);
Jens Axboe320ae512013-10-24 09:20:05 +0100297 return rq;
298}
Jens Axboe4bb659b2014-05-09 09:36:49 -0600299EXPORT_SYMBOL(blk_mq_alloc_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100300
Jens Axboe320ae512013-10-24 09:20:05 +0100301static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
302 struct blk_mq_ctx *ctx, struct request *rq)
303{
304 const int tag = rq->tag;
305 struct request_queue *q = rq->q;
306
Jens Axboe0d2602c2014-05-13 15:10:52 -0600307 if (rq->cmd_flags & REQ_MQ_INFLIGHT)
308 atomic_dec(&hctx->nr_active);
309
Christoph Hellwigaf76e552014-05-06 12:12:45 +0200310 clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
Jens Axboe0d2602c2014-05-13 15:10:52 -0600311 blk_mq_put_tag(hctx, tag, &ctx->last_tag);
Jens Axboe320ae512013-10-24 09:20:05 +0100312 blk_mq_queue_exit(q);
313}
314
315void blk_mq_free_request(struct request *rq)
316{
317 struct blk_mq_ctx *ctx = rq->mq_ctx;
318 struct blk_mq_hw_ctx *hctx;
319 struct request_queue *q = rq->q;
320
321 ctx->rq_completed[rq_is_sync(rq)]++;
322
323 hctx = q->mq_ops->map_queue(q, ctx->cpu);
324 __blk_mq_free_request(hctx, ctx, rq);
325}
326
Christoph Hellwig8727af42014-04-14 10:30:08 +0200327/*
328 * Clone all relevant state from a request that has been put on hold in
329 * the flush state machine into the preallocated flush request that hangs
330 * off the request queue.
331 *
332 * For a driver the flush request should be invisible, that's why we are
333 * impersonating the original request here.
334 */
335void blk_mq_clone_flush_request(struct request *flush_rq,
336 struct request *orig_rq)
337{
338 struct blk_mq_hw_ctx *hctx =
339 orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu);
340
341 flush_rq->mq_ctx = orig_rq->mq_ctx;
342 flush_rq->tag = orig_rq->tag;
343 memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq),
344 hctx->cmd_size);
345}
346
Christoph Hellwig63151a42014-04-16 09:44:52 +0200347inline void __blk_mq_end_io(struct request *rq, int error)
Jens Axboe320ae512013-10-24 09:20:05 +0100348{
Ming Lei0d11e6a2013-12-05 10:50:39 -0700349 blk_account_io_done(rq);
350
Christoph Hellwig91b63632014-04-16 09:44:53 +0200351 if (rq->end_io) {
Jens Axboe320ae512013-10-24 09:20:05 +0100352 rq->end_io(rq, error);
Christoph Hellwig91b63632014-04-16 09:44:53 +0200353 } else {
354 if (unlikely(blk_bidi_rq(rq)))
355 blk_mq_free_request(rq->next_rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100356 blk_mq_free_request(rq);
Christoph Hellwig91b63632014-04-16 09:44:53 +0200357 }
Jens Axboe320ae512013-10-24 09:20:05 +0100358}
Christoph Hellwig63151a42014-04-16 09:44:52 +0200359EXPORT_SYMBOL(__blk_mq_end_io);
360
361void blk_mq_end_io(struct request *rq, int error)
362{
363 if (blk_update_request(rq, error, blk_rq_bytes(rq)))
364 BUG();
365 __blk_mq_end_io(rq, error);
366}
367EXPORT_SYMBOL(blk_mq_end_io);
Jens Axboe320ae512013-10-24 09:20:05 +0100368
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800369static void __blk_mq_complete_request_remote(void *data)
Jens Axboe320ae512013-10-24 09:20:05 +0100370{
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800371 struct request *rq = data;
Jens Axboe320ae512013-10-24 09:20:05 +0100372
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800373 rq->q->softirq_done_fn(rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100374}
375
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800376void __blk_mq_complete_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100377{
378 struct blk_mq_ctx *ctx = rq->mq_ctx;
Christoph Hellwig38535202014-04-25 02:32:53 -0700379 bool shared = false;
Jens Axboe320ae512013-10-24 09:20:05 +0100380 int cpu;
381
Christoph Hellwig38535202014-04-25 02:32:53 -0700382 if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800383 rq->q->softirq_done_fn(rq);
384 return;
385 }
Jens Axboe320ae512013-10-24 09:20:05 +0100386
387 cpu = get_cpu();
Christoph Hellwig38535202014-04-25 02:32:53 -0700388 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
389 shared = cpus_share_cache(cpu, ctx->cpu);
390
391 if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800392 rq->csd.func = __blk_mq_complete_request_remote;
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800393 rq->csd.info = rq;
394 rq->csd.flags = 0;
Frederic Weisbeckerc46fff22014-02-24 16:40:02 +0100395 smp_call_function_single_async(ctx->cpu, &rq->csd);
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800396 } else {
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800397 rq->q->softirq_done_fn(rq);
Christoph Hellwig3d6efbf2014-01-08 09:33:37 -0800398 }
Jens Axboe320ae512013-10-24 09:20:05 +0100399 put_cpu();
400}
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800401
402/**
403 * blk_mq_complete_request - end I/O on a request
404 * @rq: the request being processed
405 *
406 * Description:
407 * Ends all I/O on a request. It does not handle partial completions.
408 * The actual completion happens out-of-order, through a IPI handler.
409 **/
410void blk_mq_complete_request(struct request *rq)
411{
Jens Axboe95f09682014-05-27 17:46:48 -0600412 struct request_queue *q = rq->q;
413
414 if (unlikely(blk_should_fake_timeout(q)))
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800415 return;
Jens Axboe95f09682014-05-27 17:46:48 -0600416 if (!blk_mark_rq_complete(rq)) {
417 if (q->softirq_done_fn)
418 __blk_mq_complete_request(rq);
419 else
420 blk_mq_end_io(rq, rq->errors);
421 }
Christoph Hellwig30a91cb2014-02-10 03:24:38 -0800422}
423EXPORT_SYMBOL(blk_mq_complete_request);
Jens Axboe320ae512013-10-24 09:20:05 +0100424
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800425static void blk_mq_start_request(struct request *rq, bool last)
Jens Axboe320ae512013-10-24 09:20:05 +0100426{
427 struct request_queue *q = rq->q;
428
429 trace_block_rq_issue(q, rq);
430
Christoph Hellwig742ee692014-04-14 10:30:06 +0200431 rq->resid_len = blk_rq_bytes(rq);
Christoph Hellwig91b63632014-04-16 09:44:53 +0200432 if (unlikely(blk_bidi_rq(rq)))
433 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
Christoph Hellwig742ee692014-04-14 10:30:06 +0200434
Jens Axboe320ae512013-10-24 09:20:05 +0100435 /*
436 * Just mark start time and set the started bit. Due to memory
437 * ordering, we know we'll see the correct deadline as long as
Jens Axboec22d9d82014-05-23 14:14:57 -0600438 * REQ_ATOMIC_STARTED is seen. Use the default queue timeout,
439 * unless one has been set in the request.
Jens Axboe320ae512013-10-24 09:20:05 +0100440 */
Jens Axboec22d9d82014-05-23 14:14:57 -0600441 if (!rq->timeout)
442 rq->deadline = jiffies + q->rq_timeout;
443 else
444 rq->deadline = jiffies + rq->timeout;
Jens Axboe87ee7b12014-04-24 08:51:47 -0600445
446 /*
447 * Mark us as started and clear complete. Complete might have been
448 * set if requeue raced with timeout, which then marked it as
449 * complete. So be sure to clear complete again when we start
450 * the request, otherwise we'll ignore the completion event.
451 */
Jens Axboe320ae512013-10-24 09:20:05 +0100452 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
Jens Axboe87ee7b12014-04-24 08:51:47 -0600453 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800454
455 if (q->dma_drain_size && blk_rq_bytes(rq)) {
456 /*
457 * Make sure space for the drain appears. We know we can do
458 * this because max_hw_segments has been adjusted to be one
459 * fewer than the device can handle.
460 */
461 rq->nr_phys_segments++;
462 }
463
464 /*
465 * Flag the last request in the series so that drivers know when IO
466 * should be kicked off, if they don't do it on a per-request basis.
467 *
468 * Note: the flag isn't the only condition drivers should do kick off.
469 * If drive is busy, the last request might not have the bit set.
470 */
471 if (last)
472 rq->cmd_flags |= REQ_END;
Jens Axboe320ae512013-10-24 09:20:05 +0100473}
474
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200475static void __blk_mq_requeue_request(struct request *rq)
Jens Axboe320ae512013-10-24 09:20:05 +0100476{
477 struct request_queue *q = rq->q;
478
479 trace_block_rq_requeue(q, rq);
480 clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800481
482 rq->cmd_flags &= ~REQ_END;
483
484 if (q->dma_drain_size && blk_rq_bytes(rq))
485 rq->nr_phys_segments--;
Jens Axboe320ae512013-10-24 09:20:05 +0100486}
487
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200488void blk_mq_requeue_request(struct request *rq)
489{
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200490 __blk_mq_requeue_request(rq);
491 blk_clear_rq_complete(rq);
492
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200493 BUG_ON(blk_queued_rq(rq));
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600494 blk_mq_add_to_requeue_list(rq, true);
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200495}
496EXPORT_SYMBOL(blk_mq_requeue_request);
497
Christoph Hellwig6fca6a62014-05-28 08:08:02 -0600498static void blk_mq_requeue_work(struct work_struct *work)
499{
500 struct request_queue *q =
501 container_of(work, struct request_queue, requeue_work);
502 LIST_HEAD(rq_list);
503 struct request *rq, *next;
504 unsigned long flags;
505
506 spin_lock_irqsave(&q->requeue_lock, flags);
507 list_splice_init(&q->requeue_list, &rq_list);
508 spin_unlock_irqrestore(&q->requeue_lock, flags);
509
510 list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
511 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
512 continue;
513
514 rq->cmd_flags &= ~REQ_SOFTBARRIER;
515 list_del_init(&rq->queuelist);
516 blk_mq_insert_request(rq, true, false, false);
517 }
518
519 while (!list_empty(&rq_list)) {
520 rq = list_entry(rq_list.next, struct request, queuelist);
521 list_del_init(&rq->queuelist);
522 blk_mq_insert_request(rq, false, false, false);
523 }
524
525 blk_mq_run_queues(q, false);
526}
527
528void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
529{
530 struct request_queue *q = rq->q;
531 unsigned long flags;
532
533 /*
534 * We abuse this flag that is otherwise used by the I/O scheduler to
535 * request head insertation from the workqueue.
536 */
537 BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
538
539 spin_lock_irqsave(&q->requeue_lock, flags);
540 if (at_head) {
541 rq->cmd_flags |= REQ_SOFTBARRIER;
542 list_add(&rq->queuelist, &q->requeue_list);
543 } else {
544 list_add_tail(&rq->queuelist, &q->requeue_list);
545 }
546 spin_unlock_irqrestore(&q->requeue_lock, flags);
547}
548EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
549
550void blk_mq_kick_requeue_list(struct request_queue *q)
551{
552 kblockd_schedule_work(&q->requeue_work);
553}
554EXPORT_SYMBOL(blk_mq_kick_requeue_list);
555
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600556struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
557{
558 return tags->rqs[tag];
559}
560EXPORT_SYMBOL(blk_mq_tag_to_rq);
561
Jens Axboe320ae512013-10-24 09:20:05 +0100562struct blk_mq_timeout_data {
563 struct blk_mq_hw_ctx *hctx;
564 unsigned long *next;
565 unsigned int *next_set;
566};
567
568static void blk_mq_timeout_check(void *__data, unsigned long *free_tags)
569{
570 struct blk_mq_timeout_data *data = __data;
571 struct blk_mq_hw_ctx *hctx = data->hctx;
572 unsigned int tag;
573
574 /* It may not be in flight yet (this is where
575 * the REQ_ATOMIC_STARTED flag comes in). The requests are
576 * statically allocated, so we know it's always safe to access the
577 * memory associated with a bit offset into ->rqs[].
578 */
579 tag = 0;
580 do {
581 struct request *rq;
582
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600583 tag = find_next_zero_bit(free_tags, hctx->tags->nr_tags, tag);
584 if (tag >= hctx->tags->nr_tags)
Jens Axboe320ae512013-10-24 09:20:05 +0100585 break;
586
Christoph Hellwig24d2f902014-04-15 14:14:00 -0600587 rq = blk_mq_tag_to_rq(hctx->tags, tag++);
588 if (rq->q != hctx->queue)
589 continue;
Jens Axboe320ae512013-10-24 09:20:05 +0100590 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
591 continue;
592
593 blk_rq_check_expired(rq, data->next, data->next_set);
594 } while (1);
595}
596
597static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx,
598 unsigned long *next,
599 unsigned int *next_set)
600{
601 struct blk_mq_timeout_data data = {
602 .hctx = hctx,
603 .next = next,
604 .next_set = next_set,
605 };
606
607 /*
608 * Ask the tagging code to iterate busy requests, so we can
609 * check them for timeout.
610 */
611 blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data);
612}
613
Jens Axboe87ee7b12014-04-24 08:51:47 -0600614static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq)
615{
616 struct request_queue *q = rq->q;
617
618 /*
619 * We know that complete is set at this point. If STARTED isn't set
620 * anymore, then the request isn't active and the "timeout" should
621 * just be ignored. This can happen due to the bitflag ordering.
622 * Timeout first checks if STARTED is set, and if it is, assumes
623 * the request is active. But if we race with completion, then
624 * we both flags will get cleared. So check here again, and ignore
625 * a timeout event with a request that isn't active.
626 */
627 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
628 return BLK_EH_NOT_HANDLED;
629
630 if (!q->mq_ops->timeout)
631 return BLK_EH_RESET_TIMER;
632
633 return q->mq_ops->timeout(rq);
634}
635
Jens Axboe320ae512013-10-24 09:20:05 +0100636static void blk_mq_rq_timer(unsigned long data)
637{
638 struct request_queue *q = (struct request_queue *) data;
639 struct blk_mq_hw_ctx *hctx;
640 unsigned long next = 0;
641 int i, next_set = 0;
642
Jens Axboe484b4062014-05-21 14:01:15 -0600643 queue_for_each_hw_ctx(q, hctx, i) {
644 /*
645 * If not software queues are currently mapped to this
646 * hardware queue, there's nothing to check
647 */
648 if (!hctx->nr_ctx || !hctx->tags)
649 continue;
650
Jens Axboe320ae512013-10-24 09:20:05 +0100651 blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set);
Jens Axboe484b4062014-05-21 14:01:15 -0600652 }
Jens Axboe320ae512013-10-24 09:20:05 +0100653
Jens Axboe0d2602c2014-05-13 15:10:52 -0600654 if (next_set) {
655 next = blk_rq_timeout(round_jiffies_up(next));
656 mod_timer(&q->timeout, next);
657 } else {
658 queue_for_each_hw_ctx(q, hctx, i)
659 blk_mq_tag_idle(hctx);
660 }
Jens Axboe320ae512013-10-24 09:20:05 +0100661}
662
663/*
664 * Reverse check our software queue for entries that we could potentially
665 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
666 * too much time checking for merges.
667 */
668static bool blk_mq_attempt_merge(struct request_queue *q,
669 struct blk_mq_ctx *ctx, struct bio *bio)
670{
671 struct request *rq;
672 int checked = 8;
673
674 list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
675 int el_ret;
676
677 if (!checked--)
678 break;
679
680 if (!blk_rq_merge_ok(rq, bio))
681 continue;
682
683 el_ret = blk_try_merge(rq, bio);
684 if (el_ret == ELEVATOR_BACK_MERGE) {
685 if (bio_attempt_back_merge(q, rq, bio)) {
686 ctx->rq_merged++;
687 return true;
688 }
689 break;
690 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
691 if (bio_attempt_front_merge(q, rq, bio)) {
692 ctx->rq_merged++;
693 return true;
694 }
695 break;
696 }
697 }
698
699 return false;
700}
701
Jens Axboe320ae512013-10-24 09:20:05 +0100702/*
Jens Axboe1429d7c2014-05-19 09:23:55 -0600703 * Process software queues that have been marked busy, splicing them
704 * to the for-dispatch
705 */
706static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
707{
708 struct blk_mq_ctx *ctx;
709 int i;
710
711 for (i = 0; i < hctx->ctx_map.map_size; i++) {
712 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
713 unsigned int off, bit;
714
715 if (!bm->word)
716 continue;
717
718 bit = 0;
719 off = i * hctx->ctx_map.bits_per_word;
720 do {
721 bit = find_next_bit(&bm->word, bm->depth, bit);
722 if (bit >= bm->depth)
723 break;
724
725 ctx = hctx->ctxs[bit + off];
726 clear_bit(bit, &bm->word);
727 spin_lock(&ctx->lock);
728 list_splice_tail_init(&ctx->rq_list, list);
729 spin_unlock(&ctx->lock);
730
731 bit++;
732 } while (1);
733 }
734}
735
736/*
Jens Axboe320ae512013-10-24 09:20:05 +0100737 * Run this hardware queue, pulling any software queues mapped to it in.
738 * Note that this function currently has various problems around ordering
739 * of IO. In particular, we'd like FIFO behaviour on handling existing
740 * items on the hctx->dispatch list. Ignore that for now.
741 */
742static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
743{
744 struct request_queue *q = hctx->queue;
Jens Axboe320ae512013-10-24 09:20:05 +0100745 struct request *rq;
746 LIST_HEAD(rq_list);
Jens Axboe1429d7c2014-05-19 09:23:55 -0600747 int queued;
Jens Axboe320ae512013-10-24 09:20:05 +0100748
Jens Axboefd1270d2014-04-16 09:23:48 -0600749 WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
Jens Axboee4043dc2014-04-09 10:18:23 -0600750
Jens Axboe5d12f902014-03-19 15:25:02 -0600751 if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
Jens Axboe320ae512013-10-24 09:20:05 +0100752 return;
753
754 hctx->run++;
755
756 /*
757 * Touch any software queue that has pending entries.
758 */
Jens Axboe1429d7c2014-05-19 09:23:55 -0600759 flush_busy_ctxs(hctx, &rq_list);
Jens Axboe320ae512013-10-24 09:20:05 +0100760
761 /*
762 * If we have previous entries on our dispatch list, grab them
763 * and stuff them at the front for more fair dispatch.
764 */
765 if (!list_empty_careful(&hctx->dispatch)) {
766 spin_lock(&hctx->lock);
767 if (!list_empty(&hctx->dispatch))
768 list_splice_init(&hctx->dispatch, &rq_list);
769 spin_unlock(&hctx->lock);
770 }
771
772 /*
Jens Axboe320ae512013-10-24 09:20:05 +0100773 * Now process all the entries, sending them to the driver.
774 */
Jens Axboe1429d7c2014-05-19 09:23:55 -0600775 queued = 0;
Jens Axboe320ae512013-10-24 09:20:05 +0100776 while (!list_empty(&rq_list)) {
777 int ret;
778
779 rq = list_first_entry(&rq_list, struct request, queuelist);
780 list_del_init(&rq->queuelist);
Jens Axboe320ae512013-10-24 09:20:05 +0100781
Christoph Hellwig49f5baa2014-02-11 08:27:14 -0800782 blk_mq_start_request(rq, list_empty(&rq_list));
Jens Axboe320ae512013-10-24 09:20:05 +0100783
784 ret = q->mq_ops->queue_rq(hctx, rq);
785 switch (ret) {
786 case BLK_MQ_RQ_QUEUE_OK:
787 queued++;
788 continue;
789 case BLK_MQ_RQ_QUEUE_BUSY:
Jens Axboe320ae512013-10-24 09:20:05 +0100790 list_add(&rq->queuelist, &rq_list);
Christoph Hellwiged0791b2014-04-16 09:44:57 +0200791 __blk_mq_requeue_request(rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100792 break;
793 default:
794 pr_err("blk-mq: bad return on queue: %d\n", ret);
Jens Axboe320ae512013-10-24 09:20:05 +0100795 case BLK_MQ_RQ_QUEUE_ERROR:
Christoph Hellwig1e93b8c2014-02-11 08:27:13 -0800796 rq->errors = -EIO;
Jens Axboe320ae512013-10-24 09:20:05 +0100797 blk_mq_end_io(rq, rq->errors);
798 break;
799 }
800
801 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
802 break;
803 }
804
805 if (!queued)
806 hctx->dispatched[0]++;
807 else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
808 hctx->dispatched[ilog2(queued) + 1]++;
809
810 /*
811 * Any items that need requeuing? Stuff them into hctx->dispatch,
812 * that is where we will continue on next queue run.
813 */
814 if (!list_empty(&rq_list)) {
815 spin_lock(&hctx->lock);
816 list_splice(&rq_list, &hctx->dispatch);
817 spin_unlock(&hctx->lock);
818 }
819}
820
Jens Axboe506e9312014-05-07 10:26:44 -0600821/*
822 * It'd be great if the workqueue API had a way to pass
823 * in a mask and had some smarts for more clever placement.
824 * For now we just round-robin here, switching for every
825 * BLK_MQ_CPU_WORK_BATCH queued items.
826 */
827static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
828{
829 int cpu = hctx->next_cpu;
830
831 if (--hctx->next_cpu_batch <= 0) {
832 int next_cpu;
833
834 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
835 if (next_cpu >= nr_cpu_ids)
836 next_cpu = cpumask_first(hctx->cpumask);
837
838 hctx->next_cpu = next_cpu;
839 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
840 }
841
842 return cpu;
843}
844
Jens Axboe320ae512013-10-24 09:20:05 +0100845void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
846{
Jens Axboe5d12f902014-03-19 15:25:02 -0600847 if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
Jens Axboe320ae512013-10-24 09:20:05 +0100848 return;
849
Jens Axboee4043dc2014-04-09 10:18:23 -0600850 if (!async && cpumask_test_cpu(smp_processor_id(), hctx->cpumask))
Jens Axboe320ae512013-10-24 09:20:05 +0100851 __blk_mq_run_hw_queue(hctx);
Jens Axboee4043dc2014-04-09 10:18:23 -0600852 else if (hctx->queue->nr_hw_queues == 1)
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600853 kblockd_schedule_delayed_work(&hctx->run_work, 0);
Jens Axboee4043dc2014-04-09 10:18:23 -0600854 else {
855 unsigned int cpu;
856
Jens Axboe506e9312014-05-07 10:26:44 -0600857 cpu = blk_mq_hctx_next_cpu(hctx);
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600858 kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
Jens Axboee4043dc2014-04-09 10:18:23 -0600859 }
Jens Axboe320ae512013-10-24 09:20:05 +0100860}
861
862void blk_mq_run_queues(struct request_queue *q, bool async)
863{
864 struct blk_mq_hw_ctx *hctx;
865 int i;
866
867 queue_for_each_hw_ctx(q, hctx, i) {
868 if ((!blk_mq_hctx_has_pending(hctx) &&
869 list_empty_careful(&hctx->dispatch)) ||
Jens Axboe5d12f902014-03-19 15:25:02 -0600870 test_bit(BLK_MQ_S_STOPPED, &hctx->state))
Jens Axboe320ae512013-10-24 09:20:05 +0100871 continue;
872
Jens Axboee4043dc2014-04-09 10:18:23 -0600873 preempt_disable();
Jens Axboe320ae512013-10-24 09:20:05 +0100874 blk_mq_run_hw_queue(hctx, async);
Jens Axboee4043dc2014-04-09 10:18:23 -0600875 preempt_enable();
Jens Axboe320ae512013-10-24 09:20:05 +0100876 }
877}
878EXPORT_SYMBOL(blk_mq_run_queues);
879
880void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
881{
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600882 cancel_delayed_work(&hctx->run_work);
883 cancel_delayed_work(&hctx->delay_work);
Jens Axboe320ae512013-10-24 09:20:05 +0100884 set_bit(BLK_MQ_S_STOPPED, &hctx->state);
885}
886EXPORT_SYMBOL(blk_mq_stop_hw_queue);
887
Christoph Hellwig280d45f2013-10-25 14:45:58 +0100888void blk_mq_stop_hw_queues(struct request_queue *q)
889{
890 struct blk_mq_hw_ctx *hctx;
891 int i;
892
893 queue_for_each_hw_ctx(q, hctx, i)
894 blk_mq_stop_hw_queue(hctx);
895}
896EXPORT_SYMBOL(blk_mq_stop_hw_queues);
897
Jens Axboe320ae512013-10-24 09:20:05 +0100898void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
899{
900 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
Jens Axboee4043dc2014-04-09 10:18:23 -0600901
902 preempt_disable();
Jens Axboe320ae512013-10-24 09:20:05 +0100903 __blk_mq_run_hw_queue(hctx);
Jens Axboee4043dc2014-04-09 10:18:23 -0600904 preempt_enable();
Jens Axboe320ae512013-10-24 09:20:05 +0100905}
906EXPORT_SYMBOL(blk_mq_start_hw_queue);
907
Christoph Hellwig2f268552014-04-16 09:44:56 +0200908void blk_mq_start_hw_queues(struct request_queue *q)
909{
910 struct blk_mq_hw_ctx *hctx;
911 int i;
912
913 queue_for_each_hw_ctx(q, hctx, i)
914 blk_mq_start_hw_queue(hctx);
915}
916EXPORT_SYMBOL(blk_mq_start_hw_queues);
917
918
Christoph Hellwig1b4a3252014-04-16 09:44:54 +0200919void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
Jens Axboe320ae512013-10-24 09:20:05 +0100920{
921 struct blk_mq_hw_ctx *hctx;
922 int i;
923
924 queue_for_each_hw_ctx(q, hctx, i) {
925 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
926 continue;
927
928 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
Jens Axboee4043dc2014-04-09 10:18:23 -0600929 preempt_disable();
Christoph Hellwig1b4a3252014-04-16 09:44:54 +0200930 blk_mq_run_hw_queue(hctx, async);
Jens Axboee4043dc2014-04-09 10:18:23 -0600931 preempt_enable();
Jens Axboe320ae512013-10-24 09:20:05 +0100932 }
933}
934EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
935
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600936static void blk_mq_run_work_fn(struct work_struct *work)
Jens Axboe320ae512013-10-24 09:20:05 +0100937{
938 struct blk_mq_hw_ctx *hctx;
939
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600940 hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
Jens Axboee4043dc2014-04-09 10:18:23 -0600941
Jens Axboe320ae512013-10-24 09:20:05 +0100942 __blk_mq_run_hw_queue(hctx);
943}
944
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600945static void blk_mq_delay_work_fn(struct work_struct *work)
946{
947 struct blk_mq_hw_ctx *hctx;
948
949 hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
950
951 if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
952 __blk_mq_run_hw_queue(hctx);
953}
954
955void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
956{
957 unsigned long tmo = msecs_to_jiffies(msecs);
958
959 if (hctx->queue->nr_hw_queues == 1)
960 kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
961 else {
962 unsigned int cpu;
963
Jens Axboe506e9312014-05-07 10:26:44 -0600964 cpu = blk_mq_hctx_next_cpu(hctx);
Christoph Hellwig70f4db62014-04-16 10:48:08 -0600965 kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
966 }
967}
968EXPORT_SYMBOL(blk_mq_delay_queue);
969
Jens Axboe320ae512013-10-24 09:20:05 +0100970static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
Christoph Hellwig72a0a362014-02-07 10:22:36 -0800971 struct request *rq, bool at_head)
Jens Axboe320ae512013-10-24 09:20:05 +0100972{
973 struct blk_mq_ctx *ctx = rq->mq_ctx;
974
Jens Axboe01b983c2013-11-19 18:59:10 -0700975 trace_block_rq_insert(hctx->queue, rq);
976
Christoph Hellwig72a0a362014-02-07 10:22:36 -0800977 if (at_head)
978 list_add(&rq->queuelist, &ctx->rq_list);
979 else
980 list_add_tail(&rq->queuelist, &ctx->rq_list);
Jens Axboe4bb659b2014-05-09 09:36:49 -0600981
Jens Axboe320ae512013-10-24 09:20:05 +0100982 blk_mq_hctx_mark_pending(hctx, ctx);
983
984 /*
985 * We do this early, to ensure we are on the right CPU.
986 */
Jens Axboe87ee7b12014-04-24 08:51:47 -0600987 blk_add_timer(rq);
Jens Axboe320ae512013-10-24 09:20:05 +0100988}
989
Christoph Hellwigeeabc852014-03-21 08:57:37 -0600990void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
991 bool async)
Jens Axboe320ae512013-10-24 09:20:05 +0100992{
993 struct request_queue *q = rq->q;
994 struct blk_mq_hw_ctx *hctx;
Christoph Hellwigeeabc852014-03-21 08:57:37 -0600995 struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
Jens Axboe320ae512013-10-24 09:20:05 +0100996
997 current_ctx = blk_mq_get_ctx(q);
Christoph Hellwigeeabc852014-03-21 08:57:37 -0600998 if (!cpu_online(ctx->cpu))
999 rq->mq_ctx = ctx = current_ctx;
Jens Axboe320ae512013-10-24 09:20:05 +01001000
Jens Axboe320ae512013-10-24 09:20:05 +01001001 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1002
Christoph Hellwigeeabc852014-03-21 08:57:37 -06001003 if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA) &&
1004 !(rq->cmd_flags & (REQ_FLUSH_SEQ))) {
1005 blk_insert_flush(rq);
1006 } else {
1007 spin_lock(&ctx->lock);
1008 __blk_mq_insert_request(hctx, rq, at_head);
1009 spin_unlock(&ctx->lock);
1010 }
Jens Axboe320ae512013-10-24 09:20:05 +01001011
Jens Axboe320ae512013-10-24 09:20:05 +01001012 if (run_queue)
1013 blk_mq_run_hw_queue(hctx, async);
Jens Axboee4043dc2014-04-09 10:18:23 -06001014
1015 blk_mq_put_ctx(current_ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001016}
1017
1018static void blk_mq_insert_requests(struct request_queue *q,
1019 struct blk_mq_ctx *ctx,
1020 struct list_head *list,
1021 int depth,
1022 bool from_schedule)
1023
1024{
1025 struct blk_mq_hw_ctx *hctx;
1026 struct blk_mq_ctx *current_ctx;
1027
1028 trace_block_unplug(q, depth, !from_schedule);
1029
1030 current_ctx = blk_mq_get_ctx(q);
1031
1032 if (!cpu_online(ctx->cpu))
1033 ctx = current_ctx;
1034 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1035
1036 /*
1037 * preemption doesn't flush plug list, so it's possible ctx->cpu is
1038 * offline now
1039 */
1040 spin_lock(&ctx->lock);
1041 while (!list_empty(list)) {
1042 struct request *rq;
1043
1044 rq = list_first_entry(list, struct request, queuelist);
1045 list_del_init(&rq->queuelist);
1046 rq->mq_ctx = ctx;
Christoph Hellwig72a0a362014-02-07 10:22:36 -08001047 __blk_mq_insert_request(hctx, rq, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001048 }
1049 spin_unlock(&ctx->lock);
1050
Jens Axboe320ae512013-10-24 09:20:05 +01001051 blk_mq_run_hw_queue(hctx, from_schedule);
Jens Axboee4043dc2014-04-09 10:18:23 -06001052 blk_mq_put_ctx(current_ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001053}
1054
1055static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1056{
1057 struct request *rqa = container_of(a, struct request, queuelist);
1058 struct request *rqb = container_of(b, struct request, queuelist);
1059
1060 return !(rqa->mq_ctx < rqb->mq_ctx ||
1061 (rqa->mq_ctx == rqb->mq_ctx &&
1062 blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1063}
1064
1065void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1066{
1067 struct blk_mq_ctx *this_ctx;
1068 struct request_queue *this_q;
1069 struct request *rq;
1070 LIST_HEAD(list);
1071 LIST_HEAD(ctx_list);
1072 unsigned int depth;
1073
1074 list_splice_init(&plug->mq_list, &list);
1075
1076 list_sort(NULL, &list, plug_ctx_cmp);
1077
1078 this_q = NULL;
1079 this_ctx = NULL;
1080 depth = 0;
1081
1082 while (!list_empty(&list)) {
1083 rq = list_entry_rq(list.next);
1084 list_del_init(&rq->queuelist);
1085 BUG_ON(!rq->q);
1086 if (rq->mq_ctx != this_ctx) {
1087 if (this_ctx) {
1088 blk_mq_insert_requests(this_q, this_ctx,
1089 &ctx_list, depth,
1090 from_schedule);
1091 }
1092
1093 this_ctx = rq->mq_ctx;
1094 this_q = rq->q;
1095 depth = 0;
1096 }
1097
1098 depth++;
1099 list_add_tail(&rq->queuelist, &ctx_list);
1100 }
1101
1102 /*
1103 * If 'this_ctx' is set, we know we have entries to complete
1104 * on 'ctx_list'. Do those.
1105 */
1106 if (this_ctx) {
1107 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1108 from_schedule);
1109 }
1110}
1111
1112static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1113{
1114 init_request_from_bio(rq, bio);
1115 blk_account_io_start(rq, 1);
1116}
1117
Jens Axboe07068d52014-05-22 10:40:51 -06001118static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1119 struct blk_mq_ctx *ctx,
1120 struct request *rq, struct bio *bio)
1121{
1122 struct request_queue *q = hctx->queue;
1123
1124 if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
1125 blk_mq_bio_to_request(rq, bio);
1126 spin_lock(&ctx->lock);
1127insert_rq:
1128 __blk_mq_insert_request(hctx, rq, false);
1129 spin_unlock(&ctx->lock);
1130 return false;
1131 } else {
1132 spin_lock(&ctx->lock);
1133 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1134 blk_mq_bio_to_request(rq, bio);
1135 goto insert_rq;
1136 }
1137
1138 spin_unlock(&ctx->lock);
1139 __blk_mq_free_request(hctx, ctx, rq);
1140 return true;
1141 }
1142}
1143
1144struct blk_map_ctx {
1145 struct blk_mq_hw_ctx *hctx;
1146 struct blk_mq_ctx *ctx;
1147};
1148
1149static struct request *blk_mq_map_request(struct request_queue *q,
1150 struct bio *bio,
1151 struct blk_map_ctx *data)
Jens Axboe320ae512013-10-24 09:20:05 +01001152{
1153 struct blk_mq_hw_ctx *hctx;
1154 struct blk_mq_ctx *ctx;
Jens Axboe320ae512013-10-24 09:20:05 +01001155 struct request *rq;
Jens Axboe07068d52014-05-22 10:40:51 -06001156 int rw = bio_data_dir(bio);
Jens Axboe320ae512013-10-24 09:20:05 +01001157
Jens Axboe07068d52014-05-22 10:40:51 -06001158 if (unlikely(blk_mq_queue_enter(q))) {
Nicholas Bellinger14ec77f2014-02-07 13:45:39 -07001159 bio_endio(bio, -EIO);
Jens Axboe07068d52014-05-22 10:40:51 -06001160 return NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01001161 }
1162
1163 ctx = blk_mq_get_ctx(q);
1164 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1165
Jens Axboe07068d52014-05-22 10:40:51 -06001166 if (rw_is_sync(bio->bi_rw))
Shaohua Li27fbf4e82014-02-19 20:20:21 +08001167 rw |= REQ_SYNC;
Jens Axboe07068d52014-05-22 10:40:51 -06001168
Jens Axboe320ae512013-10-24 09:20:05 +01001169 trace_block_getrq(q, bio, rw);
Christoph Hellwig5dee8572014-05-27 20:59:47 +02001170 rq = __blk_mq_alloc_request(q, hctx, ctx, rw, GFP_ATOMIC, false);
1171 if (unlikely(!rq)) {
Christoph Hellwig793597a2014-05-27 20:59:49 +02001172 __blk_mq_run_hw_queue(hctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001173 blk_mq_put_ctx(ctx);
1174 trace_block_sleeprq(q, bio, rw);
Christoph Hellwig793597a2014-05-27 20:59:49 +02001175
1176 ctx = blk_mq_get_ctx(q);
Jens Axboe320ae512013-10-24 09:20:05 +01001177 hctx = q->mq_ops->map_queue(q, ctx->cpu);
Christoph Hellwig793597a2014-05-27 20:59:49 +02001178 rq = __blk_mq_alloc_request(q, hctx, ctx, rw,
1179 __GFP_WAIT|GFP_ATOMIC, false);
Jens Axboe320ae512013-10-24 09:20:05 +01001180 }
1181
1182 hctx->queued++;
Jens Axboe07068d52014-05-22 10:40:51 -06001183 data->hctx = hctx;
1184 data->ctx = ctx;
1185 return rq;
1186}
1187
1188/*
1189 * Multiple hardware queue variant. This will not use per-process plugs,
1190 * but will attempt to bypass the hctx queueing if we can go straight to
1191 * hardware for SYNC IO.
1192 */
1193static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1194{
1195 const int is_sync = rw_is_sync(bio->bi_rw);
1196 const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1197 struct blk_map_ctx data;
1198 struct request *rq;
1199
1200 blk_queue_bounce(q, &bio);
1201
1202 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1203 bio_endio(bio, -EIO);
1204 return;
1205 }
1206
1207 rq = blk_mq_map_request(q, bio, &data);
1208 if (unlikely(!rq))
1209 return;
1210
1211 if (unlikely(is_flush_fua)) {
1212 blk_mq_bio_to_request(rq, bio);
1213 blk_insert_flush(rq);
1214 goto run_queue;
1215 }
1216
1217 if (is_sync) {
1218 int ret;
1219
1220 blk_mq_bio_to_request(rq, bio);
1221 blk_mq_start_request(rq, true);
1222
1223 /*
1224 * For OK queue, we are done. For error, kill it. Any other
1225 * error (busy), just add it to our list as we previously
1226 * would have done
1227 */
1228 ret = q->mq_ops->queue_rq(data.hctx, rq);
1229 if (ret == BLK_MQ_RQ_QUEUE_OK)
1230 goto done;
1231 else {
1232 __blk_mq_requeue_request(rq);
1233
1234 if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1235 rq->errors = -EIO;
1236 blk_mq_end_io(rq, rq->errors);
1237 goto done;
1238 }
1239 }
1240 }
1241
1242 if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1243 /*
1244 * For a SYNC request, send it to the hardware immediately. For
1245 * an ASYNC request, just ensure that we run it later on. The
1246 * latter allows for merging opportunities and more efficient
1247 * dispatching.
1248 */
1249run_queue:
1250 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1251 }
1252done:
1253 blk_mq_put_ctx(data.ctx);
1254}
1255
1256/*
1257 * Single hardware queue variant. This will attempt to use any per-process
1258 * plug for merging and IO deferral.
1259 */
1260static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1261{
1262 const int is_sync = rw_is_sync(bio->bi_rw);
1263 const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1264 unsigned int use_plug, request_count = 0;
1265 struct blk_map_ctx data;
1266 struct request *rq;
1267
1268 /*
1269 * If we have multiple hardware queues, just go directly to
1270 * one of those for sync IO.
1271 */
1272 use_plug = !is_flush_fua && !is_sync;
1273
1274 blk_queue_bounce(q, &bio);
1275
1276 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1277 bio_endio(bio, -EIO);
1278 return;
1279 }
1280
1281 if (use_plug && !blk_queue_nomerges(q) &&
1282 blk_attempt_plug_merge(q, bio, &request_count))
1283 return;
1284
1285 rq = blk_mq_map_request(q, bio, &data);
Jens Axboe320ae512013-10-24 09:20:05 +01001286
1287 if (unlikely(is_flush_fua)) {
1288 blk_mq_bio_to_request(rq, bio);
Jens Axboe320ae512013-10-24 09:20:05 +01001289 blk_insert_flush(rq);
1290 goto run_queue;
1291 }
1292
1293 /*
1294 * A task plug currently exists. Since this is completely lockless,
1295 * utilize that to temporarily store requests until the task is
1296 * either done or scheduled away.
1297 */
1298 if (use_plug) {
1299 struct blk_plug *plug = current->plug;
1300
1301 if (plug) {
1302 blk_mq_bio_to_request(rq, bio);
Shaohua Li92f399c2013-10-29 12:01:03 -06001303 if (list_empty(&plug->mq_list))
Jens Axboe320ae512013-10-24 09:20:05 +01001304 trace_block_plug(q);
1305 else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1306 blk_flush_plug_list(plug, false);
1307 trace_block_plug(q);
1308 }
1309 list_add_tail(&rq->queuelist, &plug->mq_list);
Jens Axboe07068d52014-05-22 10:40:51 -06001310 blk_mq_put_ctx(data.ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001311 return;
1312 }
1313 }
1314
Jens Axboe07068d52014-05-22 10:40:51 -06001315 if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1316 /*
1317 * For a SYNC request, send it to the hardware immediately. For
1318 * an ASYNC request, just ensure that we run it later on. The
1319 * latter allows for merging opportunities and more efficient
1320 * dispatching.
1321 */
1322run_queue:
1323 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
Jens Axboe320ae512013-10-24 09:20:05 +01001324 }
1325
Jens Axboe07068d52014-05-22 10:40:51 -06001326 blk_mq_put_ctx(data.ctx);
Jens Axboe320ae512013-10-24 09:20:05 +01001327}
1328
1329/*
1330 * Default mapping to a software queue, since we use one per CPU.
1331 */
1332struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1333{
1334 return q->queue_hw_ctx[q->mq_map[cpu]];
1335}
1336EXPORT_SYMBOL(blk_mq_map_queue);
1337
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001338struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *set,
Jens Axboef14bbe72014-05-27 12:06:53 -06001339 unsigned int hctx_index,
1340 int node)
Jens Axboe320ae512013-10-24 09:20:05 +01001341{
Jens Axboef14bbe72014-05-27 12:06:53 -06001342 return kzalloc_node(sizeof(struct blk_mq_hw_ctx), GFP_KERNEL, node);
Jens Axboe320ae512013-10-24 09:20:05 +01001343}
1344EXPORT_SYMBOL(blk_mq_alloc_single_hw_queue);
1345
1346void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *hctx,
1347 unsigned int hctx_index)
1348{
1349 kfree(hctx);
1350}
1351EXPORT_SYMBOL(blk_mq_free_single_hw_queue);
1352
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001353static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1354 struct blk_mq_tags *tags, unsigned int hctx_idx)
Jens Axboe320ae512013-10-24 09:20:05 +01001355{
1356 struct page *page;
1357
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001358 if (tags->rqs && set->ops->exit_request) {
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001359 int i;
1360
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001361 for (i = 0; i < tags->nr_tags; i++) {
1362 if (!tags->rqs[i])
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001363 continue;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001364 set->ops->exit_request(set->driver_data, tags->rqs[i],
1365 hctx_idx, i);
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001366 }
1367 }
1368
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001369 while (!list_empty(&tags->page_list)) {
1370 page = list_first_entry(&tags->page_list, struct page, lru);
Dave Hansen67534712014-01-08 20:17:46 -07001371 list_del_init(&page->lru);
Jens Axboe320ae512013-10-24 09:20:05 +01001372 __free_pages(page, page->private);
1373 }
1374
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001375 kfree(tags->rqs);
Jens Axboe320ae512013-10-24 09:20:05 +01001376
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001377 blk_mq_free_tags(tags);
Jens Axboe320ae512013-10-24 09:20:05 +01001378}
1379
1380static size_t order_to_size(unsigned int order)
1381{
Ming Lei4ca08502014-04-19 18:00:18 +08001382 return (size_t)PAGE_SIZE << order;
Jens Axboe320ae512013-10-24 09:20:05 +01001383}
1384
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001385static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1386 unsigned int hctx_idx)
Jens Axboe320ae512013-10-24 09:20:05 +01001387{
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001388 struct blk_mq_tags *tags;
Jens Axboe320ae512013-10-24 09:20:05 +01001389 unsigned int i, j, entries_per_page, max_order = 4;
1390 size_t rq_size, left;
1391
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001392 tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1393 set->numa_node);
1394 if (!tags)
1395 return NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01001396
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001397 INIT_LIST_HEAD(&tags->page_list);
1398
1399 tags->rqs = kmalloc_node(set->queue_depth * sizeof(struct request *),
1400 GFP_KERNEL, set->numa_node);
1401 if (!tags->rqs) {
1402 blk_mq_free_tags(tags);
1403 return NULL;
1404 }
Jens Axboe320ae512013-10-24 09:20:05 +01001405
1406 /*
1407 * rq_size is the size of the request plus driver payload, rounded
1408 * to the cacheline size
1409 */
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001410 rq_size = round_up(sizeof(struct request) + set->cmd_size,
Jens Axboe320ae512013-10-24 09:20:05 +01001411 cache_line_size());
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001412 left = rq_size * set->queue_depth;
Jens Axboe320ae512013-10-24 09:20:05 +01001413
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001414 for (i = 0; i < set->queue_depth; ) {
Jens Axboe320ae512013-10-24 09:20:05 +01001415 int this_order = max_order;
1416 struct page *page;
1417 int to_do;
1418 void *p;
1419
1420 while (left < order_to_size(this_order - 1) && this_order)
1421 this_order--;
1422
1423 do {
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001424 page = alloc_pages_node(set->numa_node, GFP_KERNEL,
1425 this_order);
Jens Axboe320ae512013-10-24 09:20:05 +01001426 if (page)
1427 break;
1428 if (!this_order--)
1429 break;
1430 if (order_to_size(this_order) < rq_size)
1431 break;
1432 } while (1);
1433
1434 if (!page)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001435 goto fail;
Jens Axboe320ae512013-10-24 09:20:05 +01001436
1437 page->private = this_order;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001438 list_add_tail(&page->lru, &tags->page_list);
Jens Axboe320ae512013-10-24 09:20:05 +01001439
1440 p = page_address(page);
1441 entries_per_page = order_to_size(this_order) / rq_size;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001442 to_do = min(entries_per_page, set->queue_depth - i);
Jens Axboe320ae512013-10-24 09:20:05 +01001443 left -= to_do * rq_size;
1444 for (j = 0; j < to_do; j++) {
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001445 tags->rqs[i] = p;
1446 if (set->ops->init_request) {
1447 if (set->ops->init_request(set->driver_data,
1448 tags->rqs[i], hctx_idx, i,
1449 set->numa_node))
1450 goto fail;
Christoph Hellwige9b267d2014-04-15 13:59:10 -06001451 }
1452
Jens Axboe320ae512013-10-24 09:20:05 +01001453 p += rq_size;
1454 i++;
1455 }
1456 }
1457
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001458 return tags;
Jens Axboe320ae512013-10-24 09:20:05 +01001459
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001460fail:
1461 pr_warn("%s: failed to allocate requests\n", __func__);
1462 blk_mq_free_rq_map(set, tags, hctx_idx);
1463 return NULL;
Jens Axboe320ae512013-10-24 09:20:05 +01001464}
1465
Jens Axboe1429d7c2014-05-19 09:23:55 -06001466static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1467{
1468 kfree(bitmap->map);
1469}
1470
1471static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1472{
1473 unsigned int bpw = 8, total, num_maps, i;
1474
1475 bitmap->bits_per_word = bpw;
1476
1477 num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1478 bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1479 GFP_KERNEL, node);
1480 if (!bitmap->map)
1481 return -ENOMEM;
1482
1483 bitmap->map_size = num_maps;
1484
1485 total = nr_cpu_ids;
1486 for (i = 0; i < num_maps; i++) {
1487 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1488 total -= bitmap->map[i].depth;
1489 }
1490
1491 return 0;
1492}
1493
Jens Axboe484b4062014-05-21 14:01:15 -06001494static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1495{
1496 struct request_queue *q = hctx->queue;
1497 struct blk_mq_ctx *ctx;
1498 LIST_HEAD(tmp);
1499
1500 /*
1501 * Move ctx entries to new CPU, if this one is going away.
1502 */
1503 ctx = __blk_mq_get_ctx(q, cpu);
1504
1505 spin_lock(&ctx->lock);
1506 if (!list_empty(&ctx->rq_list)) {
1507 list_splice_init(&ctx->rq_list, &tmp);
1508 blk_mq_hctx_clear_pending(hctx, ctx);
1509 }
1510 spin_unlock(&ctx->lock);
1511
1512 if (list_empty(&tmp))
1513 return NOTIFY_OK;
1514
1515 ctx = blk_mq_get_ctx(q);
1516 spin_lock(&ctx->lock);
1517
1518 while (!list_empty(&tmp)) {
1519 struct request *rq;
1520
1521 rq = list_first_entry(&tmp, struct request, queuelist);
1522 rq->mq_ctx = ctx;
1523 list_move_tail(&rq->queuelist, &ctx->rq_list);
1524 }
1525
1526 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1527 blk_mq_hctx_mark_pending(hctx, ctx);
1528
1529 spin_unlock(&ctx->lock);
1530
1531 blk_mq_run_hw_queue(hctx, true);
1532 blk_mq_put_ctx(ctx);
1533 return NOTIFY_OK;
1534}
1535
1536static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1537{
1538 struct request_queue *q = hctx->queue;
1539 struct blk_mq_tag_set *set = q->tag_set;
1540
1541 if (set->tags[hctx->queue_num])
1542 return NOTIFY_OK;
1543
1544 set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1545 if (!set->tags[hctx->queue_num])
1546 return NOTIFY_STOP;
1547
1548 hctx->tags = set->tags[hctx->queue_num];
1549 return NOTIFY_OK;
1550}
1551
1552static int blk_mq_hctx_notify(void *data, unsigned long action,
1553 unsigned int cpu)
1554{
1555 struct blk_mq_hw_ctx *hctx = data;
1556
1557 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1558 return blk_mq_hctx_cpu_offline(hctx, cpu);
1559 else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1560 return blk_mq_hctx_cpu_online(hctx, cpu);
1561
1562 return NOTIFY_OK;
1563}
1564
Ming Lei624dbe42014-05-27 23:35:13 +08001565static void blk_mq_exit_hw_queues(struct request_queue *q,
1566 struct blk_mq_tag_set *set, int nr_queue)
1567{
1568 struct blk_mq_hw_ctx *hctx;
1569 unsigned int i;
1570
1571 queue_for_each_hw_ctx(q, hctx, i) {
1572 if (i == nr_queue)
1573 break;
1574
1575 if (set->ops->exit_hctx)
1576 set->ops->exit_hctx(hctx, i);
1577
1578 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1579 kfree(hctx->ctxs);
1580 blk_mq_free_bitmap(&hctx->ctx_map);
1581 }
1582
1583}
1584
1585static void blk_mq_free_hw_queues(struct request_queue *q,
1586 struct blk_mq_tag_set *set)
1587{
1588 struct blk_mq_hw_ctx *hctx;
1589 unsigned int i;
1590
1591 queue_for_each_hw_ctx(q, hctx, i) {
1592 free_cpumask_var(hctx->cpumask);
1593 set->ops->free_hctx(hctx, i);
1594 }
1595}
1596
Jens Axboe320ae512013-10-24 09:20:05 +01001597static int blk_mq_init_hw_queues(struct request_queue *q,
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001598 struct blk_mq_tag_set *set)
Jens Axboe320ae512013-10-24 09:20:05 +01001599{
1600 struct blk_mq_hw_ctx *hctx;
Ming Lei624dbe42014-05-27 23:35:13 +08001601 unsigned int i;
Jens Axboe320ae512013-10-24 09:20:05 +01001602
1603 /*
1604 * Initialize hardware queues
1605 */
1606 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboe320ae512013-10-24 09:20:05 +01001607 int node;
1608
1609 node = hctx->numa_node;
1610 if (node == NUMA_NO_NODE)
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001611 node = hctx->numa_node = set->numa_node;
Jens Axboe320ae512013-10-24 09:20:05 +01001612
Christoph Hellwig70f4db62014-04-16 10:48:08 -06001613 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1614 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
Jens Axboe320ae512013-10-24 09:20:05 +01001615 spin_lock_init(&hctx->lock);
1616 INIT_LIST_HEAD(&hctx->dispatch);
1617 hctx->queue = q;
1618 hctx->queue_num = i;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001619 hctx->flags = set->flags;
1620 hctx->cmd_size = set->cmd_size;
Jens Axboe320ae512013-10-24 09:20:05 +01001621
1622 blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1623 blk_mq_hctx_notify, hctx);
1624 blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1625
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001626 hctx->tags = set->tags[i];
Jens Axboe320ae512013-10-24 09:20:05 +01001627
1628 /*
1629 * Allocate space for all possible cpus to avoid allocation in
1630 * runtime
1631 */
1632 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1633 GFP_KERNEL, node);
1634 if (!hctx->ctxs)
1635 break;
1636
Jens Axboe1429d7c2014-05-19 09:23:55 -06001637 if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
Jens Axboe320ae512013-10-24 09:20:05 +01001638 break;
1639
Jens Axboe320ae512013-10-24 09:20:05 +01001640 hctx->nr_ctx = 0;
1641
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001642 if (set->ops->init_hctx &&
1643 set->ops->init_hctx(hctx, set->driver_data, i))
Jens Axboe320ae512013-10-24 09:20:05 +01001644 break;
1645 }
1646
1647 if (i == q->nr_hw_queues)
1648 return 0;
1649
1650 /*
1651 * Init failed
1652 */
Ming Lei624dbe42014-05-27 23:35:13 +08001653 blk_mq_exit_hw_queues(q, set, i);
Jens Axboe320ae512013-10-24 09:20:05 +01001654
1655 return 1;
1656}
1657
1658static void blk_mq_init_cpu_queues(struct request_queue *q,
1659 unsigned int nr_hw_queues)
1660{
1661 unsigned int i;
1662
1663 for_each_possible_cpu(i) {
1664 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1665 struct blk_mq_hw_ctx *hctx;
1666
1667 memset(__ctx, 0, sizeof(*__ctx));
1668 __ctx->cpu = i;
1669 spin_lock_init(&__ctx->lock);
1670 INIT_LIST_HEAD(&__ctx->rq_list);
1671 __ctx->queue = q;
1672
1673 /* If the cpu isn't online, the cpu is mapped to first hctx */
Jens Axboe320ae512013-10-24 09:20:05 +01001674 if (!cpu_online(i))
1675 continue;
1676
Jens Axboee4043dc2014-04-09 10:18:23 -06001677 hctx = q->mq_ops->map_queue(q, i);
1678 cpumask_set_cpu(i, hctx->cpumask);
1679 hctx->nr_ctx++;
1680
Jens Axboe320ae512013-10-24 09:20:05 +01001681 /*
1682 * Set local node, IFF we have more than one hw queue. If
1683 * not, we remain on the home node of the device
1684 */
1685 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1686 hctx->numa_node = cpu_to_node(i);
1687 }
1688}
1689
1690static void blk_mq_map_swqueue(struct request_queue *q)
1691{
1692 unsigned int i;
1693 struct blk_mq_hw_ctx *hctx;
1694 struct blk_mq_ctx *ctx;
1695
1696 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboee4043dc2014-04-09 10:18:23 -06001697 cpumask_clear(hctx->cpumask);
Jens Axboe320ae512013-10-24 09:20:05 +01001698 hctx->nr_ctx = 0;
1699 }
1700
1701 /*
1702 * Map software to hardware queues
1703 */
1704 queue_for_each_ctx(q, ctx, i) {
1705 /* If the cpu isn't online, the cpu is mapped to first hctx */
Jens Axboee4043dc2014-04-09 10:18:23 -06001706 if (!cpu_online(i))
1707 continue;
1708
Jens Axboe320ae512013-10-24 09:20:05 +01001709 hctx = q->mq_ops->map_queue(q, i);
Jens Axboee4043dc2014-04-09 10:18:23 -06001710 cpumask_set_cpu(i, hctx->cpumask);
Jens Axboe320ae512013-10-24 09:20:05 +01001711 ctx->index_hw = hctx->nr_ctx;
1712 hctx->ctxs[hctx->nr_ctx++] = ctx;
1713 }
Jens Axboe506e9312014-05-07 10:26:44 -06001714
1715 queue_for_each_hw_ctx(q, hctx, i) {
Jens Axboe484b4062014-05-21 14:01:15 -06001716 /*
1717 * If not software queues are mapped to this hardware queue,
1718 * disable it and free the request entries
1719 */
1720 if (!hctx->nr_ctx) {
1721 struct blk_mq_tag_set *set = q->tag_set;
1722
1723 if (set->tags[i]) {
1724 blk_mq_free_rq_map(set, set->tags[i], i);
1725 set->tags[i] = NULL;
1726 hctx->tags = NULL;
1727 }
1728 continue;
1729 }
1730
1731 /*
1732 * Initialize batch roundrobin counts
1733 */
Jens Axboe506e9312014-05-07 10:26:44 -06001734 hctx->next_cpu = cpumask_first(hctx->cpumask);
1735 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1736 }
Jens Axboe320ae512013-10-24 09:20:05 +01001737}
1738
Jens Axboe0d2602c2014-05-13 15:10:52 -06001739static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1740{
1741 struct blk_mq_hw_ctx *hctx;
1742 struct request_queue *q;
1743 bool shared;
1744 int i;
1745
1746 if (set->tag_list.next == set->tag_list.prev)
1747 shared = false;
1748 else
1749 shared = true;
1750
1751 list_for_each_entry(q, &set->tag_list, tag_set_list) {
1752 blk_mq_freeze_queue(q);
1753
1754 queue_for_each_hw_ctx(q, hctx, i) {
1755 if (shared)
1756 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1757 else
1758 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1759 }
1760 blk_mq_unfreeze_queue(q);
1761 }
1762}
1763
1764static void blk_mq_del_queue_tag_set(struct request_queue *q)
1765{
1766 struct blk_mq_tag_set *set = q->tag_set;
1767
1768 blk_mq_freeze_queue(q);
1769
1770 mutex_lock(&set->tag_list_lock);
1771 list_del_init(&q->tag_set_list);
1772 blk_mq_update_tag_set_depth(set);
1773 mutex_unlock(&set->tag_list_lock);
1774
1775 blk_mq_unfreeze_queue(q);
1776}
1777
1778static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1779 struct request_queue *q)
1780{
1781 q->tag_set = set;
1782
1783 mutex_lock(&set->tag_list_lock);
1784 list_add_tail(&q->tag_set_list, &set->tag_list);
1785 blk_mq_update_tag_set_depth(set);
1786 mutex_unlock(&set->tag_list_lock);
1787}
1788
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001789struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
Jens Axboe320ae512013-10-24 09:20:05 +01001790{
1791 struct blk_mq_hw_ctx **hctxs;
1792 struct blk_mq_ctx *ctx;
1793 struct request_queue *q;
Jens Axboef14bbe72014-05-27 12:06:53 -06001794 unsigned int *map;
Jens Axboe320ae512013-10-24 09:20:05 +01001795 int i;
1796
Jens Axboe320ae512013-10-24 09:20:05 +01001797 ctx = alloc_percpu(struct blk_mq_ctx);
1798 if (!ctx)
1799 return ERR_PTR(-ENOMEM);
1800
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001801 hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1802 set->numa_node);
Jens Axboe320ae512013-10-24 09:20:05 +01001803
1804 if (!hctxs)
1805 goto err_percpu;
1806
Jens Axboef14bbe72014-05-27 12:06:53 -06001807 map = blk_mq_make_queue_map(set);
1808 if (!map)
1809 goto err_map;
1810
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001811 for (i = 0; i < set->nr_hw_queues; i++) {
Jens Axboef14bbe72014-05-27 12:06:53 -06001812 int node = blk_mq_hw_queue_to_node(map, i);
1813
1814 hctxs[i] = set->ops->alloc_hctx(set, i, node);
Jens Axboe320ae512013-10-24 09:20:05 +01001815 if (!hctxs[i])
1816 goto err_hctxs;
1817
Jens Axboee4043dc2014-04-09 10:18:23 -06001818 if (!zalloc_cpumask_var(&hctxs[i]->cpumask, GFP_KERNEL))
1819 goto err_hctxs;
1820
Jens Axboe0d2602c2014-05-13 15:10:52 -06001821 atomic_set(&hctxs[i]->nr_active, 0);
Jens Axboef14bbe72014-05-27 12:06:53 -06001822 hctxs[i]->numa_node = node;
Jens Axboe320ae512013-10-24 09:20:05 +01001823 hctxs[i]->queue_num = i;
1824 }
1825
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001826 q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
Jens Axboe320ae512013-10-24 09:20:05 +01001827 if (!q)
1828 goto err_hctxs;
1829
Ming Lei3d2936f2014-05-27 23:35:14 +08001830 if (percpu_counter_init(&q->mq_usage_counter, 0))
1831 goto err_map;
1832
Jens Axboe320ae512013-10-24 09:20:05 +01001833 setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1834 blk_queue_rq_timeout(q, 30000);
1835
1836 q->nr_queues = nr_cpu_ids;
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001837 q->nr_hw_queues = set->nr_hw_queues;
Jens Axboef14bbe72014-05-27 12:06:53 -06001838 q->mq_map = map;
Jens Axboe320ae512013-10-24 09:20:05 +01001839
1840 q->queue_ctx = ctx;
1841 q->queue_hw_ctx = hctxs;
1842
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001843 q->mq_ops = set->ops;
Jens Axboe94eddfb2013-11-19 09:25:07 -07001844 q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
Jens Axboe320ae512013-10-24 09:20:05 +01001845
Christoph Hellwig1be036e2014-02-07 10:22:39 -08001846 q->sg_reserved_size = INT_MAX;
1847
Christoph Hellwig6fca6a62014-05-28 08:08:02 -06001848 INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1849 INIT_LIST_HEAD(&q->requeue_list);
1850 spin_lock_init(&q->requeue_lock);
1851
Jens Axboe07068d52014-05-22 10:40:51 -06001852 if (q->nr_hw_queues > 1)
1853 blk_queue_make_request(q, blk_mq_make_request);
1854 else
1855 blk_queue_make_request(q, blk_sq_make_request);
1856
Jens Axboe87ee7b12014-04-24 08:51:47 -06001857 blk_queue_rq_timed_out(q, blk_mq_rq_timed_out);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001858 if (set->timeout)
1859 blk_queue_rq_timeout(q, set->timeout);
Jens Axboe320ae512013-10-24 09:20:05 +01001860
Jens Axboeeba71762014-05-20 15:17:27 -06001861 /*
1862 * Do this after blk_queue_make_request() overrides it...
1863 */
1864 q->nr_requests = set->queue_depth;
1865
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001866 if (set->ops->complete)
1867 blk_queue_softirq_done(q, set->ops->complete);
Christoph Hellwig30a91cb2014-02-10 03:24:38 -08001868
Jens Axboe320ae512013-10-24 09:20:05 +01001869 blk_mq_init_flush(q);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001870 blk_mq_init_cpu_queues(q, set->nr_hw_queues);
Jens Axboe320ae512013-10-24 09:20:05 +01001871
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001872 q->flush_rq = kzalloc(round_up(sizeof(struct request) +
1873 set->cmd_size, cache_line_size()),
1874 GFP_KERNEL);
Christoph Hellwig18741982014-02-10 09:29:00 -07001875 if (!q->flush_rq)
Jens Axboe320ae512013-10-24 09:20:05 +01001876 goto err_hw;
1877
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001878 if (blk_mq_init_hw_queues(q, set))
Christoph Hellwig18741982014-02-10 09:29:00 -07001879 goto err_flush_rq;
1880
Jens Axboe320ae512013-10-24 09:20:05 +01001881 mutex_lock(&all_q_mutex);
1882 list_add_tail(&q->all_q_node, &all_q_list);
1883 mutex_unlock(&all_q_mutex);
1884
Jens Axboe0d2602c2014-05-13 15:10:52 -06001885 blk_mq_add_queue_tag_set(set, q);
1886
Jens Axboe484b4062014-05-21 14:01:15 -06001887 blk_mq_map_swqueue(q);
1888
Jens Axboe320ae512013-10-24 09:20:05 +01001889 return q;
Christoph Hellwig18741982014-02-10 09:29:00 -07001890
1891err_flush_rq:
1892 kfree(q->flush_rq);
Jens Axboe320ae512013-10-24 09:20:05 +01001893err_hw:
Jens Axboe320ae512013-10-24 09:20:05 +01001894 blk_cleanup_queue(q);
1895err_hctxs:
Jens Axboef14bbe72014-05-27 12:06:53 -06001896 kfree(map);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001897 for (i = 0; i < set->nr_hw_queues; i++) {
Jens Axboe320ae512013-10-24 09:20:05 +01001898 if (!hctxs[i])
1899 break;
Jens Axboee4043dc2014-04-09 10:18:23 -06001900 free_cpumask_var(hctxs[i]->cpumask);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001901 set->ops->free_hctx(hctxs[i], i);
Jens Axboe320ae512013-10-24 09:20:05 +01001902 }
Jens Axboef14bbe72014-05-27 12:06:53 -06001903err_map:
Jens Axboe320ae512013-10-24 09:20:05 +01001904 kfree(hctxs);
1905err_percpu:
1906 free_percpu(ctx);
1907 return ERR_PTR(-ENOMEM);
1908}
1909EXPORT_SYMBOL(blk_mq_init_queue);
1910
1911void blk_mq_free_queue(struct request_queue *q)
1912{
Ming Lei624dbe42014-05-27 23:35:13 +08001913 struct blk_mq_tag_set *set = q->tag_set;
Jens Axboe320ae512013-10-24 09:20:05 +01001914
Jens Axboe0d2602c2014-05-13 15:10:52 -06001915 blk_mq_del_queue_tag_set(q);
1916
Ming Lei624dbe42014-05-27 23:35:13 +08001917 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1918 blk_mq_free_hw_queues(q, set);
Jens Axboe320ae512013-10-24 09:20:05 +01001919
Ming Lei3d2936f2014-05-27 23:35:14 +08001920 percpu_counter_destroy(&q->mq_usage_counter);
1921
Jens Axboe320ae512013-10-24 09:20:05 +01001922 free_percpu(q->queue_ctx);
1923 kfree(q->queue_hw_ctx);
1924 kfree(q->mq_map);
1925
1926 q->queue_ctx = NULL;
1927 q->queue_hw_ctx = NULL;
1928 q->mq_map = NULL;
1929
1930 mutex_lock(&all_q_mutex);
1931 list_del_init(&q->all_q_node);
1932 mutex_unlock(&all_q_mutex);
1933}
Jens Axboe320ae512013-10-24 09:20:05 +01001934
1935/* Basically redo blk_mq_init_queue with queue frozen */
Paul Gortmakerf618ef72013-11-14 08:26:02 -07001936static void blk_mq_queue_reinit(struct request_queue *q)
Jens Axboe320ae512013-10-24 09:20:05 +01001937{
1938 blk_mq_freeze_queue(q);
1939
1940 blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1941
1942 /*
1943 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1944 * we should change hctx numa_node according to new topology (this
1945 * involves free and re-allocate memory, worthy doing?)
1946 */
1947
1948 blk_mq_map_swqueue(q);
1949
1950 blk_mq_unfreeze_queue(q);
1951}
1952
Paul Gortmakerf618ef72013-11-14 08:26:02 -07001953static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1954 unsigned long action, void *hcpu)
Jens Axboe320ae512013-10-24 09:20:05 +01001955{
1956 struct request_queue *q;
1957
1958 /*
Jens Axboe9fccfed2014-05-08 14:50:19 -06001959 * Before new mappings are established, hotadded cpu might already
1960 * start handling requests. This doesn't break anything as we map
1961 * offline CPUs to first hardware queue. We will re-init the queue
1962 * below to get optimal settings.
Jens Axboe320ae512013-10-24 09:20:05 +01001963 */
1964 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1965 action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1966 return NOTIFY_OK;
1967
1968 mutex_lock(&all_q_mutex);
1969 list_for_each_entry(q, &all_q_list, all_q_node)
1970 blk_mq_queue_reinit(q);
1971 mutex_unlock(&all_q_mutex);
1972 return NOTIFY_OK;
1973}
1974
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001975int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
1976{
1977 int i;
1978
1979 if (!set->nr_hw_queues)
1980 return -EINVAL;
1981 if (!set->queue_depth || set->queue_depth > BLK_MQ_MAX_DEPTH)
1982 return -EINVAL;
1983 if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
1984 return -EINVAL;
1985
1986 if (!set->nr_hw_queues ||
1987 !set->ops->queue_rq || !set->ops->map_queue ||
1988 !set->ops->alloc_hctx || !set->ops->free_hctx)
1989 return -EINVAL;
1990
1991
Ming Lei484790052014-04-19 18:00:17 +08001992 set->tags = kmalloc_node(set->nr_hw_queues *
1993 sizeof(struct blk_mq_tags *),
Christoph Hellwig24d2f902014-04-15 14:14:00 -06001994 GFP_KERNEL, set->numa_node);
1995 if (!set->tags)
1996 goto out;
1997
1998 for (i = 0; i < set->nr_hw_queues; i++) {
1999 set->tags[i] = blk_mq_init_rq_map(set, i);
2000 if (!set->tags[i])
2001 goto out_unwind;
2002 }
2003
Jens Axboe0d2602c2014-05-13 15:10:52 -06002004 mutex_init(&set->tag_list_lock);
2005 INIT_LIST_HEAD(&set->tag_list);
2006
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002007 return 0;
2008
2009out_unwind:
2010 while (--i >= 0)
2011 blk_mq_free_rq_map(set, set->tags[i], i);
2012out:
2013 return -ENOMEM;
2014}
2015EXPORT_SYMBOL(blk_mq_alloc_tag_set);
2016
2017void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
2018{
2019 int i;
2020
Jens Axboe484b4062014-05-21 14:01:15 -06002021 for (i = 0; i < set->nr_hw_queues; i++) {
2022 if (set->tags[i])
2023 blk_mq_free_rq_map(set, set->tags[i], i);
2024 }
2025
Ming Lei981bd182014-04-24 00:07:34 +08002026 kfree(set->tags);
Christoph Hellwig24d2f902014-04-15 14:14:00 -06002027}
2028EXPORT_SYMBOL(blk_mq_free_tag_set);
2029
Jens Axboee3a2b3f2014-05-20 11:49:02 -06002030int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2031{
2032 struct blk_mq_tag_set *set = q->tag_set;
2033 struct blk_mq_hw_ctx *hctx;
2034 int i, ret;
2035
2036 if (!set || nr > set->queue_depth)
2037 return -EINVAL;
2038
2039 ret = 0;
2040 queue_for_each_hw_ctx(q, hctx, i) {
2041 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2042 if (ret)
2043 break;
2044 }
2045
2046 if (!ret)
2047 q->nr_requests = nr;
2048
2049 return ret;
2050}
2051
Jens Axboe676141e2014-03-20 13:29:18 -06002052void blk_mq_disable_hotplug(void)
2053{
2054 mutex_lock(&all_q_mutex);
2055}
2056
2057void blk_mq_enable_hotplug(void)
2058{
2059 mutex_unlock(&all_q_mutex);
2060}
2061
Jens Axboe320ae512013-10-24 09:20:05 +01002062static int __init blk_mq_init(void)
2063{
Jens Axboe320ae512013-10-24 09:20:05 +01002064 blk_mq_cpu_init();
2065
2066 /* Must be called after percpu_counter_hotcpu_callback() */
2067 hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
2068
2069 return 0;
2070}
2071subsys_initcall(blk_mq_init);