blob: a1d585c351d663f3cc64f0f874b0a38562797dc7 [file] [log] [blame]
Arjan van de Ven22a9d642009-01-07 08:45:46 -08001/*
2 * async.c: Asynchronous function calls for boot performance
3 *
4 * (C) Copyright 2009 Intel Corporation
5 * Author: Arjan van de Ven <arjan@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
10 * of the License.
11 */
12
13
14/*
15
16Goals and Theory of Operation
17
18The primary goal of this feature is to reduce the kernel boot time,
19by doing various independent hardware delays and discovery operations
20decoupled and not strictly serialized.
21
22More specifically, the asynchronous function call concept allows
23certain operations (primarily during system boot) to happen
24asynchronously, out of order, while these operations still
25have their externally visible parts happen sequentially and in-order.
26(not unlike how out-of-order CPUs retire their instructions in order)
27
28Key to the asynchronous function call implementation is the concept of
29a "sequence cookie" (which, although it has an abstracted type, can be
30thought of as a monotonically incrementing number).
31
32The async core will assign each scheduled event such a sequence cookie and
33pass this to the called functions.
34
35The asynchronously called function should before doing a globally visible
36operation, such as registering device numbers, call the
37async_synchronize_cookie() function and pass in its own cookie. The
38async_synchronize_cookie() function will make sure that all asynchronous
39operations that were scheduled prior to the operation corresponding with the
40cookie have completed.
41
42Subsystem/driver initialization code that scheduled asynchronous probe
43functions, but which shares global resources with other drivers/subsystems
44that do not use the asynchronous call feature, need to do a full
45synchronization with the async_synchronize_full() function, before returning
46from their init function. This is to maintain strict ordering between the
47asynchronous and synchronous parts of the kernel.
48
49*/
50
51#include <linux/async.h>
Paul McQuade84c15022011-05-31 20:51:55 +010052#include <linux/atomic.h>
53#include <linux/ktime.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040054#include <linux/export.h>
Arjan van de Ven22a9d642009-01-07 08:45:46 -080055#include <linux/wait.h>
56#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090057#include <linux/slab.h>
Tejun Heo083b8042010-07-02 10:03:52 +020058#include <linux/workqueue.h>
Arjan van de Ven22a9d642009-01-07 08:45:46 -080059
60static async_cookie_t next_cookie = 1;
61
Arjan van de Ven22a9d642009-01-07 08:45:46 -080062#define MAX_WORK 32768
63
64static LIST_HEAD(async_pending);
Dan Williams2955b472012-07-09 19:33:25 -070065static ASYNC_DOMAIN(async_running);
Dan Williamsa4683482012-07-09 19:33:30 -070066static LIST_HEAD(async_domains);
Arjan van de Ven22a9d642009-01-07 08:45:46 -080067static DEFINE_SPINLOCK(async_lock);
Dan Williamsa4683482012-07-09 19:33:30 -070068static DEFINE_MUTEX(async_register_mutex);
Arjan van de Ven22a9d642009-01-07 08:45:46 -080069
70struct async_entry {
Tejun Heo083b8042010-07-02 10:03:52 +020071 struct list_head list;
72 struct work_struct work;
73 async_cookie_t cookie;
74 async_func_ptr *func;
75 void *data;
Dan Williams2955b472012-07-09 19:33:25 -070076 struct async_domain *running;
Arjan van de Ven22a9d642009-01-07 08:45:46 -080077};
78
79static DECLARE_WAIT_QUEUE_HEAD(async_done);
Arjan van de Ven22a9d642009-01-07 08:45:46 -080080
81static atomic_t entry_count;
Arjan van de Ven22a9d642009-01-07 08:45:46 -080082
Arjan van de Ven22a9d642009-01-07 08:45:46 -080083
84/*
85 * MUST be called with the lock held!
86 */
Dan Williams2955b472012-07-09 19:33:25 -070087static async_cookie_t __lowest_in_progress(struct async_domain *running)
Arjan van de Ven22a9d642009-01-07 08:45:46 -080088{
89 struct async_entry *entry;
James Bottomleyd5a877e2009-05-24 13:03:43 -070090
Dan Williams2955b472012-07-09 19:33:25 -070091 if (!list_empty(&running->domain)) {
92 entry = list_first_entry(&running->domain, typeof(*entry), list);
Linus Torvalds3af968e2009-06-08 12:31:53 -070093 return entry->cookie;
Arjan van de Ven22a9d642009-01-07 08:45:46 -080094 }
95
Linus Torvalds3af968e2009-06-08 12:31:53 -070096 list_for_each_entry(entry, &async_pending, list)
97 if (entry->running == running)
98 return entry->cookie;
James Bottomleyd5a877e2009-05-24 13:03:43 -070099
Linus Torvalds3af968e2009-06-08 12:31:53 -0700100 return next_cookie; /* "infinity" value */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800101}
Arjan van de Ven37a76bd2009-01-11 15:35:01 +0000102
Dan Williams2955b472012-07-09 19:33:25 -0700103static async_cookie_t lowest_in_progress(struct async_domain *running)
Arjan van de Ven37a76bd2009-01-11 15:35:01 +0000104{
105 unsigned long flags;
106 async_cookie_t ret;
107
108 spin_lock_irqsave(&async_lock, flags);
109 ret = __lowest_in_progress(running);
110 spin_unlock_irqrestore(&async_lock, flags);
111 return ret;
112}
Tejun Heo083b8042010-07-02 10:03:52 +0200113
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800114/*
115 * pick the first pending entry and run it
116 */
Tejun Heo083b8042010-07-02 10:03:52 +0200117static void async_run_entry_fn(struct work_struct *work)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800118{
Tejun Heo083b8042010-07-02 10:03:52 +0200119 struct async_entry *entry =
120 container_of(work, struct async_entry, work);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800121 unsigned long flags;
Vitaliy Ivanov124ff4e2011-07-07 14:10:40 +0300122 ktime_t uninitialized_var(calltime), delta, rettime;
Dan Williams2955b472012-07-09 19:33:25 -0700123 struct async_domain *running = entry->running;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800124
Tejun Heo083b8042010-07-02 10:03:52 +0200125 /* 1) move self to the running queue */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800126 spin_lock_irqsave(&async_lock, flags);
Dan Williams2955b472012-07-09 19:33:25 -0700127 list_move_tail(&entry->list, &running->domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800128 spin_unlock_irqrestore(&async_lock, flags);
129
Tejun Heo083b8042010-07-02 10:03:52 +0200130 /* 2) run (and print duration) */
Arjan van de Venad160d22009-01-07 09:28:53 -0800131 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Paul McQuade84c15022011-05-31 20:51:55 +0100132 printk(KERN_DEBUG "calling %lli_%pF @ %i\n",
133 (long long)entry->cookie,
Andrew Morton58763a22009-02-04 15:11:58 -0800134 entry->func, task_pid_nr(current));
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800135 calltime = ktime_get();
136 }
137 entry->func(entry->data, entry->cookie);
Arjan van de Venad160d22009-01-07 09:28:53 -0800138 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800139 rettime = ktime_get();
140 delta = ktime_sub(rettime, calltime);
Paul McQuade84c15022011-05-31 20:51:55 +0100141 printk(KERN_DEBUG "initcall %lli_%pF returned 0 after %lld usecs\n",
Andrew Morton58763a22009-02-04 15:11:58 -0800142 (long long)entry->cookie,
143 entry->func,
144 (long long)ktime_to_ns(delta) >> 10);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800145 }
146
Tejun Heo083b8042010-07-02 10:03:52 +0200147 /* 3) remove self from the running queue */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800148 spin_lock_irqsave(&async_lock, flags);
149 list_del(&entry->list);
Dan Williamsa4683482012-07-09 19:33:30 -0700150 if (running->registered && --running->count == 0)
151 list_del_init(&running->node);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800152
Tejun Heo083b8042010-07-02 10:03:52 +0200153 /* 4) free the entry */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800154 kfree(entry);
155 atomic_dec(&entry_count);
156
157 spin_unlock_irqrestore(&async_lock, flags);
158
Tejun Heo083b8042010-07-02 10:03:52 +0200159 /* 5) wake up any waiters */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800160 wake_up(&async_done);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800161}
162
Dan Williams2955b472012-07-09 19:33:25 -0700163static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct async_domain *running)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800164{
165 struct async_entry *entry;
166 unsigned long flags;
167 async_cookie_t newcookie;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800168
169 /* allow irq-off callers */
170 entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
171
172 /*
173 * If we're out of memory or if there's too much work
174 * pending already, we execute synchronously.
175 */
Tejun Heo083b8042010-07-02 10:03:52 +0200176 if (!entry || atomic_read(&entry_count) > MAX_WORK) {
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800177 kfree(entry);
178 spin_lock_irqsave(&async_lock, flags);
179 newcookie = next_cookie++;
180 spin_unlock_irqrestore(&async_lock, flags);
181
182 /* low on memory.. run synchronously */
183 ptr(data, newcookie);
184 return newcookie;
185 }
Tejun Heo083b8042010-07-02 10:03:52 +0200186 INIT_WORK(&entry->work, async_run_entry_fn);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800187 entry->func = ptr;
188 entry->data = data;
189 entry->running = running;
190
191 spin_lock_irqsave(&async_lock, flags);
192 newcookie = entry->cookie = next_cookie++;
193 list_add_tail(&entry->list, &async_pending);
Dan Williamsa4683482012-07-09 19:33:30 -0700194 if (running->registered && running->count++ == 0)
195 list_add_tail(&running->node, &async_domains);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800196 atomic_inc(&entry_count);
197 spin_unlock_irqrestore(&async_lock, flags);
Tejun Heo083b8042010-07-02 10:03:52 +0200198
Tejun Heo774a1222013-01-15 18:52:51 -0800199 /* mark that this task has queued an async job, used by module init */
200 current->flags |= PF_USED_ASYNC;
201
Tejun Heo083b8042010-07-02 10:03:52 +0200202 /* schedule for execution */
203 queue_work(system_unbound_wq, &entry->work);
204
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800205 return newcookie;
206}
207
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100208/**
209 * async_schedule - schedule a function for asynchronous execution
210 * @ptr: function to execute asynchronously
211 * @data: data pointer to pass to the function
212 *
213 * Returns an async_cookie_t that may be used for checkpointing later.
214 * Note: This function may be called from atomic or non-atomic contexts.
215 */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800216async_cookie_t async_schedule(async_func_ptr *ptr, void *data)
217{
Cornelia Huck7a89bbc2009-01-19 13:45:28 +0100218 return __async_schedule(ptr, data, &async_running);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800219}
220EXPORT_SYMBOL_GPL(async_schedule);
221
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100222/**
Cornelia Huck766ccb92009-01-20 15:31:31 +0100223 * async_schedule_domain - schedule a function for asynchronous execution within a certain domain
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100224 * @ptr: function to execute asynchronously
225 * @data: data pointer to pass to the function
Cornelia Huck766ccb92009-01-20 15:31:31 +0100226 * @running: running list for the domain
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100227 *
228 * Returns an async_cookie_t that may be used for checkpointing later.
Cornelia Huck766ccb92009-01-20 15:31:31 +0100229 * @running may be used in the async_synchronize_*_domain() functions
230 * to wait within a certain synchronization domain rather than globally.
231 * A synchronization domain is specified via the running queue @running to use.
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100232 * Note: This function may be called from atomic or non-atomic contexts.
233 */
Cornelia Huck766ccb92009-01-20 15:31:31 +0100234async_cookie_t async_schedule_domain(async_func_ptr *ptr, void *data,
Dan Williams2955b472012-07-09 19:33:25 -0700235 struct async_domain *running)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800236{
237 return __async_schedule(ptr, data, running);
238}
Cornelia Huck766ccb92009-01-20 15:31:31 +0100239EXPORT_SYMBOL_GPL(async_schedule_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800240
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100241/**
242 * async_synchronize_full - synchronize all asynchronous function calls
243 *
244 * This function waits until all asynchronous function calls have been done.
245 */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800246void async_synchronize_full(void)
247{
Dan Williamsa4683482012-07-09 19:33:30 -0700248 mutex_lock(&async_register_mutex);
Arjan van de Ven33b04b92009-01-08 12:35:11 -0800249 do {
Dan Williamsa4683482012-07-09 19:33:30 -0700250 struct async_domain *domain = NULL;
251
252 spin_lock_irq(&async_lock);
253 if (!list_empty(&async_domains))
254 domain = list_first_entry(&async_domains, typeof(*domain), node);
255 spin_unlock_irq(&async_lock);
256
257 async_synchronize_cookie_domain(next_cookie, domain);
258 } while (!list_empty(&async_domains));
259 mutex_unlock(&async_register_mutex);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800260}
261EXPORT_SYMBOL_GPL(async_synchronize_full);
262
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100263/**
Dan Williamsa4683482012-07-09 19:33:30 -0700264 * async_unregister_domain - ensure no more anonymous waiters on this domain
265 * @domain: idle domain to flush out of any async_synchronize_full instances
266 *
267 * async_synchronize_{cookie|full}_domain() are not flushed since callers
268 * of these routines should know the lifetime of @domain
269 *
270 * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
271 */
272void async_unregister_domain(struct async_domain *domain)
273{
274 mutex_lock(&async_register_mutex);
275 spin_lock_irq(&async_lock);
276 WARN_ON(!domain->registered || !list_empty(&domain->node) ||
277 !list_empty(&domain->domain));
278 domain->registered = 0;
279 spin_unlock_irq(&async_lock);
280 mutex_unlock(&async_register_mutex);
281}
282EXPORT_SYMBOL_GPL(async_unregister_domain);
283
284/**
Cornelia Huck766ccb92009-01-20 15:31:31 +0100285 * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
Dan Williams2955b472012-07-09 19:33:25 -0700286 * @domain: running list to synchronize on
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100287 *
Cornelia Huck766ccb92009-01-20 15:31:31 +0100288 * This function waits until all asynchronous function calls for the
Dan Williams2955b472012-07-09 19:33:25 -0700289 * synchronization domain specified by the running list @domain have been done.
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100290 */
Dan Williams2955b472012-07-09 19:33:25 -0700291void async_synchronize_full_domain(struct async_domain *domain)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800292{
Dan Williams2955b472012-07-09 19:33:25 -0700293 async_synchronize_cookie_domain(next_cookie, domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800294}
Cornelia Huck766ccb92009-01-20 15:31:31 +0100295EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800296
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100297/**
Cornelia Huck766ccb92009-01-20 15:31:31 +0100298 * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100299 * @cookie: async_cookie_t to use as checkpoint
300 * @running: running list to synchronize on
301 *
Cornelia Huck766ccb92009-01-20 15:31:31 +0100302 * This function waits until all asynchronous function calls for the
Dan Williams2955b472012-07-09 19:33:25 -0700303 * synchronization domain specified by running list @running submitted
Cornelia Huck766ccb92009-01-20 15:31:31 +0100304 * prior to @cookie have been done.
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100305 */
Dan Williams2955b472012-07-09 19:33:25 -0700306void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *running)
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800307{
Vitaliy Ivanov124ff4e2011-07-07 14:10:40 +0300308 ktime_t uninitialized_var(starttime), delta, endtime;
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800309
Dan Williamsa4683482012-07-09 19:33:30 -0700310 if (!running)
311 return;
312
Arjan van de Venad160d22009-01-07 09:28:53 -0800313 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Paul McQuade84c15022011-05-31 20:51:55 +0100314 printk(KERN_DEBUG "async_waiting @ %i\n", task_pid_nr(current));
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800315 starttime = ktime_get();
316 }
317
Arjan van de Ven37a76bd2009-01-11 15:35:01 +0000318 wait_event(async_done, lowest_in_progress(running) >= cookie);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800319
Arjan van de Venad160d22009-01-07 09:28:53 -0800320 if (initcall_debug && system_state == SYSTEM_BOOTING) {
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800321 endtime = ktime_get();
322 delta = ktime_sub(endtime, starttime);
323
Paul McQuade84c15022011-05-31 20:51:55 +0100324 printk(KERN_DEBUG "async_continuing @ %i after %lli usec\n",
Andrew Morton58763a22009-02-04 15:11:58 -0800325 task_pid_nr(current),
326 (long long)ktime_to_ns(delta) >> 10);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800327 }
328}
Cornelia Huck766ccb92009-01-20 15:31:31 +0100329EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800330
Cornelia Huckf30d5b32009-01-19 13:45:33 +0100331/**
332 * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
333 * @cookie: async_cookie_t to use as checkpoint
334 *
335 * This function waits until all asynchronous function calls prior to @cookie
336 * have been done.
337 */
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800338void async_synchronize_cookie(async_cookie_t cookie)
339{
Cornelia Huck766ccb92009-01-20 15:31:31 +0100340 async_synchronize_cookie_domain(cookie, &async_running);
Arjan van de Ven22a9d642009-01-07 08:45:46 -0800341}
342EXPORT_SYMBOL_GPL(async_synchronize_cookie);