Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) |
| 3 | * |
| 4 | * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
| 5 | * |
| 6 | * Interactivity improvements by Mike Galbraith |
| 7 | * (C) 2007 Mike Galbraith <efault@gmx.de> |
| 8 | * |
| 9 | * Various enhancements by Dmitry Adamushko. |
| 10 | * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> |
| 11 | * |
| 12 | * Group scheduling enhancements by Srivatsa Vaddagiri |
| 13 | * Copyright IBM Corporation, 2007 |
| 14 | * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> |
| 15 | * |
| 16 | * Scaled math optimizations by Thomas Gleixner |
| 17 | * Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de> |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 18 | * |
| 19 | * Adaptive scheduling granularity, math enhancements by Peter Zijlstra |
| 20 | * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 21 | */ |
| 22 | |
| 23 | /* |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 24 | * Targeted preemption latency for CPU-bound tasks: |
| 25 | * (default: 20ms, units: nanoseconds) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 26 | * |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 27 | * NOTE: this latency value is not the same as the concept of |
| 28 | * 'timeslice length' - timeslices in CFS are of variable length. |
| 29 | * (to see the precise effective timeslice length of your workload, |
| 30 | * run vmstat and monitor the context-switches field) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 31 | * |
| 32 | * On SMP systems the value of this is multiplied by the log2 of the |
| 33 | * number of CPUs. (i.e. factor 2x on 2-way systems, 3x on 4-way |
| 34 | * systems, 4x on 8-way systems, 5x on 16-way systems, etc.) |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 35 | * Targeted preemption latency for CPU-bound tasks: |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 36 | */ |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 37 | unsigned int sysctl_sched_latency __read_mostly = 20000000ULL; |
| 38 | |
| 39 | /* |
| 40 | * Minimal preemption granularity for CPU-bound tasks: |
| 41 | * (default: 2 msec, units: nanoseconds) |
| 42 | */ |
Ingo Molnar | 172ac3d | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 43 | unsigned int sysctl_sched_min_granularity __read_mostly = 2000000ULL; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 44 | |
| 45 | /* |
| 46 | * SCHED_BATCH wake-up granularity. |
Ingo Molnar | 71fd371 | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 47 | * (default: 25 msec, units: nanoseconds) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 48 | * |
| 49 | * This option delays the preemption effects of decoupled workloads |
| 50 | * and reduces their over-scheduling. Synchronous workloads will still |
| 51 | * have immediate wakeup/sleep latencies. |
| 52 | */ |
Ingo Molnar | 71fd371 | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 53 | unsigned int sysctl_sched_batch_wakeup_granularity __read_mostly = 25000000UL; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 54 | |
| 55 | /* |
| 56 | * SCHED_OTHER wake-up granularity. |
| 57 | * (default: 1 msec, units: nanoseconds) |
| 58 | * |
| 59 | * This option delays the preemption effects of decoupled workloads |
| 60 | * and reduces their over-scheduling. Synchronous workloads will still |
| 61 | * have immediate wakeup/sleep latencies. |
| 62 | */ |
Ingo Molnar | 71fd371 | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 63 | unsigned int sysctl_sched_wakeup_granularity __read_mostly = 1000000UL; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 64 | |
| 65 | unsigned int sysctl_sched_stat_granularity __read_mostly; |
| 66 | |
| 67 | /* |
Ingo Molnar | 71fd371 | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 68 | * Initialized in sched_init_granularity() [to 5 times the base granularity]: |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 69 | */ |
| 70 | unsigned int sysctl_sched_runtime_limit __read_mostly; |
| 71 | |
| 72 | /* |
| 73 | * Debugging: various feature bits |
| 74 | */ |
| 75 | enum { |
| 76 | SCHED_FEAT_FAIR_SLEEPERS = 1, |
| 77 | SCHED_FEAT_SLEEPER_AVG = 2, |
| 78 | SCHED_FEAT_SLEEPER_LOAD_AVG = 4, |
| 79 | SCHED_FEAT_PRECISE_CPU_LOAD = 8, |
| 80 | SCHED_FEAT_START_DEBIT = 16, |
| 81 | SCHED_FEAT_SKIP_INITIAL = 32, |
| 82 | }; |
| 83 | |
| 84 | unsigned int sysctl_sched_features __read_mostly = |
| 85 | SCHED_FEAT_FAIR_SLEEPERS *1 | |
Ingo Molnar | 5d2b3d3 | 2007-08-12 18:08:19 +0200 | [diff] [blame] | 86 | SCHED_FEAT_SLEEPER_AVG *0 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 87 | SCHED_FEAT_SLEEPER_LOAD_AVG *1 | |
| 88 | SCHED_FEAT_PRECISE_CPU_LOAD *1 | |
| 89 | SCHED_FEAT_START_DEBIT *1 | |
| 90 | SCHED_FEAT_SKIP_INITIAL *0; |
| 91 | |
| 92 | extern struct sched_class fair_sched_class; |
| 93 | |
| 94 | /************************************************************** |
| 95 | * CFS operations on generic schedulable entities: |
| 96 | */ |
| 97 | |
| 98 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 99 | |
| 100 | /* cpu runqueue to which this cfs_rq is attached */ |
| 101 | static inline struct rq *rq_of(struct cfs_rq *cfs_rq) |
| 102 | { |
| 103 | return cfs_rq->rq; |
| 104 | } |
| 105 | |
| 106 | /* currently running entity (if any) on this cfs_rq */ |
| 107 | static inline struct sched_entity *cfs_rq_curr(struct cfs_rq *cfs_rq) |
| 108 | { |
| 109 | return cfs_rq->curr; |
| 110 | } |
| 111 | |
| 112 | /* An entity is a task if it doesn't "own" a runqueue */ |
| 113 | #define entity_is_task(se) (!se->my_q) |
| 114 | |
| 115 | static inline void |
| 116 | set_cfs_rq_curr(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 117 | { |
| 118 | cfs_rq->curr = se; |
| 119 | } |
| 120 | |
| 121 | #else /* CONFIG_FAIR_GROUP_SCHED */ |
| 122 | |
| 123 | static inline struct rq *rq_of(struct cfs_rq *cfs_rq) |
| 124 | { |
| 125 | return container_of(cfs_rq, struct rq, cfs); |
| 126 | } |
| 127 | |
| 128 | static inline struct sched_entity *cfs_rq_curr(struct cfs_rq *cfs_rq) |
| 129 | { |
| 130 | struct rq *rq = rq_of(cfs_rq); |
| 131 | |
| 132 | if (unlikely(rq->curr->sched_class != &fair_sched_class)) |
| 133 | return NULL; |
| 134 | |
| 135 | return &rq->curr->se; |
| 136 | } |
| 137 | |
| 138 | #define entity_is_task(se) 1 |
| 139 | |
| 140 | static inline void |
| 141 | set_cfs_rq_curr(struct cfs_rq *cfs_rq, struct sched_entity *se) { } |
| 142 | |
| 143 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
| 144 | |
| 145 | static inline struct task_struct *task_of(struct sched_entity *se) |
| 146 | { |
| 147 | return container_of(se, struct task_struct, se); |
| 148 | } |
| 149 | |
| 150 | |
| 151 | /************************************************************** |
| 152 | * Scheduling class tree data structure manipulation methods: |
| 153 | */ |
| 154 | |
| 155 | /* |
| 156 | * Enqueue an entity into the rb-tree: |
| 157 | */ |
| 158 | static inline void |
| 159 | __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 160 | { |
| 161 | struct rb_node **link = &cfs_rq->tasks_timeline.rb_node; |
| 162 | struct rb_node *parent = NULL; |
| 163 | struct sched_entity *entry; |
| 164 | s64 key = se->fair_key; |
| 165 | int leftmost = 1; |
| 166 | |
| 167 | /* |
| 168 | * Find the right place in the rbtree: |
| 169 | */ |
| 170 | while (*link) { |
| 171 | parent = *link; |
| 172 | entry = rb_entry(parent, struct sched_entity, run_node); |
| 173 | /* |
| 174 | * We dont care about collisions. Nodes with |
| 175 | * the same key stay together. |
| 176 | */ |
| 177 | if (key - entry->fair_key < 0) { |
| 178 | link = &parent->rb_left; |
| 179 | } else { |
| 180 | link = &parent->rb_right; |
| 181 | leftmost = 0; |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | /* |
| 186 | * Maintain a cache of leftmost tree entries (it is frequently |
| 187 | * used): |
| 188 | */ |
| 189 | if (leftmost) |
| 190 | cfs_rq->rb_leftmost = &se->run_node; |
| 191 | |
| 192 | rb_link_node(&se->run_node, parent, link); |
| 193 | rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline); |
| 194 | update_load_add(&cfs_rq->load, se->load.weight); |
| 195 | cfs_rq->nr_running++; |
| 196 | se->on_rq = 1; |
| 197 | } |
| 198 | |
| 199 | static inline void |
| 200 | __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 201 | { |
| 202 | if (cfs_rq->rb_leftmost == &se->run_node) |
| 203 | cfs_rq->rb_leftmost = rb_next(&se->run_node); |
| 204 | rb_erase(&se->run_node, &cfs_rq->tasks_timeline); |
| 205 | update_load_sub(&cfs_rq->load, se->load.weight); |
| 206 | cfs_rq->nr_running--; |
| 207 | se->on_rq = 0; |
| 208 | } |
| 209 | |
| 210 | static inline struct rb_node *first_fair(struct cfs_rq *cfs_rq) |
| 211 | { |
| 212 | return cfs_rq->rb_leftmost; |
| 213 | } |
| 214 | |
| 215 | static struct sched_entity *__pick_next_entity(struct cfs_rq *cfs_rq) |
| 216 | { |
| 217 | return rb_entry(first_fair(cfs_rq), struct sched_entity, run_node); |
| 218 | } |
| 219 | |
| 220 | /************************************************************** |
| 221 | * Scheduling class statistics methods: |
| 222 | */ |
| 223 | |
| 224 | /* |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 225 | * Calculate the preemption granularity needed to schedule every |
| 226 | * runnable task once per sysctl_sched_latency amount of time. |
| 227 | * (down to a sensible low limit on granularity) |
| 228 | * |
| 229 | * For example, if there are 2 tasks running and latency is 10 msecs, |
| 230 | * we switch tasks every 5 msecs. If we have 3 tasks running, we have |
| 231 | * to switch tasks every 3.33 msecs to get a 10 msecs observed latency |
| 232 | * for each task. We do finer and finer scheduling up to until we |
| 233 | * reach the minimum granularity value. |
| 234 | * |
| 235 | * To achieve this we use the following dynamic-granularity rule: |
| 236 | * |
| 237 | * gran = lat/nr - lat/nr/nr |
| 238 | * |
| 239 | * This comes out of the following equations: |
| 240 | * |
| 241 | * kA1 + gran = kB1 |
| 242 | * kB2 + gran = kA2 |
| 243 | * kA2 = kA1 |
| 244 | * kB2 = kB1 - d + d/nr |
| 245 | * lat = d * nr |
| 246 | * |
| 247 | * Where 'k' is key, 'A' is task A (waiting), 'B' is task B (running), |
| 248 | * '1' is start of time, '2' is end of time, 'd' is delay between |
| 249 | * 1 and 2 (during which task B was running), 'nr' is number of tasks |
| 250 | * running, 'lat' is the the period of each task. ('lat' is the |
| 251 | * sched_latency that we aim for.) |
| 252 | */ |
| 253 | static long |
| 254 | sched_granularity(struct cfs_rq *cfs_rq) |
| 255 | { |
| 256 | unsigned int gran = sysctl_sched_latency; |
| 257 | unsigned int nr = cfs_rq->nr_running; |
| 258 | |
| 259 | if (nr > 1) { |
| 260 | gran = gran/nr - gran/nr/nr; |
Ingo Molnar | 172ac3d | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 261 | gran = max(gran, sysctl_sched_min_granularity); |
Peter Zijlstra | 2180508 | 2007-08-25 18:41:53 +0200 | [diff] [blame] | 262 | } |
| 263 | |
| 264 | return gran; |
| 265 | } |
| 266 | |
| 267 | /* |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 268 | * We rescale the rescheduling granularity of tasks according to their |
| 269 | * nice level, but only linearly, not exponentially: |
| 270 | */ |
| 271 | static long |
| 272 | niced_granularity(struct sched_entity *curr, unsigned long granularity) |
| 273 | { |
| 274 | u64 tmp; |
| 275 | |
Ingo Molnar | 7cff8cf | 2007-08-09 11:16:52 +0200 | [diff] [blame] | 276 | if (likely(curr->load.weight == NICE_0_LOAD)) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 277 | return granularity; |
| 278 | /* |
Ingo Molnar | 7cff8cf | 2007-08-09 11:16:52 +0200 | [diff] [blame] | 279 | * Positive nice levels get the same granularity as nice-0: |
| 280 | */ |
| 281 | if (likely(curr->load.weight < NICE_0_LOAD)) { |
| 282 | tmp = curr->load.weight * (u64)granularity; |
| 283 | return (long) (tmp >> NICE_0_SHIFT); |
| 284 | } |
| 285 | /* |
| 286 | * Negative nice level tasks get linearly finer |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 287 | * granularity: |
| 288 | */ |
Ingo Molnar | 7cff8cf | 2007-08-09 11:16:52 +0200 | [diff] [blame] | 289 | tmp = curr->load.inv_weight * (u64)granularity; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 290 | |
| 291 | /* |
| 292 | * It will always fit into 'long': |
| 293 | */ |
Ingo Molnar | 7cff8cf | 2007-08-09 11:16:52 +0200 | [diff] [blame] | 294 | return (long) (tmp >> WMULT_SHIFT); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | static inline void |
| 298 | limit_wait_runtime(struct cfs_rq *cfs_rq, struct sched_entity *se) |
| 299 | { |
| 300 | long limit = sysctl_sched_runtime_limit; |
| 301 | |
| 302 | /* |
| 303 | * Niced tasks have the same history dynamic range as |
| 304 | * non-niced tasks: |
| 305 | */ |
| 306 | if (unlikely(se->wait_runtime > limit)) { |
| 307 | se->wait_runtime = limit; |
| 308 | schedstat_inc(se, wait_runtime_overruns); |
| 309 | schedstat_inc(cfs_rq, wait_runtime_overruns); |
| 310 | } |
| 311 | if (unlikely(se->wait_runtime < -limit)) { |
| 312 | se->wait_runtime = -limit; |
| 313 | schedstat_inc(se, wait_runtime_underruns); |
| 314 | schedstat_inc(cfs_rq, wait_runtime_underruns); |
| 315 | } |
| 316 | } |
| 317 | |
| 318 | static inline void |
| 319 | __add_wait_runtime(struct cfs_rq *cfs_rq, struct sched_entity *se, long delta) |
| 320 | { |
| 321 | se->wait_runtime += delta; |
| 322 | schedstat_add(se, sum_wait_runtime, delta); |
| 323 | limit_wait_runtime(cfs_rq, se); |
| 324 | } |
| 325 | |
| 326 | static void |
| 327 | add_wait_runtime(struct cfs_rq *cfs_rq, struct sched_entity *se, long delta) |
| 328 | { |
| 329 | schedstat_add(cfs_rq, wait_runtime, -se->wait_runtime); |
| 330 | __add_wait_runtime(cfs_rq, se, delta); |
| 331 | schedstat_add(cfs_rq, wait_runtime, se->wait_runtime); |
| 332 | } |
| 333 | |
| 334 | /* |
| 335 | * Update the current task's runtime statistics. Skip current tasks that |
| 336 | * are not in our scheduling class. |
| 337 | */ |
| 338 | static inline void |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 339 | __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 340 | { |
Ingo Molnar | c5dcfe7 | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 341 | unsigned long delta, delta_exec, delta_fair, delta_mine; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 342 | struct load_weight *lw = &cfs_rq->load; |
| 343 | unsigned long load = lw->weight; |
| 344 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 345 | delta_exec = curr->delta_exec; |
Ingo Molnar | 8179ca23 | 2007-08-02 17:41:40 +0200 | [diff] [blame] | 346 | schedstat_set(curr->exec_max, max((u64)delta_exec, curr->exec_max)); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 347 | |
| 348 | curr->sum_exec_runtime += delta_exec; |
| 349 | cfs_rq->exec_clock += delta_exec; |
| 350 | |
Ingo Molnar | fd8bb43 | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 351 | if (unlikely(!load)) |
| 352 | return; |
| 353 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 354 | delta_fair = calc_delta_fair(delta_exec, lw); |
| 355 | delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw); |
| 356 | |
Mike Galbraith | 5f01d51 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 357 | if (cfs_rq->sleeper_bonus > sysctl_sched_min_granularity) { |
Peter Zijlstra | ea0aa3b | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 358 | delta = min((u64)delta_mine, cfs_rq->sleeper_bonus); |
Ingo Molnar | b2133c8 | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 359 | delta = min(delta, (unsigned long)( |
| 360 | (long)sysctl_sched_runtime_limit - curr->wait_runtime)); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 361 | cfs_rq->sleeper_bonus -= delta; |
| 362 | delta_mine -= delta; |
| 363 | } |
| 364 | |
| 365 | cfs_rq->fair_clock += delta_fair; |
| 366 | /* |
| 367 | * We executed delta_exec amount of time on the CPU, |
| 368 | * but we were only entitled to delta_mine amount of |
| 369 | * time during that period (if nr_running == 1 then |
| 370 | * the two values are equal) |
| 371 | * [Note: delta_mine - delta_exec is negative]: |
| 372 | */ |
| 373 | add_wait_runtime(cfs_rq, curr, delta_mine - delta_exec); |
| 374 | } |
| 375 | |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 376 | static void update_curr(struct cfs_rq *cfs_rq) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 377 | { |
| 378 | struct sched_entity *curr = cfs_rq_curr(cfs_rq); |
| 379 | unsigned long delta_exec; |
| 380 | |
| 381 | if (unlikely(!curr)) |
| 382 | return; |
| 383 | |
| 384 | /* |
| 385 | * Get the amount of time the current task was running |
| 386 | * since the last time we changed load (this cannot |
| 387 | * overflow on 32 bits): |
| 388 | */ |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 389 | delta_exec = (unsigned long)(rq_of(cfs_rq)->clock - curr->exec_start); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 390 | |
| 391 | curr->delta_exec += delta_exec; |
| 392 | |
| 393 | if (unlikely(curr->delta_exec > sysctl_sched_stat_granularity)) { |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 394 | __update_curr(cfs_rq, curr); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 395 | curr->delta_exec = 0; |
| 396 | } |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 397 | curr->exec_start = rq_of(cfs_rq)->clock; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 398 | } |
| 399 | |
| 400 | static inline void |
Ingo Molnar | 5870db5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 401 | update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 402 | { |
| 403 | se->wait_start_fair = cfs_rq->fair_clock; |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 404 | schedstat_set(se->wait_start, rq_of(cfs_rq)->clock); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 405 | } |
| 406 | |
| 407 | /* |
| 408 | * We calculate fair deltas here, so protect against the random effects |
| 409 | * of a multiplication overflow by capping it to the runtime limit: |
| 410 | */ |
| 411 | #if BITS_PER_LONG == 32 |
| 412 | static inline unsigned long |
| 413 | calc_weighted(unsigned long delta, unsigned long weight, int shift) |
| 414 | { |
| 415 | u64 tmp = (u64)delta * weight >> shift; |
| 416 | |
| 417 | if (unlikely(tmp > sysctl_sched_runtime_limit*2)) |
| 418 | return sysctl_sched_runtime_limit*2; |
| 419 | return tmp; |
| 420 | } |
| 421 | #else |
| 422 | static inline unsigned long |
| 423 | calc_weighted(unsigned long delta, unsigned long weight, int shift) |
| 424 | { |
| 425 | return delta * weight >> shift; |
| 426 | } |
| 427 | #endif |
| 428 | |
| 429 | /* |
| 430 | * Task is being enqueued - update stats: |
| 431 | */ |
Ingo Molnar | d2417e5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 432 | static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 433 | { |
| 434 | s64 key; |
| 435 | |
| 436 | /* |
| 437 | * Are we enqueueing a waiting task? (for current tasks |
| 438 | * a dequeue/enqueue event is a NOP) |
| 439 | */ |
| 440 | if (se != cfs_rq_curr(cfs_rq)) |
Ingo Molnar | 5870db5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 441 | update_stats_wait_start(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 442 | /* |
| 443 | * Update the key: |
| 444 | */ |
| 445 | key = cfs_rq->fair_clock; |
| 446 | |
| 447 | /* |
| 448 | * Optimize the common nice 0 case: |
| 449 | */ |
| 450 | if (likely(se->load.weight == NICE_0_LOAD)) { |
| 451 | key -= se->wait_runtime; |
| 452 | } else { |
| 453 | u64 tmp; |
| 454 | |
| 455 | if (se->wait_runtime < 0) { |
| 456 | tmp = -se->wait_runtime; |
| 457 | key += (tmp * se->load.inv_weight) >> |
| 458 | (WMULT_SHIFT - NICE_0_SHIFT); |
| 459 | } else { |
| 460 | tmp = se->wait_runtime; |
Ingo Molnar | a69edb5 | 2007-08-09 11:16:52 +0200 | [diff] [blame] | 461 | key -= (tmp * se->load.inv_weight) >> |
| 462 | (WMULT_SHIFT - NICE_0_SHIFT); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 463 | } |
| 464 | } |
| 465 | |
| 466 | se->fair_key = key; |
| 467 | } |
| 468 | |
| 469 | /* |
| 470 | * Note: must be called with a freshly updated rq->fair_clock. |
| 471 | */ |
| 472 | static inline void |
Ingo Molnar | eac55ea | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 473 | __update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 474 | { |
| 475 | unsigned long delta_fair = se->delta_fair_run; |
| 476 | |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 477 | schedstat_set(se->wait_max, max(se->wait_max, |
| 478 | rq_of(cfs_rq)->clock - se->wait_start)); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 479 | |
| 480 | if (unlikely(se->load.weight != NICE_0_LOAD)) |
| 481 | delta_fair = calc_weighted(delta_fair, se->load.weight, |
| 482 | NICE_0_SHIFT); |
| 483 | |
| 484 | add_wait_runtime(cfs_rq, se, delta_fair); |
| 485 | } |
| 486 | |
| 487 | static void |
Ingo Molnar | 9ef0a96 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 488 | update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 489 | { |
| 490 | unsigned long delta_fair; |
| 491 | |
Ingo Molnar | b77d69d | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 492 | if (unlikely(!se->wait_start_fair)) |
| 493 | return; |
| 494 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 495 | delta_fair = (unsigned long)min((u64)(2*sysctl_sched_runtime_limit), |
| 496 | (u64)(cfs_rq->fair_clock - se->wait_start_fair)); |
| 497 | |
| 498 | se->delta_fair_run += delta_fair; |
| 499 | if (unlikely(abs(se->delta_fair_run) >= |
| 500 | sysctl_sched_stat_granularity)) { |
Ingo Molnar | eac55ea | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 501 | __update_stats_wait_end(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 502 | se->delta_fair_run = 0; |
| 503 | } |
| 504 | |
| 505 | se->wait_start_fair = 0; |
Ingo Molnar | 6cfb0d5 | 2007-08-02 17:41:40 +0200 | [diff] [blame] | 506 | schedstat_set(se->wait_start, 0); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 507 | } |
| 508 | |
| 509 | static inline void |
Ingo Molnar | 19b6a2e | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 510 | update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 511 | { |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 512 | update_curr(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 513 | /* |
| 514 | * Mark the end of the wait period if dequeueing a |
| 515 | * waiting task: |
| 516 | */ |
| 517 | if (se != cfs_rq_curr(cfs_rq)) |
Ingo Molnar | 9ef0a96 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 518 | update_stats_wait_end(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 519 | } |
| 520 | |
| 521 | /* |
| 522 | * We are picking a new current task - update its stats: |
| 523 | */ |
| 524 | static inline void |
Ingo Molnar | 79303e9 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 525 | update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 526 | { |
| 527 | /* |
| 528 | * We are starting a new run period: |
| 529 | */ |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 530 | se->exec_start = rq_of(cfs_rq)->clock; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 531 | } |
| 532 | |
| 533 | /* |
| 534 | * We are descheduling a task - update its stats: |
| 535 | */ |
| 536 | static inline void |
Ingo Molnar | c7e9b5b | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 537 | update_stats_curr_end(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 538 | { |
| 539 | se->exec_start = 0; |
| 540 | } |
| 541 | |
| 542 | /************************************************** |
| 543 | * Scheduling class queueing methods: |
| 544 | */ |
| 545 | |
Ingo Molnar | dfdc119e | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 546 | static void __enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 547 | { |
| 548 | unsigned long load = cfs_rq->load.weight, delta_fair; |
| 549 | long prev_runtime; |
| 550 | |
Ingo Molnar | b2133c8 | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 551 | /* |
| 552 | * Do not boost sleepers if there's too much bonus 'in flight' |
| 553 | * already: |
| 554 | */ |
| 555 | if (unlikely(cfs_rq->sleeper_bonus > sysctl_sched_runtime_limit)) |
| 556 | return; |
| 557 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 558 | if (sysctl_sched_features & SCHED_FEAT_SLEEPER_LOAD_AVG) |
| 559 | load = rq_of(cfs_rq)->cpu_load[2]; |
| 560 | |
| 561 | delta_fair = se->delta_fair_sleep; |
| 562 | |
| 563 | /* |
| 564 | * Fix up delta_fair with the effect of us running |
| 565 | * during the whole sleep period: |
| 566 | */ |
| 567 | if (sysctl_sched_features & SCHED_FEAT_SLEEPER_AVG) |
| 568 | delta_fair = div64_likely32((u64)delta_fair * load, |
| 569 | load + se->load.weight); |
| 570 | |
| 571 | if (unlikely(se->load.weight != NICE_0_LOAD)) |
| 572 | delta_fair = calc_weighted(delta_fair, se->load.weight, |
| 573 | NICE_0_SHIFT); |
| 574 | |
| 575 | prev_runtime = se->wait_runtime; |
| 576 | __add_wait_runtime(cfs_rq, se, delta_fair); |
Ingo Molnar | b2133c8 | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 577 | schedstat_add(cfs_rq, wait_runtime, se->wait_runtime); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 578 | delta_fair = se->wait_runtime - prev_runtime; |
| 579 | |
| 580 | /* |
| 581 | * Track the amount of bonus we've given to sleepers: |
| 582 | */ |
| 583 | cfs_rq->sleeper_bonus += delta_fair; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 584 | } |
| 585 | |
Ingo Molnar | 2396af6 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 586 | static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 587 | { |
| 588 | struct task_struct *tsk = task_of(se); |
| 589 | unsigned long delta_fair; |
| 590 | |
| 591 | if ((entity_is_task(se) && tsk->policy == SCHED_BATCH) || |
| 592 | !(sysctl_sched_features & SCHED_FEAT_FAIR_SLEEPERS)) |
| 593 | return; |
| 594 | |
| 595 | delta_fair = (unsigned long)min((u64)(2*sysctl_sched_runtime_limit), |
| 596 | (u64)(cfs_rq->fair_clock - se->sleep_start_fair)); |
| 597 | |
| 598 | se->delta_fair_sleep += delta_fair; |
| 599 | if (unlikely(abs(se->delta_fair_sleep) >= |
| 600 | sysctl_sched_stat_granularity)) { |
Ingo Molnar | dfdc119e | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 601 | __enqueue_sleeper(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 602 | se->delta_fair_sleep = 0; |
| 603 | } |
| 604 | |
| 605 | se->sleep_start_fair = 0; |
| 606 | |
| 607 | #ifdef CONFIG_SCHEDSTATS |
| 608 | if (se->sleep_start) { |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 609 | u64 delta = rq_of(cfs_rq)->clock - se->sleep_start; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 610 | |
| 611 | if ((s64)delta < 0) |
| 612 | delta = 0; |
| 613 | |
| 614 | if (unlikely(delta > se->sleep_max)) |
| 615 | se->sleep_max = delta; |
| 616 | |
| 617 | se->sleep_start = 0; |
| 618 | se->sum_sleep_runtime += delta; |
| 619 | } |
| 620 | if (se->block_start) { |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 621 | u64 delta = rq_of(cfs_rq)->clock - se->block_start; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 622 | |
| 623 | if ((s64)delta < 0) |
| 624 | delta = 0; |
| 625 | |
| 626 | if (unlikely(delta > se->block_max)) |
| 627 | se->block_max = delta; |
| 628 | |
| 629 | se->block_start = 0; |
| 630 | se->sum_sleep_runtime += delta; |
| 631 | } |
| 632 | #endif |
| 633 | } |
| 634 | |
| 635 | static void |
Ingo Molnar | 668031c | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 636 | enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 637 | { |
| 638 | /* |
| 639 | * Update the fair clock. |
| 640 | */ |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 641 | update_curr(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 642 | |
| 643 | if (wakeup) |
Ingo Molnar | 2396af6 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 644 | enqueue_sleeper(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 645 | |
Ingo Molnar | d2417e5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 646 | update_stats_enqueue(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 647 | __enqueue_entity(cfs_rq, se); |
| 648 | } |
| 649 | |
| 650 | static void |
Ingo Molnar | 525c271 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 651 | dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 652 | { |
Ingo Molnar | 19b6a2e | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 653 | update_stats_dequeue(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 654 | if (sleep) { |
| 655 | se->sleep_start_fair = cfs_rq->fair_clock; |
| 656 | #ifdef CONFIG_SCHEDSTATS |
| 657 | if (entity_is_task(se)) { |
| 658 | struct task_struct *tsk = task_of(se); |
| 659 | |
| 660 | if (tsk->state & TASK_INTERRUPTIBLE) |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 661 | se->sleep_start = rq_of(cfs_rq)->clock; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 662 | if (tsk->state & TASK_UNINTERRUPTIBLE) |
Ingo Molnar | d281918 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 663 | se->block_start = rq_of(cfs_rq)->clock; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 664 | } |
| 665 | cfs_rq->wait_runtime -= se->wait_runtime; |
| 666 | #endif |
| 667 | } |
| 668 | __dequeue_entity(cfs_rq, se); |
| 669 | } |
| 670 | |
| 671 | /* |
| 672 | * Preempt the current task with a newly woken task if needed: |
| 673 | */ |
Ingo Molnar | f6cf891 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 674 | static int |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 675 | __check_preempt_curr_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, |
| 676 | struct sched_entity *curr, unsigned long granularity) |
| 677 | { |
| 678 | s64 __delta = curr->fair_key - se->fair_key; |
| 679 | |
| 680 | /* |
| 681 | * Take scheduling granularity into account - do not |
| 682 | * preempt the current task unless the best task has |
| 683 | * a larger than sched_granularity fairness advantage: |
| 684 | */ |
Ingo Molnar | f6cf891 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 685 | if (__delta > niced_granularity(curr, granularity)) { |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 686 | resched_task(rq_of(cfs_rq)->curr); |
Ingo Molnar | f6cf891 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 687 | return 1; |
| 688 | } |
| 689 | return 0; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 690 | } |
| 691 | |
| 692 | static inline void |
Ingo Molnar | 8494f41 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 693 | set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 694 | { |
| 695 | /* |
| 696 | * Any task has to be enqueued before it get to execute on |
| 697 | * a CPU. So account for the time it spent waiting on the |
| 698 | * runqueue. (note, here we rely on pick_next_task() having |
| 699 | * done a put_prev_task_fair() shortly before this, which |
| 700 | * updated rq->fair_clock - used by update_stats_wait_end()) |
| 701 | */ |
Ingo Molnar | 9ef0a96 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 702 | update_stats_wait_end(cfs_rq, se); |
Ingo Molnar | 79303e9 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 703 | update_stats_curr_start(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 704 | set_cfs_rq_curr(cfs_rq, se); |
| 705 | } |
| 706 | |
Ingo Molnar | 9948f4b | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 707 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 708 | { |
| 709 | struct sched_entity *se = __pick_next_entity(cfs_rq); |
| 710 | |
Ingo Molnar | 8494f41 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 711 | set_next_entity(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 712 | |
| 713 | return se; |
| 714 | } |
| 715 | |
Ingo Molnar | ab6cde2 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 716 | static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 717 | { |
| 718 | /* |
| 719 | * If still on the runqueue then deactivate_task() |
| 720 | * was not called and update_curr() has to be done: |
| 721 | */ |
| 722 | if (prev->on_rq) |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 723 | update_curr(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 724 | |
Ingo Molnar | c7e9b5b | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 725 | update_stats_curr_end(cfs_rq, prev); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 726 | |
| 727 | if (prev->on_rq) |
Ingo Molnar | 5870db5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 728 | update_stats_wait_start(cfs_rq, prev); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 729 | set_cfs_rq_curr(cfs_rq, NULL); |
| 730 | } |
| 731 | |
| 732 | static void entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) |
| 733 | { |
Ingo Molnar | f6cf891 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 734 | unsigned long gran, ideal_runtime, delta_exec; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 735 | struct sched_entity *next; |
Ingo Molnar | c1b3da3 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 736 | |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 737 | /* |
| 738 | * Dequeue and enqueue the task to update its |
| 739 | * position within the tree: |
| 740 | */ |
Ingo Molnar | 525c271 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 741 | dequeue_entity(cfs_rq, curr, 0); |
Ingo Molnar | 668031c | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 742 | enqueue_entity(cfs_rq, curr, 0); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 743 | |
| 744 | /* |
| 745 | * Reschedule if another task tops the current one. |
| 746 | */ |
| 747 | next = __pick_next_entity(cfs_rq); |
| 748 | if (next == curr) |
| 749 | return; |
| 750 | |
Ingo Molnar | f6cf891 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 751 | gran = sched_granularity(cfs_rq); |
| 752 | ideal_runtime = niced_granularity(curr, |
| 753 | max(sysctl_sched_latency / cfs_rq->nr_running, |
| 754 | (unsigned long)sysctl_sched_min_granularity)); |
| 755 | /* |
| 756 | * If we executed more than what the latency constraint suggests, |
| 757 | * reduce the rescheduling granularity. This way the total latency |
| 758 | * of how much a task is not scheduled converges to |
| 759 | * sysctl_sched_latency: |
| 760 | */ |
| 761 | delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; |
| 762 | if (delta_exec > ideal_runtime) |
| 763 | gran = 0; |
| 764 | |
| 765 | if (__check_preempt_curr_fair(cfs_rq, next, curr, gran)) |
| 766 | curr->prev_sum_exec_runtime = curr->sum_exec_runtime; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 767 | } |
| 768 | |
| 769 | /************************************************** |
| 770 | * CFS operations on tasks: |
| 771 | */ |
| 772 | |
| 773 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 774 | |
| 775 | /* Walk up scheduling entities hierarchy */ |
| 776 | #define for_each_sched_entity(se) \ |
| 777 | for (; se; se = se->parent) |
| 778 | |
| 779 | static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) |
| 780 | { |
| 781 | return p->se.cfs_rq; |
| 782 | } |
| 783 | |
| 784 | /* runqueue on which this entity is (to be) queued */ |
| 785 | static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) |
| 786 | { |
| 787 | return se->cfs_rq; |
| 788 | } |
| 789 | |
| 790 | /* runqueue "owned" by this group */ |
| 791 | static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) |
| 792 | { |
| 793 | return grp->my_q; |
| 794 | } |
| 795 | |
| 796 | /* Given a group's cfs_rq on one cpu, return its corresponding cfs_rq on |
| 797 | * another cpu ('this_cpu') |
| 798 | */ |
| 799 | static inline struct cfs_rq *cpu_cfs_rq(struct cfs_rq *cfs_rq, int this_cpu) |
| 800 | { |
| 801 | /* A later patch will take group into account */ |
| 802 | return &cpu_rq(this_cpu)->cfs; |
| 803 | } |
| 804 | |
| 805 | /* Iterate thr' all leaf cfs_rq's on a runqueue */ |
| 806 | #define for_each_leaf_cfs_rq(rq, cfs_rq) \ |
| 807 | list_for_each_entry(cfs_rq, &rq->leaf_cfs_rq_list, leaf_cfs_rq_list) |
| 808 | |
| 809 | /* Do the two (enqueued) tasks belong to the same group ? */ |
| 810 | static inline int is_same_group(struct task_struct *curr, struct task_struct *p) |
| 811 | { |
| 812 | if (curr->se.cfs_rq == p->se.cfs_rq) |
| 813 | return 1; |
| 814 | |
| 815 | return 0; |
| 816 | } |
| 817 | |
| 818 | #else /* CONFIG_FAIR_GROUP_SCHED */ |
| 819 | |
| 820 | #define for_each_sched_entity(se) \ |
| 821 | for (; se; se = NULL) |
| 822 | |
| 823 | static inline struct cfs_rq *task_cfs_rq(struct task_struct *p) |
| 824 | { |
| 825 | return &task_rq(p)->cfs; |
| 826 | } |
| 827 | |
| 828 | static inline struct cfs_rq *cfs_rq_of(struct sched_entity *se) |
| 829 | { |
| 830 | struct task_struct *p = task_of(se); |
| 831 | struct rq *rq = task_rq(p); |
| 832 | |
| 833 | return &rq->cfs; |
| 834 | } |
| 835 | |
| 836 | /* runqueue "owned" by this group */ |
| 837 | static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) |
| 838 | { |
| 839 | return NULL; |
| 840 | } |
| 841 | |
| 842 | static inline struct cfs_rq *cpu_cfs_rq(struct cfs_rq *cfs_rq, int this_cpu) |
| 843 | { |
| 844 | return &cpu_rq(this_cpu)->cfs; |
| 845 | } |
| 846 | |
| 847 | #define for_each_leaf_cfs_rq(rq, cfs_rq) \ |
| 848 | for (cfs_rq = &rq->cfs; cfs_rq; cfs_rq = NULL) |
| 849 | |
| 850 | static inline int is_same_group(struct task_struct *curr, struct task_struct *p) |
| 851 | { |
| 852 | return 1; |
| 853 | } |
| 854 | |
| 855 | #endif /* CONFIG_FAIR_GROUP_SCHED */ |
| 856 | |
| 857 | /* |
| 858 | * The enqueue_task method is called before nr_running is |
| 859 | * increased. Here we update the fair scheduling stats and |
| 860 | * then put the task into the rbtree: |
| 861 | */ |
Ingo Molnar | fd390f6 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 862 | static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 863 | { |
| 864 | struct cfs_rq *cfs_rq; |
| 865 | struct sched_entity *se = &p->se; |
| 866 | |
| 867 | for_each_sched_entity(se) { |
| 868 | if (se->on_rq) |
| 869 | break; |
| 870 | cfs_rq = cfs_rq_of(se); |
Ingo Molnar | 668031c | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 871 | enqueue_entity(cfs_rq, se, wakeup); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 872 | } |
| 873 | } |
| 874 | |
| 875 | /* |
| 876 | * The dequeue_task method is called before nr_running is |
| 877 | * decreased. We remove the task from the rbtree and |
| 878 | * update the fair scheduling stats: |
| 879 | */ |
Ingo Molnar | f02231e | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 880 | static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 881 | { |
| 882 | struct cfs_rq *cfs_rq; |
| 883 | struct sched_entity *se = &p->se; |
| 884 | |
| 885 | for_each_sched_entity(se) { |
| 886 | cfs_rq = cfs_rq_of(se); |
Ingo Molnar | 525c271 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 887 | dequeue_entity(cfs_rq, se, sleep); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 888 | /* Don't dequeue parent if it has other entities besides us */ |
| 889 | if (cfs_rq->load.weight) |
| 890 | break; |
| 891 | } |
| 892 | } |
| 893 | |
| 894 | /* |
| 895 | * sched_yield() support is very simple - we dequeue and enqueue |
| 896 | */ |
| 897 | static void yield_task_fair(struct rq *rq, struct task_struct *p) |
| 898 | { |
| 899 | struct cfs_rq *cfs_rq = task_cfs_rq(p); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 900 | |
Ingo Molnar | c1b3da3 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 901 | __update_rq_clock(rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 902 | /* |
| 903 | * Dequeue and enqueue the task to update its |
| 904 | * position within the tree: |
| 905 | */ |
Ingo Molnar | 525c271 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 906 | dequeue_entity(cfs_rq, &p->se, 0); |
Ingo Molnar | 668031c | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 907 | enqueue_entity(cfs_rq, &p->se, 0); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 908 | } |
| 909 | |
| 910 | /* |
| 911 | * Preempt the current task with a newly woken task if needed: |
| 912 | */ |
| 913 | static void check_preempt_curr_fair(struct rq *rq, struct task_struct *p) |
| 914 | { |
| 915 | struct task_struct *curr = rq->curr; |
| 916 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); |
| 917 | unsigned long gran; |
| 918 | |
| 919 | if (unlikely(rt_prio(p->prio))) { |
Ingo Molnar | a8e504d | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 920 | update_rq_clock(rq); |
Ingo Molnar | b7cc089 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 921 | update_curr(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 922 | resched_task(curr); |
| 923 | return; |
| 924 | } |
| 925 | |
| 926 | gran = sysctl_sched_wakeup_granularity; |
| 927 | /* |
| 928 | * Batch tasks prefer throughput over latency: |
| 929 | */ |
| 930 | if (unlikely(p->policy == SCHED_BATCH)) |
| 931 | gran = sysctl_sched_batch_wakeup_granularity; |
| 932 | |
| 933 | if (is_same_group(curr, p)) |
| 934 | __check_preempt_curr_fair(cfs_rq, &p->se, &curr->se, gran); |
| 935 | } |
| 936 | |
Ingo Molnar | fb8d472 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 937 | static struct task_struct *pick_next_task_fair(struct rq *rq) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 938 | { |
| 939 | struct cfs_rq *cfs_rq = &rq->cfs; |
| 940 | struct sched_entity *se; |
| 941 | |
| 942 | if (unlikely(!cfs_rq->nr_running)) |
| 943 | return NULL; |
| 944 | |
| 945 | do { |
Ingo Molnar | 9948f4b | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 946 | se = pick_next_entity(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 947 | cfs_rq = group_cfs_rq(se); |
| 948 | } while (cfs_rq); |
| 949 | |
| 950 | return task_of(se); |
| 951 | } |
| 952 | |
| 953 | /* |
| 954 | * Account for a descheduled task: |
| 955 | */ |
Ingo Molnar | 31ee529 | 2007-08-09 11:16:49 +0200 | [diff] [blame] | 956 | static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 957 | { |
| 958 | struct sched_entity *se = &prev->se; |
| 959 | struct cfs_rq *cfs_rq; |
| 960 | |
| 961 | for_each_sched_entity(se) { |
| 962 | cfs_rq = cfs_rq_of(se); |
Ingo Molnar | ab6cde2 | 2007-08-09 11:16:48 +0200 | [diff] [blame] | 963 | put_prev_entity(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 964 | } |
| 965 | } |
| 966 | |
| 967 | /************************************************** |
| 968 | * Fair scheduling class load-balancing methods: |
| 969 | */ |
| 970 | |
| 971 | /* |
| 972 | * Load-balancing iterator. Note: while the runqueue stays locked |
| 973 | * during the whole iteration, the current task might be |
| 974 | * dequeued so the iterator has to be dequeue-safe. Here we |
| 975 | * achieve that by always pre-iterating before returning |
| 976 | * the current task: |
| 977 | */ |
| 978 | static inline struct task_struct * |
| 979 | __load_balance_iterator(struct cfs_rq *cfs_rq, struct rb_node *curr) |
| 980 | { |
| 981 | struct task_struct *p; |
| 982 | |
| 983 | if (!curr) |
| 984 | return NULL; |
| 985 | |
| 986 | p = rb_entry(curr, struct task_struct, se.run_node); |
| 987 | cfs_rq->rb_load_balance_curr = rb_next(curr); |
| 988 | |
| 989 | return p; |
| 990 | } |
| 991 | |
| 992 | static struct task_struct *load_balance_start_fair(void *arg) |
| 993 | { |
| 994 | struct cfs_rq *cfs_rq = arg; |
| 995 | |
| 996 | return __load_balance_iterator(cfs_rq, first_fair(cfs_rq)); |
| 997 | } |
| 998 | |
| 999 | static struct task_struct *load_balance_next_fair(void *arg) |
| 1000 | { |
| 1001 | struct cfs_rq *cfs_rq = arg; |
| 1002 | |
| 1003 | return __load_balance_iterator(cfs_rq, cfs_rq->rb_load_balance_curr); |
| 1004 | } |
| 1005 | |
Peter Williams | a4ac01c | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1006 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1007 | static int cfs_rq_best_prio(struct cfs_rq *cfs_rq) |
| 1008 | { |
| 1009 | struct sched_entity *curr; |
| 1010 | struct task_struct *p; |
| 1011 | |
| 1012 | if (!cfs_rq->nr_running) |
| 1013 | return MAX_PRIO; |
| 1014 | |
| 1015 | curr = __pick_next_entity(cfs_rq); |
| 1016 | p = task_of(curr); |
| 1017 | |
| 1018 | return p->prio; |
| 1019 | } |
Peter Williams | a4ac01c | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1020 | #endif |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1021 | |
Peter Williams | 4301065 | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1022 | static unsigned long |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1023 | load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, |
Peter Williams | a4ac01c | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1024 | unsigned long max_nr_move, unsigned long max_load_move, |
| 1025 | struct sched_domain *sd, enum cpu_idle_type idle, |
| 1026 | int *all_pinned, int *this_best_prio) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1027 | { |
| 1028 | struct cfs_rq *busy_cfs_rq; |
| 1029 | unsigned long load_moved, total_nr_moved = 0, nr_moved; |
| 1030 | long rem_load_move = max_load_move; |
| 1031 | struct rq_iterator cfs_rq_iterator; |
| 1032 | |
| 1033 | cfs_rq_iterator.start = load_balance_start_fair; |
| 1034 | cfs_rq_iterator.next = load_balance_next_fair; |
| 1035 | |
| 1036 | for_each_leaf_cfs_rq(busiest, busy_cfs_rq) { |
Peter Williams | a4ac01c | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1037 | #ifdef CONFIG_FAIR_GROUP_SCHED |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1038 | struct cfs_rq *this_cfs_rq; |
Ingo Molnar | e56f31a | 2007-08-10 23:05:11 +0200 | [diff] [blame] | 1039 | long imbalance; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1040 | unsigned long maxload; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1041 | |
| 1042 | this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu); |
| 1043 | |
Ingo Molnar | e56f31a | 2007-08-10 23:05:11 +0200 | [diff] [blame] | 1044 | imbalance = busy_cfs_rq->load.weight - this_cfs_rq->load.weight; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1045 | /* Don't pull if this_cfs_rq has more load than busy_cfs_rq */ |
| 1046 | if (imbalance <= 0) |
| 1047 | continue; |
| 1048 | |
| 1049 | /* Don't pull more than imbalance/2 */ |
| 1050 | imbalance /= 2; |
| 1051 | maxload = min(rem_load_move, imbalance); |
| 1052 | |
Peter Williams | a4ac01c | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1053 | *this_best_prio = cfs_rq_best_prio(this_cfs_rq); |
| 1054 | #else |
Ingo Molnar | e56f31a | 2007-08-10 23:05:11 +0200 | [diff] [blame] | 1055 | # define maxload rem_load_move |
Peter Williams | a4ac01c | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1056 | #endif |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1057 | /* pass busy_cfs_rq argument into |
| 1058 | * load_balance_[start|next]_fair iterators |
| 1059 | */ |
| 1060 | cfs_rq_iterator.arg = busy_cfs_rq; |
| 1061 | nr_moved = balance_tasks(this_rq, this_cpu, busiest, |
| 1062 | max_nr_move, maxload, sd, idle, all_pinned, |
Peter Williams | a4ac01c | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1063 | &load_moved, this_best_prio, &cfs_rq_iterator); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1064 | |
| 1065 | total_nr_moved += nr_moved; |
| 1066 | max_nr_move -= nr_moved; |
| 1067 | rem_load_move -= load_moved; |
| 1068 | |
| 1069 | if (max_nr_move <= 0 || rem_load_move <= 0) |
| 1070 | break; |
| 1071 | } |
| 1072 | |
Peter Williams | 4301065 | 2007-08-09 11:16:46 +0200 | [diff] [blame] | 1073 | return max_load_move - rem_load_move; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1074 | } |
| 1075 | |
| 1076 | /* |
| 1077 | * scheduler tick hitting a task of our scheduling class: |
| 1078 | */ |
| 1079 | static void task_tick_fair(struct rq *rq, struct task_struct *curr) |
| 1080 | { |
| 1081 | struct cfs_rq *cfs_rq; |
| 1082 | struct sched_entity *se = &curr->se; |
| 1083 | |
| 1084 | for_each_sched_entity(se) { |
| 1085 | cfs_rq = cfs_rq_of(se); |
| 1086 | entity_tick(cfs_rq, se); |
| 1087 | } |
| 1088 | } |
| 1089 | |
| 1090 | /* |
| 1091 | * Share the fairness runtime between parent and child, thus the |
| 1092 | * total amount of pressure for CPU stays equal - new tasks |
| 1093 | * get a chance to run but frequent forkers are not allowed to |
| 1094 | * monopolize the CPU. Note: the parent runqueue is locked, |
| 1095 | * the child is not running yet. |
| 1096 | */ |
Ingo Molnar | ee0827d | 2007-08-09 11:16:49 +0200 | [diff] [blame] | 1097 | static void task_new_fair(struct rq *rq, struct task_struct *p) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1098 | { |
| 1099 | struct cfs_rq *cfs_rq = task_cfs_rq(p); |
Ting Yang | 7109c442 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 1100 | struct sched_entity *se = &p->se, *curr = cfs_rq_curr(cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1101 | |
| 1102 | sched_info_queued(p); |
| 1103 | |
Ting Yang | 7109c442 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 1104 | update_curr(cfs_rq); |
Ingo Molnar | d2417e5 | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 1105 | update_stats_enqueue(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1106 | /* |
| 1107 | * Child runs first: we let it run before the parent |
| 1108 | * until it reschedules once. We set up the key so that |
| 1109 | * it will preempt the parent: |
| 1110 | */ |
Ingo Molnar | 9f508f8 | 2007-08-28 12:53:24 +0200 | [diff] [blame^] | 1111 | se->fair_key = curr->fair_key - |
Ting Yang | 7109c442 | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 1112 | niced_granularity(curr, sched_granularity(cfs_rq)) - 1; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1113 | /* |
| 1114 | * The first wait is dominated by the child-runs-first logic, |
| 1115 | * so do not credit it with that waiting time yet: |
| 1116 | */ |
| 1117 | if (sysctl_sched_features & SCHED_FEAT_SKIP_INITIAL) |
Ingo Molnar | 9f508f8 | 2007-08-28 12:53:24 +0200 | [diff] [blame^] | 1118 | se->wait_start_fair = 0; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1119 | |
| 1120 | /* |
| 1121 | * The statistical average of wait_runtime is about |
| 1122 | * -granularity/2, so initialize the task with that: |
| 1123 | */ |
Ingo Molnar | 213c8af | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 1124 | if (sysctl_sched_features & SCHED_FEAT_START_DEBIT) { |
Ingo Molnar | 9f508f8 | 2007-08-28 12:53:24 +0200 | [diff] [blame^] | 1125 | se->wait_runtime = -(sched_granularity(cfs_rq) / 2); |
Ingo Molnar | 213c8af | 2007-08-28 12:53:24 +0200 | [diff] [blame] | 1126 | schedstat_add(cfs_rq, wait_runtime, se->wait_runtime); |
| 1127 | } |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1128 | |
| 1129 | __enqueue_entity(cfs_rq, se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1130 | } |
| 1131 | |
| 1132 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 1133 | /* Account for a task changing its policy or group. |
| 1134 | * |
| 1135 | * This routine is mostly called to set cfs_rq->curr field when a task |
| 1136 | * migrates between groups/classes. |
| 1137 | */ |
| 1138 | static void set_curr_task_fair(struct rq *rq) |
| 1139 | { |
Bruce Ashfield | 7c6c16f | 2007-08-24 20:39:10 +0200 | [diff] [blame] | 1140 | struct sched_entity *se = &rq->curr->se; |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1141 | |
Ingo Molnar | c3b64f1 | 2007-08-09 11:16:51 +0200 | [diff] [blame] | 1142 | for_each_sched_entity(se) |
| 1143 | set_next_entity(cfs_rq_of(se), se); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1144 | } |
| 1145 | #else |
| 1146 | static void set_curr_task_fair(struct rq *rq) |
| 1147 | { |
| 1148 | } |
| 1149 | #endif |
| 1150 | |
| 1151 | /* |
| 1152 | * All the scheduling class methods: |
| 1153 | */ |
| 1154 | struct sched_class fair_sched_class __read_mostly = { |
| 1155 | .enqueue_task = enqueue_task_fair, |
| 1156 | .dequeue_task = dequeue_task_fair, |
| 1157 | .yield_task = yield_task_fair, |
| 1158 | |
| 1159 | .check_preempt_curr = check_preempt_curr_fair, |
| 1160 | |
| 1161 | .pick_next_task = pick_next_task_fair, |
| 1162 | .put_prev_task = put_prev_task_fair, |
| 1163 | |
| 1164 | .load_balance = load_balance_fair, |
| 1165 | |
| 1166 | .set_curr_task = set_curr_task_fair, |
| 1167 | .task_tick = task_tick_fair, |
| 1168 | .task_new = task_new_fair, |
| 1169 | }; |
| 1170 | |
| 1171 | #ifdef CONFIG_SCHED_DEBUG |
Ingo Molnar | 5cef9ec | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 1172 | static void print_cfs_stats(struct seq_file *m, int cpu) |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1173 | { |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1174 | struct cfs_rq *cfs_rq; |
| 1175 | |
Ingo Molnar | c3b64f1 | 2007-08-09 11:16:51 +0200 | [diff] [blame] | 1176 | for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq) |
Ingo Molnar | 5cef9ec | 2007-08-09 11:16:47 +0200 | [diff] [blame] | 1177 | print_cfs_rq(m, cpu, cfs_rq); |
Ingo Molnar | bf0f6f2 | 2007-07-09 18:51:58 +0200 | [diff] [blame] | 1178 | } |
| 1179 | #endif |