blob: ce05271219abcc5bf7dd4087d91ce98f487ee2e9 [file] [log] [blame]
Peter Zijlstra3e51f332008-05-03 18:29:28 +02001/*
2 * sched_clock for unstable cpu clocks
3 *
4 * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
5 *
6 * Based on code by:
7 * Ingo Molnar <mingo@redhat.com>
8 * Guillaume Chazarain <guichaz@gmail.com>
9 *
10 * Create a semi stable clock from a mixture of other events, including:
11 * - gtod
12 * - jiffies
13 * - sched_clock()
14 * - explicit idle events
15 *
16 * We use gtod as base and the unstable clock deltas. The deltas are filtered,
17 * making it monotonic and keeping it within an expected window. This window
18 * is set up using jiffies.
19 *
20 * Furthermore, explicit sleep and wakeup hooks allow us to account for time
21 * that is otherwise invisible (TSC gets stopped).
22 *
23 * The clock: sched_clock_cpu() is monotonic per cpu, and should be somewhat
24 * consistent between cpus (never more than 1 jiffies difference).
25 */
26#include <linux/sched.h>
27#include <linux/percpu.h>
28#include <linux/spinlock.h>
29#include <linux/ktime.h>
30#include <linux/module.h>
31
32
33#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
34
35struct sched_clock_data {
36 /*
37 * Raw spinlock - this is a special case: this might be called
38 * from within instrumentation code so we dont want to do any
39 * instrumentation ourselves.
40 */
41 raw_spinlock_t lock;
42
43 unsigned long prev_jiffies;
44 u64 prev_raw;
45 u64 tick_raw;
46 u64 tick_gtod;
47 u64 clock;
48};
49
50static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
51
52static inline struct sched_clock_data *this_scd(void)
53{
54 return &__get_cpu_var(sched_clock_data);
55}
56
57static inline struct sched_clock_data *cpu_sdc(int cpu)
58{
59 return &per_cpu(sched_clock_data, cpu);
60}
61
Peter Zijlstraa3817592008-05-29 10:07:15 +020062static __read_mostly int sched_clock_running;
63
Peter Zijlstra3e51f332008-05-03 18:29:28 +020064void sched_clock_init(void)
65{
66 u64 ktime_now = ktime_to_ns(ktime_get());
Peter Zijlstraa3817592008-05-29 10:07:15 +020067 unsigned long now_jiffies = jiffies;
Peter Zijlstra3e51f332008-05-03 18:29:28 +020068 int cpu;
69
70 for_each_possible_cpu(cpu) {
71 struct sched_clock_data *scd = cpu_sdc(cpu);
72
73 scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
Peter Zijlstraa3817592008-05-29 10:07:15 +020074 scd->prev_jiffies = now_jiffies;
75 scd->prev_raw = 0;
76 scd->tick_raw = 0;
Peter Zijlstra3e51f332008-05-03 18:29:28 +020077 scd->tick_gtod = ktime_now;
78 scd->clock = ktime_now;
79 }
Peter Zijlstraa3817592008-05-29 10:07:15 +020080
81 sched_clock_running = 1;
Peter Zijlstra3e51f332008-05-03 18:29:28 +020082}
83
84/*
85 * update the percpu scd from the raw @now value
86 *
87 * - filter out backward motion
88 * - use jiffies to generate a min,max window to clip the raw values
89 */
90static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
91{
92 unsigned long now_jiffies = jiffies;
93 long delta_jiffies = now_jiffies - scd->prev_jiffies;
94 u64 clock = scd->clock;
95 u64 min_clock, max_clock;
96 s64 delta = now - scd->prev_raw;
97
98 WARN_ON_ONCE(!irqs_disabled());
99 min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
100
101 if (unlikely(delta < 0)) {
102 clock++;
103 goto out;
104 }
105
106 max_clock = min_clock + TICK_NSEC;
107
108 if (unlikely(clock + delta > max_clock)) {
109 if (clock < max_clock)
110 clock = max_clock;
111 else
112 clock++;
113 } else {
114 clock += delta;
115 }
116
117 out:
118 if (unlikely(clock < min_clock))
119 clock = min_clock;
120
121 scd->prev_raw = now;
122 scd->prev_jiffies = now_jiffies;
123 scd->clock = clock;
124}
125
126static void lock_double_clock(struct sched_clock_data *data1,
127 struct sched_clock_data *data2)
128{
129 if (data1 < data2) {
130 __raw_spin_lock(&data1->lock);
131 __raw_spin_lock(&data2->lock);
132 } else {
133 __raw_spin_lock(&data2->lock);
134 __raw_spin_lock(&data1->lock);
135 }
136}
137
138u64 sched_clock_cpu(int cpu)
139{
140 struct sched_clock_data *scd = cpu_sdc(cpu);
141 u64 now, clock;
142
Peter Zijlstraa3817592008-05-29 10:07:15 +0200143 if (unlikely(!sched_clock_running))
144 return 0ull;
145
Peter Zijlstra3e51f332008-05-03 18:29:28 +0200146 WARN_ON_ONCE(!irqs_disabled());
147 now = sched_clock();
148
149 if (cpu != raw_smp_processor_id()) {
150 /*
151 * in order to update a remote cpu's clock based on our
152 * unstable raw time rebase it against:
153 * tick_raw (offset between raw counters)
154 * tick_gotd (tick offset between cpus)
155 */
156 struct sched_clock_data *my_scd = this_scd();
157
158 lock_double_clock(scd, my_scd);
159
160 now -= my_scd->tick_raw;
161 now += scd->tick_raw;
162
163 now -= my_scd->tick_gtod;
164 now += scd->tick_gtod;
165
166 __raw_spin_unlock(&my_scd->lock);
167 } else {
168 __raw_spin_lock(&scd->lock);
169 }
170
171 __update_sched_clock(scd, now);
172 clock = scd->clock;
173
174 __raw_spin_unlock(&scd->lock);
175
176 return clock;
177}
178
179void sched_clock_tick(void)
180{
181 struct sched_clock_data *scd = this_scd();
182 u64 now, now_gtod;
183
Peter Zijlstraa3817592008-05-29 10:07:15 +0200184 if (unlikely(!sched_clock_running))
185 return;
186
Peter Zijlstra3e51f332008-05-03 18:29:28 +0200187 WARN_ON_ONCE(!irqs_disabled());
188
189 now = sched_clock();
190 now_gtod = ktime_to_ns(ktime_get());
191
192 __raw_spin_lock(&scd->lock);
193 __update_sched_clock(scd, now);
194 /*
195 * update tick_gtod after __update_sched_clock() because that will
196 * already observe 1 new jiffy; adding a new tick_gtod to that would
197 * increase the clock 2 jiffies.
198 */
199 scd->tick_raw = now;
200 scd->tick_gtod = now_gtod;
201 __raw_spin_unlock(&scd->lock);
202}
203
204/*
205 * We are going deep-idle (irqs are disabled):
206 */
207void sched_clock_idle_sleep_event(void)
208{
209 sched_clock_cpu(smp_processor_id());
210}
211EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);
212
213/*
214 * We just idled delta nanoseconds (called with irqs disabled):
215 */
216void sched_clock_idle_wakeup_event(u64 delta_ns)
217{
218 struct sched_clock_data *scd = this_scd();
219 u64 now = sched_clock();
220
221 /*
222 * Override the previous timestamp and ignore all
223 * sched_clock() deltas that occured while we idled,
224 * and use the PM-provided delta_ns to advance the
225 * rq clock:
226 */
227 __raw_spin_lock(&scd->lock);
228 scd->prev_raw = now;
229 scd->clock += delta_ns;
230 __raw_spin_unlock(&scd->lock);
231
232 touch_softlockup_watchdog();
233}
234EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
235
236#endif
237
238/*
239 * Scheduler clock - returns current time in nanosec units.
240 * This is default implementation.
241 * Architectures and sub-architectures can override this.
242 */
243unsigned long long __attribute__((weak)) sched_clock(void)
244{
245 return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ);
246}