blob: 8c3094d0f7b78426e367e2cf3ed538e54d686cbd [file] [log] [blame]
Jamie Iles1b8873a2010-02-02 20:25:44 +01001#undef DEBUG
2
3/*
4 * ARM performance counter support.
5 *
6 * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
Will Deacon43eab872010-11-13 19:04:32 +00007 * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
Jean PIHET796d1292010-01-26 18:51:05 +01008 *
Jamie Iles1b8873a2010-02-02 20:25:44 +01009 * This code is based on the sparc64 perf event code, which is in turn based
10 * on the x86 code. Callchain code is based on the ARM OProfile backtrace
11 * code.
12 */
13#define pr_fmt(fmt) "hw perfevents: " fmt
14
Jamie Iles1b8873a2010-02-02 20:25:44 +010015#include <linux/kernel.h>
Will Deacon49c006b2010-04-29 17:13:24 +010016#include <linux/platform_device.h>
Jon Hunter7be29582012-05-31 13:05:20 -050017#include <linux/pm_runtime.h>
Will Deacon5505b202012-07-29 13:09:14 +010018#include <linux/uaccess.h>
Jamie Iles1b8873a2010-02-02 20:25:44 +010019
Jamie Iles1b8873a2010-02-02 20:25:44 +010020#include <asm/irq_regs.h>
21#include <asm/pmu.h>
22#include <asm/stacktrace.h>
23
Jamie Iles1b8873a2010-02-02 20:25:44 +010024static int
Mark Rutlande1f431b2011-04-28 15:47:10 +010025armpmu_map_cache_event(const unsigned (*cache_map)
26 [PERF_COUNT_HW_CACHE_MAX]
27 [PERF_COUNT_HW_CACHE_OP_MAX]
28 [PERF_COUNT_HW_CACHE_RESULT_MAX],
29 u64 config)
Jamie Iles1b8873a2010-02-02 20:25:44 +010030{
31 unsigned int cache_type, cache_op, cache_result, ret;
32
33 cache_type = (config >> 0) & 0xff;
34 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
35 return -EINVAL;
36
37 cache_op = (config >> 8) & 0xff;
38 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
39 return -EINVAL;
40
41 cache_result = (config >> 16) & 0xff;
42 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
43 return -EINVAL;
44
Mark Rutlande1f431b2011-04-28 15:47:10 +010045 ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
Jamie Iles1b8873a2010-02-02 20:25:44 +010046
47 if (ret == CACHE_OP_UNSUPPORTED)
48 return -ENOENT;
49
50 return ret;
51}
52
53static int
Will Deacon6dbc0022012-07-29 12:36:28 +010054armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
Will Deacon84fee972010-11-13 17:13:56 +000055{
Mark Rutlande1f431b2011-04-28 15:47:10 +010056 int mapping = (*event_map)[config];
57 return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
Will Deacon84fee972010-11-13 17:13:56 +000058}
59
60static int
Mark Rutlande1f431b2011-04-28 15:47:10 +010061armpmu_map_raw_event(u32 raw_event_mask, u64 config)
Will Deacon84fee972010-11-13 17:13:56 +000062{
Mark Rutlande1f431b2011-04-28 15:47:10 +010063 return (int)(config & raw_event_mask);
64}
65
Will Deacon6dbc0022012-07-29 12:36:28 +010066int
67armpmu_map_event(struct perf_event *event,
68 const unsigned (*event_map)[PERF_COUNT_HW_MAX],
69 const unsigned (*cache_map)
70 [PERF_COUNT_HW_CACHE_MAX]
71 [PERF_COUNT_HW_CACHE_OP_MAX]
72 [PERF_COUNT_HW_CACHE_RESULT_MAX],
73 u32 raw_event_mask)
Mark Rutlande1f431b2011-04-28 15:47:10 +010074{
75 u64 config = event->attr.config;
76
77 switch (event->attr.type) {
78 case PERF_TYPE_HARDWARE:
Will Deacon6dbc0022012-07-29 12:36:28 +010079 return armpmu_map_hw_event(event_map, config);
Mark Rutlande1f431b2011-04-28 15:47:10 +010080 case PERF_TYPE_HW_CACHE:
81 return armpmu_map_cache_event(cache_map, config);
82 case PERF_TYPE_RAW:
83 return armpmu_map_raw_event(raw_event_mask, config);
84 }
85
86 return -ENOENT;
Will Deacon84fee972010-11-13 17:13:56 +000087}
88
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +010089int armpmu_event_set_period(struct perf_event *event)
Jamie Iles1b8873a2010-02-02 20:25:44 +010090{
Mark Rutland8a16b342011-04-28 16:27:54 +010091 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +010092 struct hw_perf_event *hwc = &event->hw;
Peter Zijlstrae7850592010-05-21 14:43:08 +020093 s64 left = local64_read(&hwc->period_left);
Jamie Iles1b8873a2010-02-02 20:25:44 +010094 s64 period = hwc->sample_period;
95 int ret = 0;
96
Will Deacon3581fe02012-10-17 12:01:34 +010097 /* The period may have been changed by PERF_EVENT_IOC_PERIOD */
98 if (unlikely(period != hwc->last_period))
99 left = period - (hwc->last_period - left);
100
Jamie Iles1b8873a2010-02-02 20:25:44 +0100101 if (unlikely(left <= -period)) {
102 left = period;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200103 local64_set(&hwc->period_left, left);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100104 hwc->last_period = period;
105 ret = 1;
106 }
107
108 if (unlikely(left <= 0)) {
109 left += period;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200110 local64_set(&hwc->period_left, left);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100111 hwc->last_period = period;
112 ret = 1;
113 }
114
115 if (left > (s64)armpmu->max_period)
116 left = armpmu->max_period;
117
Peter Zijlstrae7850592010-05-21 14:43:08 +0200118 local64_set(&hwc->prev_count, (u64)-left);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100119
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100120 armpmu->write_counter(event, (u64)(-left) & 0xffffffff);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100121
122 perf_event_update_userpage(event);
123
124 return ret;
125}
126
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100127u64 armpmu_event_update(struct perf_event *event)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100128{
Mark Rutland8a16b342011-04-28 16:27:54 +0100129 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100130 struct hw_perf_event *hwc = &event->hw;
Will Deacona7378232011-03-25 17:12:37 +0100131 u64 delta, prev_raw_count, new_raw_count;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100132
133again:
Peter Zijlstrae7850592010-05-21 14:43:08 +0200134 prev_raw_count = local64_read(&hwc->prev_count);
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100135 new_raw_count = armpmu->read_counter(event);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100136
Peter Zijlstrae7850592010-05-21 14:43:08 +0200137 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
Jamie Iles1b8873a2010-02-02 20:25:44 +0100138 new_raw_count) != prev_raw_count)
139 goto again;
140
Will Deacon57273472012-03-06 17:33:17 +0100141 delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100142
Peter Zijlstrae7850592010-05-21 14:43:08 +0200143 local64_add(delta, &event->count);
144 local64_sub(delta, &hwc->period_left);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100145
146 return new_raw_count;
147}
148
149static void
Jamie Iles1b8873a2010-02-02 20:25:44 +0100150armpmu_read(struct perf_event *event)
151{
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100152 armpmu_event_update(event);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100153}
154
155static void
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200156armpmu_stop(struct perf_event *event, int flags)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100157{
Mark Rutland8a16b342011-04-28 16:27:54 +0100158 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100159 struct hw_perf_event *hwc = &event->hw;
160
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200161 /*
162 * ARM pmu always has to update the counter, so ignore
163 * PERF_EF_UPDATE, see comments in armpmu_start().
164 */
165 if (!(hwc->state & PERF_HES_STOPPED)) {
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100166 armpmu->disable(event);
167 armpmu_event_update(event);
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200168 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
169 }
170}
171
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100172static void armpmu_start(struct perf_event *event, int flags)
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200173{
Mark Rutland8a16b342011-04-28 16:27:54 +0100174 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200175 struct hw_perf_event *hwc = &event->hw;
176
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200177 /*
178 * ARM pmu always has to reprogram the period, so ignore
179 * PERF_EF_RELOAD, see the comment below.
180 */
181 if (flags & PERF_EF_RELOAD)
182 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
183
184 hwc->state = 0;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100185 /*
186 * Set the period again. Some counters can't be stopped, so when we
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200187 * were stopped we simply disabled the IRQ source and the counter
Jamie Iles1b8873a2010-02-02 20:25:44 +0100188 * may have been left counting. If we don't do this step then we may
189 * get an interrupt too soon or *way* too late if the overflow has
190 * happened since disabling.
191 */
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100192 armpmu_event_set_period(event);
193 armpmu->enable(event);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100194}
195
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200196static void
197armpmu_del(struct perf_event *event, int flags)
198{
Mark Rutland8a16b342011-04-28 16:27:54 +0100199 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100200 struct pmu_hw_events *hw_events = armpmu->get_hw_events();
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200201 struct hw_perf_event *hwc = &event->hw;
202 int idx = hwc->idx;
203
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200204 armpmu_stop(event, PERF_EF_UPDATE);
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100205 hw_events->events[idx] = NULL;
206 clear_bit(idx, hw_events->used_mask);
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200207
208 perf_event_update_userpage(event);
209}
210
Jamie Iles1b8873a2010-02-02 20:25:44 +0100211static int
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200212armpmu_add(struct perf_event *event, int flags)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100213{
Mark Rutland8a16b342011-04-28 16:27:54 +0100214 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100215 struct pmu_hw_events *hw_events = armpmu->get_hw_events();
Jamie Iles1b8873a2010-02-02 20:25:44 +0100216 struct hw_perf_event *hwc = &event->hw;
217 int idx;
218 int err = 0;
219
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200220 perf_pmu_disable(event->pmu);
Peter Zijlstra24cd7f52010-06-11 17:32:03 +0200221
Jamie Iles1b8873a2010-02-02 20:25:44 +0100222 /* If we don't have a space for the counter then finish early. */
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100223 idx = armpmu->get_event_idx(hw_events, event);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100224 if (idx < 0) {
225 err = idx;
226 goto out;
227 }
228
229 /*
230 * If there is an event in the counter we are going to use then make
231 * sure it is disabled.
232 */
233 event->hw.idx = idx;
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100234 armpmu->disable(event);
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100235 hw_events->events[idx] = event;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100236
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200237 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
238 if (flags & PERF_EF_START)
239 armpmu_start(event, PERF_EF_RELOAD);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100240
241 /* Propagate our changes to the userspace mapping. */
242 perf_event_update_userpage(event);
243
244out:
Peter Zijlstra33696fc2010-06-14 08:49:00 +0200245 perf_pmu_enable(event->pmu);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100246 return err;
247}
248
Jamie Iles1b8873a2010-02-02 20:25:44 +0100249static int
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100250validate_event(struct pmu_hw_events *hw_events,
Jamie Iles1b8873a2010-02-02 20:25:44 +0100251 struct perf_event *event)
252{
Mark Rutland8a16b342011-04-28 16:27:54 +0100253 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Mark Rutland7b9f72c2011-04-27 16:22:21 +0100254 struct pmu *leader_pmu = event->group_leader->pmu;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100255
Will Deaconcb2d8b32013-04-12 19:04:19 +0100256 if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF)
257 return 1;
258
259 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
Will Deacon65b47112010-09-02 09:32:08 +0100260 return 1;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100261
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100262 return armpmu->get_event_idx(hw_events, event) >= 0;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100263}
264
265static int
266validate_group(struct perf_event *event)
267{
268 struct perf_event *sibling, *leader = event->group_leader;
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100269 struct pmu_hw_events fake_pmu;
Will Deaconbce34d12011-11-17 15:05:14 +0000270 DECLARE_BITMAP(fake_used_mask, ARMPMU_MAX_HWEVENTS);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100271
Will Deaconbce34d12011-11-17 15:05:14 +0000272 /*
273 * Initialise the fake PMU. We only need to populate the
274 * used_mask for the purposes of validation.
275 */
276 memset(fake_used_mask, 0, sizeof(fake_used_mask));
277 fake_pmu.used_mask = fake_used_mask;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100278
279 if (!validate_event(&fake_pmu, leader))
Peter Zijlstraaa2bc1a2011-11-09 17:56:37 +0100280 return -EINVAL;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100281
282 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
283 if (!validate_event(&fake_pmu, sibling))
Peter Zijlstraaa2bc1a2011-11-09 17:56:37 +0100284 return -EINVAL;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100285 }
286
287 if (!validate_event(&fake_pmu, event))
Peter Zijlstraaa2bc1a2011-11-09 17:56:37 +0100288 return -EINVAL;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100289
290 return 0;
291}
292
Sudeep KarkadaNagesha051f1b12012-07-31 10:34:25 +0100293static irqreturn_t armpmu_dispatch_irq(int irq, void *dev)
Rabin Vincent0e25a5c2011-02-08 09:24:36 +0530294{
Mark Rutland8a16b342011-04-28 16:27:54 +0100295 struct arm_pmu *armpmu = (struct arm_pmu *) dev;
Mark Rutlanda9356a02011-05-04 09:23:15 +0100296 struct platform_device *plat_device = armpmu->plat_device;
297 struct arm_pmu_platdata *plat = dev_get_platdata(&plat_device->dev);
Rabin Vincent0e25a5c2011-02-08 09:24:36 +0530298
Sudeep KarkadaNagesha051f1b12012-07-31 10:34:25 +0100299 if (plat && plat->handle_irq)
300 return plat->handle_irq(irq, dev, armpmu->handle_irq);
301 else
302 return armpmu->handle_irq(irq, dev);
Rabin Vincent0e25a5c2011-02-08 09:24:36 +0530303}
304
Will Deacon0b390e22011-07-27 15:18:59 +0100305static void
Mark Rutland8a16b342011-04-28 16:27:54 +0100306armpmu_release_hardware(struct arm_pmu *armpmu)
Will Deacon0b390e22011-07-27 15:18:59 +0100307{
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100308 armpmu->free_irq(armpmu);
Sudeep KarkadaNagesha051f1b12012-07-31 10:34:25 +0100309 pm_runtime_put_sync(&armpmu->plat_device->dev);
Will Deacon0b390e22011-07-27 15:18:59 +0100310}
311
Jamie Iles1b8873a2010-02-02 20:25:44 +0100312static int
Mark Rutland8a16b342011-04-28 16:27:54 +0100313armpmu_reserve_hardware(struct arm_pmu *armpmu)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100314{
Sudeep KarkadaNagesha051f1b12012-07-31 10:34:25 +0100315 int err;
Mark Rutlanda9356a02011-05-04 09:23:15 +0100316 struct platform_device *pmu_device = armpmu->plat_device;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100317
Will Deacone5a21322011-11-22 18:01:46 +0000318 if (!pmu_device)
319 return -ENODEV;
320
Jon Hunter7be29582012-05-31 13:05:20 -0500321 pm_runtime_get_sync(&pmu_device->dev);
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100322 err = armpmu->request_irq(armpmu, armpmu_dispatch_irq);
Sudeep KarkadaNagesha051f1b12012-07-31 10:34:25 +0100323 if (err) {
324 armpmu_release_hardware(armpmu);
325 return err;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100326 }
327
Will Deacon0b390e22011-07-27 15:18:59 +0100328 return 0;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100329}
330
Jamie Iles1b8873a2010-02-02 20:25:44 +0100331static void
332hw_perf_event_destroy(struct perf_event *event)
333{
Mark Rutland8a16b342011-04-28 16:27:54 +0100334 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Mark Rutland03b78982011-04-27 11:20:11 +0100335 atomic_t *active_events = &armpmu->active_events;
336 struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;
337
338 if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
Mark Rutland8a16b342011-04-28 16:27:54 +0100339 armpmu_release_hardware(armpmu);
Mark Rutland03b78982011-04-27 11:20:11 +0100340 mutex_unlock(pmu_reserve_mutex);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100341 }
342}
343
344static int
Will Deacon05d22fd2011-07-19 11:57:30 +0100345event_requires_mode_exclusion(struct perf_event_attr *attr)
346{
347 return attr->exclude_idle || attr->exclude_user ||
348 attr->exclude_kernel || attr->exclude_hv;
349}
350
351static int
Jamie Iles1b8873a2010-02-02 20:25:44 +0100352__hw_perf_event_init(struct perf_event *event)
353{
Mark Rutland8a16b342011-04-28 16:27:54 +0100354 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100355 struct hw_perf_event *hwc = &event->hw;
Mark Rutland9dcbf462013-01-18 16:10:06 +0000356 int mapping;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100357
Mark Rutlande1f431b2011-04-28 15:47:10 +0100358 mapping = armpmu->map_event(event);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100359
360 if (mapping < 0) {
361 pr_debug("event %x:%llx not supported\n", event->attr.type,
362 event->attr.config);
363 return mapping;
364 }
365
366 /*
Will Deacon05d22fd2011-07-19 11:57:30 +0100367 * We don't assign an index until we actually place the event onto
368 * hardware. Use -1 to signify that we haven't decided where to put it
369 * yet. For SMP systems, each core has it's own PMU so we can't do any
370 * clever allocation or constraints checking at this point.
Jamie Iles1b8873a2010-02-02 20:25:44 +0100371 */
Will Deacon05d22fd2011-07-19 11:57:30 +0100372 hwc->idx = -1;
373 hwc->config_base = 0;
374 hwc->config = 0;
375 hwc->event_base = 0;
376
377 /*
378 * Check whether we need to exclude the counter from certain modes.
379 */
380 if ((!armpmu->set_event_filter ||
381 armpmu->set_event_filter(hwc, &event->attr)) &&
382 event_requires_mode_exclusion(&event->attr)) {
Jamie Iles1b8873a2010-02-02 20:25:44 +0100383 pr_debug("ARM performance counters do not support "
384 "mode exclusion\n");
Will Deaconfdeb8e32012-07-04 18:15:42 +0100385 return -EOPNOTSUPP;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100386 }
387
388 /*
Will Deacon05d22fd2011-07-19 11:57:30 +0100389 * Store the event encoding into the config_base field.
Jamie Iles1b8873a2010-02-02 20:25:44 +0100390 */
Will Deacon05d22fd2011-07-19 11:57:30 +0100391 hwc->config_base |= (unsigned long)mapping;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100392
393 if (!hwc->sample_period) {
Will Deacon57273472012-03-06 17:33:17 +0100394 /*
395 * For non-sampling runs, limit the sample_period to half
396 * of the counter width. That way, the new counter value
397 * is far less likely to overtake the previous one unless
398 * you have some serious IRQ latency issues.
399 */
400 hwc->sample_period = armpmu->max_period >> 1;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100401 hwc->last_period = hwc->sample_period;
Peter Zijlstrae7850592010-05-21 14:43:08 +0200402 local64_set(&hwc->period_left, hwc->sample_period);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100403 }
404
Jamie Iles1b8873a2010-02-02 20:25:44 +0100405 if (event->group_leader != event) {
Chen Gange595ede2013-02-28 17:51:29 +0100406 if (validate_group(event) != 0)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100407 return -EINVAL;
408 }
409
Mark Rutland9dcbf462013-01-18 16:10:06 +0000410 return 0;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100411}
412
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200413static int armpmu_event_init(struct perf_event *event)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100414{
Mark Rutland8a16b342011-04-28 16:27:54 +0100415 struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100416 int err = 0;
Mark Rutland03b78982011-04-27 11:20:11 +0100417 atomic_t *active_events = &armpmu->active_events;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100418
Stephane Eranian2481c5f2012-02-09 23:20:59 +0100419 /* does not support taken branch sampling */
420 if (has_branch_stack(event))
421 return -EOPNOTSUPP;
422
Mark Rutlande1f431b2011-04-28 15:47:10 +0100423 if (armpmu->map_event(event) == -ENOENT)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200424 return -ENOENT;
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200425
Jamie Iles1b8873a2010-02-02 20:25:44 +0100426 event->destroy = hw_perf_event_destroy;
427
Mark Rutland03b78982011-04-27 11:20:11 +0100428 if (!atomic_inc_not_zero(active_events)) {
429 mutex_lock(&armpmu->reserve_mutex);
430 if (atomic_read(active_events) == 0)
Mark Rutland8a16b342011-04-28 16:27:54 +0100431 err = armpmu_reserve_hardware(armpmu);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100432
433 if (!err)
Mark Rutland03b78982011-04-27 11:20:11 +0100434 atomic_inc(active_events);
435 mutex_unlock(&armpmu->reserve_mutex);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100436 }
437
438 if (err)
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200439 return err;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100440
441 err = __hw_perf_event_init(event);
442 if (err)
443 hw_perf_event_destroy(event);
444
Peter Zijlstrab0a873e2010-06-11 13:35:08 +0200445 return err;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100446}
447
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200448static void armpmu_enable(struct pmu *pmu)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100449{
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100450 struct arm_pmu *armpmu = to_arm_pmu(pmu);
Mark Rutland8be3f9a2011-05-17 11:20:11 +0100451 struct pmu_hw_events *hw_events = armpmu->get_hw_events();
Mark Rutland7325eae2011-08-23 11:59:49 +0100452 int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100453
Will Deaconf4f38432011-07-01 14:38:12 +0100454 if (enabled)
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100455 armpmu->start(armpmu);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100456}
457
Peter Zijlstraa4eaf7f2010-06-16 14:37:10 +0200458static void armpmu_disable(struct pmu *pmu)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100459{
Mark Rutland8a16b342011-04-28 16:27:54 +0100460 struct arm_pmu *armpmu = to_arm_pmu(pmu);
Sudeep KarkadaNageshaed6f2a52012-07-30 12:00:02 +0100461 armpmu->stop(armpmu);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100462}
463
Jon Hunter7be29582012-05-31 13:05:20 -0500464#ifdef CONFIG_PM_RUNTIME
465static int armpmu_runtime_resume(struct device *dev)
466{
467 struct arm_pmu_platdata *plat = dev_get_platdata(dev);
468
469 if (plat && plat->runtime_resume)
470 return plat->runtime_resume(dev);
471
472 return 0;
473}
474
475static int armpmu_runtime_suspend(struct device *dev)
476{
477 struct arm_pmu_platdata *plat = dev_get_platdata(dev);
478
479 if (plat && plat->runtime_suspend)
480 return plat->runtime_suspend(dev);
481
482 return 0;
483}
484#endif
485
Will Deacon6dbc0022012-07-29 12:36:28 +0100486const struct dev_pm_ops armpmu_dev_pm_ops = {
487 SET_RUNTIME_PM_OPS(armpmu_runtime_suspend, armpmu_runtime_resume, NULL)
488};
489
Stephen Boyd44d6b1f2013-03-05 03:54:06 +0100490static void armpmu_init(struct arm_pmu *armpmu)
Mark Rutland03b78982011-04-27 11:20:11 +0100491{
492 atomic_set(&armpmu->active_events, 0);
493 mutex_init(&armpmu->reserve_mutex);
Mark Rutland8a16b342011-04-28 16:27:54 +0100494
495 armpmu->pmu = (struct pmu) {
496 .pmu_enable = armpmu_enable,
497 .pmu_disable = armpmu_disable,
498 .event_init = armpmu_event_init,
499 .add = armpmu_add,
500 .del = armpmu_del,
501 .start = armpmu_start,
502 .stop = armpmu_stop,
503 .read = armpmu_read,
504 };
505}
506
Will Deacon03052302012-09-21 14:23:47 +0100507int armpmu_register(struct arm_pmu *armpmu, int type)
Mark Rutland8a16b342011-04-28 16:27:54 +0100508{
509 armpmu_init(armpmu);
Jon Hunter2ac29a12012-10-25 21:23:18 +0100510 pm_runtime_enable(&armpmu->plat_device->dev);
Will Deacon04236f92012-07-28 17:42:22 +0100511 pr_info("enabled with %s PMU driver, %d counters available\n",
512 armpmu->name, armpmu->num_events);
Will Deacon03052302012-09-21 14:23:47 +0100513 return perf_pmu_register(&armpmu->pmu, armpmu->name, type);
Mark Rutland03b78982011-04-27 11:20:11 +0100514}
515
Jamie Iles1b8873a2010-02-02 20:25:44 +0100516/*
517 * Callchain handling code.
518 */
Jamie Iles1b8873a2010-02-02 20:25:44 +0100519
520/*
521 * The registers we're interested in are at the end of the variable
522 * length saved register structure. The fp points at the end of this
523 * structure so the address of this struct is:
524 * (struct frame_tail *)(xxx->fp)-1
525 *
526 * This code has been adapted from the ARM OProfile support.
527 */
528struct frame_tail {
Will Deacon4d6b7a72010-11-30 18:15:53 +0100529 struct frame_tail __user *fp;
530 unsigned long sp;
531 unsigned long lr;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100532} __attribute__((packed));
533
534/*
535 * Get the return address for a single stackframe and return a pointer to the
536 * next frame tail.
537 */
Will Deacon4d6b7a72010-11-30 18:15:53 +0100538static struct frame_tail __user *
539user_backtrace(struct frame_tail __user *tail,
Jamie Iles1b8873a2010-02-02 20:25:44 +0100540 struct perf_callchain_entry *entry)
541{
542 struct frame_tail buftail;
543
544 /* Also check accessibility of one struct frame_tail beyond */
545 if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
546 return NULL;
547 if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail)))
548 return NULL;
549
Frederic Weisbecker70791ce2010-06-29 19:34:05 +0200550 perf_callchain_store(entry, buftail.lr);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100551
552 /*
553 * Frame pointers should strictly progress back up the stack
554 * (towards higher addresses).
555 */
Rabin Vincentcb061992011-02-09 11:35:12 +0100556 if (tail + 1 >= buftail.fp)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100557 return NULL;
558
559 return buftail.fp - 1;
560}
561
Frederic Weisbecker56962b4442010-06-30 23:03:51 +0200562void
563perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100564{
Will Deacon4d6b7a72010-11-30 18:15:53 +0100565 struct frame_tail __user *tail;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100566
Marc Zyngiere50c5412012-09-13 16:40:46 +0100567 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
568 /* We don't support guest os callchain now */
569 return;
570 }
Jamie Iles1b8873a2010-02-02 20:25:44 +0100571
Will Deacon4d6b7a72010-11-30 18:15:53 +0100572 tail = (struct frame_tail __user *)regs->ARM_fp - 1;
Jamie Iles1b8873a2010-02-02 20:25:44 +0100573
Sonny Rao860ad782011-04-18 22:12:59 +0100574 while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
575 tail && !((unsigned long)tail & 0x3))
Jamie Iles1b8873a2010-02-02 20:25:44 +0100576 tail = user_backtrace(tail, entry);
577}
578
579/*
580 * Gets called by walk_stackframe() for every stackframe. This will be called
581 * whist unwinding the stackframe and is like a subroutine return so we use
582 * the PC.
583 */
584static int
585callchain_trace(struct stackframe *fr,
586 void *data)
587{
588 struct perf_callchain_entry *entry = data;
Frederic Weisbecker70791ce2010-06-29 19:34:05 +0200589 perf_callchain_store(entry, fr->pc);
Jamie Iles1b8873a2010-02-02 20:25:44 +0100590 return 0;
591}
592
Frederic Weisbecker56962b4442010-06-30 23:03:51 +0200593void
594perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
Jamie Iles1b8873a2010-02-02 20:25:44 +0100595{
596 struct stackframe fr;
597
Marc Zyngiere50c5412012-09-13 16:40:46 +0100598 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
599 /* We don't support guest os callchain now */
600 return;
601 }
602
Jamie Iles1b8873a2010-02-02 20:25:44 +0100603 fr.fp = regs->ARM_fp;
604 fr.sp = regs->ARM_sp;
605 fr.lr = regs->ARM_lr;
606 fr.pc = regs->ARM_pc;
607 walk_stackframe(&fr, callchain_trace, entry);
608}
Marc Zyngiere50c5412012-09-13 16:40:46 +0100609
610unsigned long perf_instruction_pointer(struct pt_regs *regs)
611{
612 if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
613 return perf_guest_cbs->get_guest_ip();
614
615 return instruction_pointer(regs);
616}
617
618unsigned long perf_misc_flags(struct pt_regs *regs)
619{
620 int misc = 0;
621
622 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
623 if (perf_guest_cbs->is_user_mode())
624 misc |= PERF_RECORD_MISC_GUEST_USER;
625 else
626 misc |= PERF_RECORD_MISC_GUEST_KERNEL;
627 } else {
628 if (user_mode(regs))
629 misc |= PERF_RECORD_MISC_USER;
630 else
631 misc |= PERF_RECORD_MISC_KERNEL;
632 }
633
634 return misc;
635}