blob: 95c833e8dc8165400af0e8f4db76d397e26cbf57 [file] [log] [blame]
Deng-Cheng Zhu14f70012010-10-12 19:37:22 +08001/*
2 * Linux performance counter support for MIPS.
3 *
4 * Copyright (C) 2010 MIPS Technologies, Inc.
5 * Author: Deng-Cheng Zhu
6 *
7 * This code is based on the implementation for ARM, which is in turn
8 * based on the sparc64 perf event code and the x86 code. Performance
Deng-Cheng Zhu7e788d92010-10-12 19:37:23 +08009 * counter access is based on the MIPS Oprofile code. And the callchain
10 * support references the code of MIPS stacktrace.c.
Deng-Cheng Zhu14f70012010-10-12 19:37:22 +080011 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
15 */
16
17#include <linux/cpumask.h>
18#include <linux/interrupt.h>
19#include <linux/smp.h>
20#include <linux/kernel.h>
21#include <linux/perf_event.h>
22#include <linux/uaccess.h>
23
24#include <asm/irq.h>
25#include <asm/irq_regs.h>
26#include <asm/stacktrace.h>
27#include <asm/time.h> /* For perf_irq */
28
29/* These are for 32bit counters. For 64bit ones, define them accordingly. */
30#define MAX_PERIOD ((1ULL << 32) - 1)
31#define VALID_COUNT 0x7fffffff
32#define TOTAL_BITS 32
33#define HIGHEST_BIT 31
34
35#define MIPS_MAX_HWEVENTS 4
36
37struct cpu_hw_events {
38 /* Array of events on this cpu. */
39 struct perf_event *events[MIPS_MAX_HWEVENTS];
40
41 /*
42 * Set the bit (indexed by the counter number) when the counter
43 * is used for an event.
44 */
45 unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
46
47 /*
48 * The borrowed MSB for the performance counter. A MIPS performance
49 * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit
50 * counters) as a factor of determining whether a counter overflow
51 * should be signaled. So here we use a separate MSB for each
52 * counter to make things easy.
53 */
54 unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
55
56 /*
57 * Software copy of the control register for each performance counter.
58 * MIPS CPUs vary in performance counters. They use this differently,
59 * and even may not use it.
60 */
61 unsigned int saved_ctrl[MIPS_MAX_HWEVENTS];
62};
63DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
64 .saved_ctrl = {0},
65};
66
67/* The description of MIPS performance events. */
68struct mips_perf_event {
69 unsigned int event_id;
70 /*
71 * MIPS performance counters are indexed starting from 0.
72 * CNTR_EVEN indicates the indexes of the counters to be used are
73 * even numbers.
74 */
75 unsigned int cntr_mask;
76 #define CNTR_EVEN 0x55555555
77 #define CNTR_ODD 0xaaaaaaaa
78#ifdef CONFIG_MIPS_MT_SMP
79 enum {
80 T = 0,
81 V = 1,
82 P = 2,
83 } range;
84#else
85 #define T
86 #define V
87 #define P
88#endif
89};
90
91#define UNSUPPORTED_PERF_EVENT_ID 0xffffffff
92#define C(x) PERF_COUNT_HW_CACHE_##x
93
94struct mips_pmu {
95 const char *name;
96 int irq;
97 irqreturn_t (*handle_irq)(int irq, void *dev);
98 int (*handle_shared_irq)(void);
99 void (*start)(void);
100 void (*stop)(void);
101 int (*alloc_counter)(struct cpu_hw_events *cpuc,
102 struct hw_perf_event *hwc);
103 u64 (*read_counter)(unsigned int idx);
104 void (*write_counter)(unsigned int idx, u64 val);
105 void (*enable_event)(struct hw_perf_event *evt, int idx);
106 void (*disable_event)(int idx);
107 const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
108 const struct mips_perf_event (*cache_event_map)
109 [PERF_COUNT_HW_CACHE_MAX]
110 [PERF_COUNT_HW_CACHE_OP_MAX]
111 [PERF_COUNT_HW_CACHE_RESULT_MAX];
112 unsigned int num_counters;
113};
114
115static const struct mips_pmu *mipspmu;
116
117static int
118mipspmu_event_set_period(struct perf_event *event,
119 struct hw_perf_event *hwc,
120 int idx)
121{
122 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
123 s64 left = local64_read(&hwc->period_left);
124 s64 period = hwc->sample_period;
125 int ret = 0;
126 u64 uleft;
127 unsigned long flags;
128
129 if (unlikely(left <= -period)) {
130 left = period;
131 local64_set(&hwc->period_left, left);
132 hwc->last_period = period;
133 ret = 1;
134 }
135
136 if (unlikely(left <= 0)) {
137 left += period;
138 local64_set(&hwc->period_left, left);
139 hwc->last_period = period;
140 ret = 1;
141 }
142
143 if (left > (s64)MAX_PERIOD)
144 left = MAX_PERIOD;
145
146 local64_set(&hwc->prev_count, (u64)-left);
147
148 local_irq_save(flags);
149 uleft = (u64)(-left) & MAX_PERIOD;
150 uleft > VALID_COUNT ?
151 set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs);
152 mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT);
153 local_irq_restore(flags);
154
155 perf_event_update_userpage(event);
156
157 return ret;
158}
159
160static int mipspmu_enable(struct perf_event *event)
161{
162 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
163 struct hw_perf_event *hwc = &event->hw;
164 int idx;
165 int err = 0;
166
167 /* To look for a free counter for this event. */
168 idx = mipspmu->alloc_counter(cpuc, hwc);
169 if (idx < 0) {
170 err = idx;
171 goto out;
172 }
173
174 /*
175 * If there is an event in the counter we are going to use then
176 * make sure it is disabled.
177 */
178 event->hw.idx = idx;
179 mipspmu->disable_event(idx);
180 cpuc->events[idx] = event;
181
182 /* Set the period for the event. */
183 mipspmu_event_set_period(event, hwc, idx);
184
185 /* Enable the event. */
186 mipspmu->enable_event(hwc, idx);
187
188 /* Propagate our changes to the userspace mapping. */
189 perf_event_update_userpage(event);
190
191out:
192 return err;
193}
194
195static void mipspmu_event_update(struct perf_event *event,
196 struct hw_perf_event *hwc,
197 int idx)
198{
199 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
200 unsigned long flags;
201 int shift = 64 - TOTAL_BITS;
202 s64 prev_raw_count, new_raw_count;
203 s64 delta;
204
205again:
206 prev_raw_count = local64_read(&hwc->prev_count);
207 local_irq_save(flags);
208 /* Make the counter value be a "real" one. */
209 new_raw_count = mipspmu->read_counter(idx);
210 if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) {
211 new_raw_count &= VALID_COUNT;
212 clear_bit(idx, cpuc->msbs);
213 } else
214 new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT);
215 local_irq_restore(flags);
216
217 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
218 new_raw_count) != prev_raw_count)
219 goto again;
220
221 delta = (new_raw_count << shift) - (prev_raw_count << shift);
222 delta >>= shift;
223
224 local64_add(delta, &event->count);
225 local64_sub(delta, &hwc->period_left);
226
227 return;
228}
229
230static void mipspmu_disable(struct perf_event *event)
231{
232 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
233 struct hw_perf_event *hwc = &event->hw;
234 int idx = hwc->idx;
235
236
237 WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
238
239 /* We are working on a local event. */
240 mipspmu->disable_event(idx);
241
242 barrier();
243
244 mipspmu_event_update(event, hwc, idx);
245 cpuc->events[idx] = NULL;
246 clear_bit(idx, cpuc->used_mask);
247
248 perf_event_update_userpage(event);
249}
250
251static void mipspmu_unthrottle(struct perf_event *event)
252{
253 struct hw_perf_event *hwc = &event->hw;
254
255 mipspmu->enable_event(hwc, hwc->idx);
256}
257
258static void mipspmu_read(struct perf_event *event)
259{
260 struct hw_perf_event *hwc = &event->hw;
261
262 /* Don't read disabled counters! */
263 if (hwc->idx < 0)
264 return;
265
266 mipspmu_event_update(event, hwc, hwc->idx);
267}
268
269static struct pmu pmu = {
270 .enable = mipspmu_enable,
271 .disable = mipspmu_disable,
272 .unthrottle = mipspmu_unthrottle,
273 .read = mipspmu_read,
274};
275
276static atomic_t active_events = ATOMIC_INIT(0);
277static DEFINE_MUTEX(pmu_reserve_mutex);
278static int (*save_perf_irq)(void);
279
280static int mipspmu_get_irq(void)
281{
282 int err;
283
284 if (mipspmu->irq >= 0) {
285 /* Request my own irq handler. */
286 err = request_irq(mipspmu->irq, mipspmu->handle_irq,
287 IRQF_DISABLED | IRQF_NOBALANCING,
288 "mips_perf_pmu", NULL);
289 if (err) {
290 pr_warning("Unable to request IRQ%d for MIPS "
291 "performance counters!\n", mipspmu->irq);
292 }
293 } else if (cp0_perfcount_irq < 0) {
294 /*
295 * We are sharing the irq number with the timer interrupt.
296 */
297 save_perf_irq = perf_irq;
298 perf_irq = mipspmu->handle_shared_irq;
299 err = 0;
300 } else {
301 pr_warning("The platform hasn't properly defined its "
302 "interrupt controller.\n");
303 err = -ENOENT;
304 }
305
306 return err;
307}
308
309static void mipspmu_free_irq(void)
310{
311 if (mipspmu->irq >= 0)
312 free_irq(mipspmu->irq, NULL);
313 else if (cp0_perfcount_irq < 0)
314 perf_irq = save_perf_irq;
315}
316
317static inline unsigned int
318mipspmu_perf_event_encode(const struct mips_perf_event *pev)
319{
320/*
321 * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
322 * event_id.
323 */
324#ifdef CONFIG_MIPS_MT_SMP
325 return ((unsigned int)pev->range << 24) |
326 (pev->cntr_mask & 0xffff00) |
327 (pev->event_id & 0xff);
328#else
329 return (pev->cntr_mask & 0xffff00) |
330 (pev->event_id & 0xff);
331#endif
332}
333
334static const struct mips_perf_event *
335mipspmu_map_general_event(int idx)
336{
337 const struct mips_perf_event *pev;
338
339 pev = ((*mipspmu->general_event_map)[idx].event_id ==
340 UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) :
341 &(*mipspmu->general_event_map)[idx]);
342
343 return pev;
344}
345
346static const struct mips_perf_event *
347mipspmu_map_cache_event(u64 config)
348{
349 unsigned int cache_type, cache_op, cache_result;
350 const struct mips_perf_event *pev;
351
352 cache_type = (config >> 0) & 0xff;
353 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
354 return ERR_PTR(-EINVAL);
355
356 cache_op = (config >> 8) & 0xff;
357 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
358 return ERR_PTR(-EINVAL);
359
360 cache_result = (config >> 16) & 0xff;
361 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
362 return ERR_PTR(-EINVAL);
363
364 pev = &((*mipspmu->cache_event_map)
365 [cache_type]
366 [cache_op]
367 [cache_result]);
368
369 if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID)
370 return ERR_PTR(-EOPNOTSUPP);
371
372 return pev;
373
374}
375
376static int validate_event(struct cpu_hw_events *cpuc,
377 struct perf_event *event)
378{
379 struct hw_perf_event fake_hwc = event->hw;
380
381 if (event->pmu && event->pmu != &pmu)
382 return 0;
383
384 return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0;
385}
386
387static int validate_group(struct perf_event *event)
388{
389 struct perf_event *sibling, *leader = event->group_leader;
390 struct cpu_hw_events fake_cpuc;
391
392 memset(&fake_cpuc, 0, sizeof(fake_cpuc));
393
394 if (!validate_event(&fake_cpuc, leader))
395 return -ENOSPC;
396
397 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
398 if (!validate_event(&fake_cpuc, sibling))
399 return -ENOSPC;
400 }
401
402 if (!validate_event(&fake_cpuc, event))
403 return -ENOSPC;
404
405 return 0;
406}
407
408/*
409 * mipsxx/rm9000/loongson2 have different performance counters, they have
410 * specific low-level init routines.
411 */
412static int __hw_perf_event_init(struct perf_event *event);
413
414static void hw_perf_event_destroy(struct perf_event *event)
415{
416 if (atomic_dec_and_mutex_lock(&active_events,
417 &pmu_reserve_mutex)) {
418 /*
419 * We must not call the destroy function with interrupts
420 * disabled.
421 */
422 on_each_cpu(reset_counters,
423 (void *)(long)mipspmu->num_counters, 1);
424 mipspmu_free_irq();
425 mutex_unlock(&pmu_reserve_mutex);
426 }
427}
428
429const struct pmu *hw_perf_event_init(struct perf_event *event)
430{
431 int err = 0;
432
433 if (!mipspmu || event->cpu >= nr_cpumask_bits ||
434 (event->cpu >= 0 && !cpu_online(event->cpu)))
435 return ERR_PTR(-ENODEV);
436
437 if (!atomic_inc_not_zero(&active_events)) {
438 if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) {
439 atomic_dec(&active_events);
440 return ERR_PTR(-ENOSPC);
441 }
442
443 mutex_lock(&pmu_reserve_mutex);
444 if (atomic_read(&active_events) == 0)
445 err = mipspmu_get_irq();
446
447 if (!err)
448 atomic_inc(&active_events);
449 mutex_unlock(&pmu_reserve_mutex);
450 }
451
452 if (err)
453 return ERR_PTR(err);
454
455 err = __hw_perf_event_init(event);
456 if (err)
457 hw_perf_event_destroy(event);
458
459 return err ? ERR_PTR(err) : &pmu;
460}
461
462void hw_perf_enable(void)
463{
464 if (mipspmu)
465 mipspmu->start();
466}
467
468void hw_perf_disable(void)
469{
470 if (mipspmu)
471 mipspmu->stop();
472}
473
474/* This is needed by specific irq handlers in perf_event_*.c */
475static void
476handle_associated_event(struct cpu_hw_events *cpuc,
477 int idx, struct perf_sample_data *data, struct pt_regs *regs)
478{
479 struct perf_event *event = cpuc->events[idx];
480 struct hw_perf_event *hwc = &event->hw;
481
482 mipspmu_event_update(event, hwc, idx);
483 data->period = event->hw.last_period;
484 if (!mipspmu_event_set_period(event, hwc, idx))
485 return;
486
487 if (perf_event_overflow(event, 0, data, regs))
488 mipspmu->disable_event(idx);
489}
Deng-Cheng Zhu7e788d92010-10-12 19:37:23 +0800490
491/* Callchain handling code. */
492static inline void
493callchain_store(struct perf_callchain_entry *entry,
494 u64 ip)
495{
496 if (entry->nr < PERF_MAX_STACK_DEPTH)
497 entry->ip[entry->nr++] = ip;
498}
499
500/*
501 * Leave userspace callchain empty for now. When we find a way to trace
502 * the user stack callchains, we add here.
503 */
504static void
505perf_callchain_user(struct pt_regs *regs,
506 struct perf_callchain_entry *entry)
507{
508}
509
510static void save_raw_perf_callchain(struct perf_callchain_entry *entry,
511 unsigned long reg29)
512{
513 unsigned long *sp = (unsigned long *)reg29;
514 unsigned long addr;
515
516 while (!kstack_end(sp)) {
517 addr = *sp++;
518 if (__kernel_text_address(addr)) {
519 callchain_store(entry, addr);
520 if (entry->nr >= PERF_MAX_STACK_DEPTH)
521 break;
522 }
523 }
524}
525
526static void
527perf_callchain_kernel(struct pt_regs *regs,
528 struct perf_callchain_entry *entry)
529{
530 unsigned long sp = regs->regs[29];
531#ifdef CONFIG_KALLSYMS
532 unsigned long ra = regs->regs[31];
533 unsigned long pc = regs->cp0_epc;
534
535 callchain_store(entry, PERF_CONTEXT_KERNEL);
536 if (raw_show_trace || !__kernel_text_address(pc)) {
537 unsigned long stack_page =
538 (unsigned long)task_stack_page(current);
539 if (stack_page && sp >= stack_page &&
540 sp <= stack_page + THREAD_SIZE - 32)
541 save_raw_perf_callchain(entry, sp);
542 return;
543 }
544 do {
545 callchain_store(entry, pc);
546 if (entry->nr >= PERF_MAX_STACK_DEPTH)
547 break;
548 pc = unwind_stack(current, &sp, pc, &ra);
549 } while (pc);
550#else
551 callchain_store(entry, PERF_CONTEXT_KERNEL);
552 save_raw_perf_callchain(entry, sp);
553#endif
554}
555
556static void
557perf_do_callchain(struct pt_regs *regs,
558 struct perf_callchain_entry *entry)
559{
560 int is_user;
561
562 if (!regs)
563 return;
564
565 is_user = user_mode(regs);
566
567 if (!current || !current->pid)
568 return;
569
570 if (is_user && current->state != TASK_RUNNING)
571 return;
572
573 if (!is_user) {
574 perf_callchain_kernel(regs, entry);
575 if (current->mm)
576 regs = task_pt_regs(current);
577 else
578 regs = NULL;
579 }
580 if (regs)
581 perf_callchain_user(regs, entry);
582}
583
584static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
585
586struct perf_callchain_entry *
587perf_callchain(struct pt_regs *regs)
588{
589 struct perf_callchain_entry *entry = &__get_cpu_var(pmc_irq_entry);
590
591 entry->nr = 0;
592 perf_do_callchain(regs, entry);
593 return entry;
594}