blob: c05e10354bc95f832fd05a9280949d1b45398ddf [file] [log] [blame]
Thomas Gleixner0793a612008-12-04 20:12:29 +01001/*
2 * Performance counter core code
3 *
4 * Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
6 *
Peter Zijlstra7b732a72009-03-23 18:22:10 +01007 *
8 * For licensing details see kernel-base/COPYING
Thomas Gleixner0793a612008-12-04 20:12:29 +01009 */
10
11#include <linux/fs.h>
Peter Zijlstrab9cacc72009-03-25 12:30:22 +010012#include <linux/mm.h>
Thomas Gleixner0793a612008-12-04 20:12:29 +010013#include <linux/cpu.h>
14#include <linux/smp.h>
Ingo Molnar04289bb2008-12-11 08:38:42 +010015#include <linux/file.h>
Thomas Gleixner0793a612008-12-04 20:12:29 +010016#include <linux/poll.h>
17#include <linux/sysfs.h>
18#include <linux/ptrace.h>
19#include <linux/percpu.h>
Peter Zijlstrab9cacc72009-03-25 12:30:22 +010020#include <linux/vmstat.h>
21#include <linux/hardirq.h>
22#include <linux/rculist.h>
Thomas Gleixner0793a612008-12-04 20:12:29 +010023#include <linux/uaccess.h>
24#include <linux/syscalls.h>
25#include <linux/anon_inodes.h>
Ingo Molnaraa9c4c02008-12-17 14:10:57 +010026#include <linux/kernel_stat.h>
Thomas Gleixner0793a612008-12-04 20:12:29 +010027#include <linux/perf_counter.h>
Peter Zijlstra0a4a9392009-03-30 19:07:05 +020028#include <linux/dcache.h>
Thomas Gleixner0793a612008-12-04 20:12:29 +010029
Tim Blechmann4e193bd2009-03-14 14:29:25 +010030#include <asm/irq_regs.h>
31
Thomas Gleixner0793a612008-12-04 20:12:29 +010032/*
33 * Each CPU has a list of per CPU counters:
34 */
35DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
36
Ingo Molnar088e2852008-12-14 20:21:00 +010037int perf_max_counters __read_mostly = 1;
Thomas Gleixner0793a612008-12-04 20:12:29 +010038static int perf_reserved_percpu __read_mostly;
39static int perf_overcommit __read_mostly = 1;
40
41/*
42 * Mutex for (sysadmin-configurable) counter reservations:
43 */
44static DEFINE_MUTEX(perf_resource_mutex);
45
46/*
47 * Architecture provided APIs - weak aliases:
48 */
Ingo Molnar5c92d122008-12-11 13:21:10 +010049extern __weak const struct hw_perf_counter_ops *
Ingo Molnar621a01e2008-12-11 12:46:46 +010050hw_perf_counter_init(struct perf_counter *counter)
Thomas Gleixner0793a612008-12-04 20:12:29 +010051{
Paul Mackerrasff6f0542009-01-09 16:19:25 +110052 return NULL;
Thomas Gleixner0793a612008-12-04 20:12:29 +010053}
54
Ingo Molnar01b28382008-12-11 13:45:51 +010055u64 __weak hw_perf_save_disable(void) { return 0; }
Yinghai Lu01ea1cc2008-12-26 21:05:06 -080056void __weak hw_perf_restore(u64 ctrl) { barrier(); }
Paul Mackerras01d02872009-01-14 13:44:19 +110057void __weak hw_perf_counter_setup(int cpu) { barrier(); }
Paul Mackerras3cbed422009-01-09 16:43:42 +110058int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
59 struct perf_cpu_context *cpuctx,
60 struct perf_counter_context *ctx, int cpu)
61{
62 return 0;
63}
Thomas Gleixner0793a612008-12-04 20:12:29 +010064
Paul Mackerras4eb96fc2009-01-09 17:24:34 +110065void __weak perf_counter_print_debug(void) { }
66
Ingo Molnar04289bb2008-12-11 08:38:42 +010067static void
68list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
69{
70 struct perf_counter *group_leader = counter->group_leader;
71
72 /*
73 * Depending on whether it is a standalone or sibling counter,
74 * add it straight to the context's counter list, or to the group
75 * leader's sibling list:
76 */
77 if (counter->group_leader == counter)
78 list_add_tail(&counter->list_entry, &ctx->counter_list);
Peter Zijlstra5c148192009-03-25 12:30:23 +010079 else {
Ingo Molnar04289bb2008-12-11 08:38:42 +010080 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
Peter Zijlstra5c148192009-03-25 12:30:23 +010081 group_leader->nr_siblings++;
82 }
Peter Zijlstra592903c2009-03-13 12:21:36 +010083
84 list_add_rcu(&counter->event_entry, &ctx->event_list);
Ingo Molnar04289bb2008-12-11 08:38:42 +010085}
86
87static void
88list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
89{
90 struct perf_counter *sibling, *tmp;
91
92 list_del_init(&counter->list_entry);
Peter Zijlstra592903c2009-03-13 12:21:36 +010093 list_del_rcu(&counter->event_entry);
Ingo Molnar04289bb2008-12-11 08:38:42 +010094
Peter Zijlstra5c148192009-03-25 12:30:23 +010095 if (counter->group_leader != counter)
96 counter->group_leader->nr_siblings--;
97
Ingo Molnar04289bb2008-12-11 08:38:42 +010098 /*
99 * If this was a group counter with sibling counters then
100 * upgrade the siblings to singleton counters by adding them
101 * to the context list directly:
102 */
103 list_for_each_entry_safe(sibling, tmp,
104 &counter->sibling_list, list_entry) {
105
Peter Zijlstra75564232009-03-13 12:21:29 +0100106 list_move_tail(&sibling->list_entry, &ctx->counter_list);
Ingo Molnar04289bb2008-12-11 08:38:42 +0100107 sibling->group_leader = sibling;
108 }
109}
110
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100111static void
112counter_sched_out(struct perf_counter *counter,
113 struct perf_cpu_context *cpuctx,
114 struct perf_counter_context *ctx)
115{
116 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
117 return;
118
119 counter->state = PERF_COUNTER_STATE_INACTIVE;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100120 counter->tstamp_stopped = ctx->time_now;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100121 counter->hw_ops->disable(counter);
122 counter->oncpu = -1;
123
124 if (!is_software_counter(counter))
125 cpuctx->active_oncpu--;
126 ctx->nr_active--;
127 if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
128 cpuctx->exclusive = 0;
129}
130
Paul Mackerrasd859e292009-01-17 18:10:22 +1100131static void
132group_sched_out(struct perf_counter *group_counter,
133 struct perf_cpu_context *cpuctx,
134 struct perf_counter_context *ctx)
135{
136 struct perf_counter *counter;
137
138 if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
139 return;
140
141 counter_sched_out(group_counter, cpuctx, ctx);
142
143 /*
144 * Schedule out siblings (if any):
145 */
146 list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
147 counter_sched_out(counter, cpuctx, ctx);
148
149 if (group_counter->hw_event.exclusive)
150 cpuctx->exclusive = 0;
151}
152
Thomas Gleixner0793a612008-12-04 20:12:29 +0100153/*
154 * Cross CPU call to remove a performance counter
155 *
156 * We disable the counter on the hardware level first. After that we
157 * remove it from the context list.
158 */
Ingo Molnar04289bb2008-12-11 08:38:42 +0100159static void __perf_counter_remove_from_context(void *info)
Thomas Gleixner0793a612008-12-04 20:12:29 +0100160{
161 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
162 struct perf_counter *counter = info;
163 struct perf_counter_context *ctx = counter->ctx;
Ingo Molnar9b51f662008-12-12 13:49:45 +0100164 unsigned long flags;
Ingo Molnar5c92d122008-12-11 13:21:10 +0100165 u64 perf_flags;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100166
167 /*
168 * If this is a task context, we need to check whether it is
169 * the current task context of this cpu. If not it has been
170 * scheduled out before the smp call arrived.
171 */
172 if (ctx->task && cpuctx->task_ctx != ctx)
173 return;
174
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100175 curr_rq_lock_irq_save(&flags);
176 spin_lock(&ctx->lock);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100177
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100178 counter_sched_out(counter, cpuctx, ctx);
179
180 counter->task = NULL;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100181 ctx->nr_counters--;
182
183 /*
184 * Protect the list operation against NMI by disabling the
185 * counters on a global level. NOP for non NMI based counters.
186 */
Ingo Molnar01b28382008-12-11 13:45:51 +0100187 perf_flags = hw_perf_save_disable();
Ingo Molnar04289bb2008-12-11 08:38:42 +0100188 list_del_counter(counter, ctx);
Ingo Molnar01b28382008-12-11 13:45:51 +0100189 hw_perf_restore(perf_flags);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100190
191 if (!ctx->task) {
192 /*
193 * Allow more per task counters with respect to the
194 * reservation:
195 */
196 cpuctx->max_pertask =
197 min(perf_max_counters - ctx->nr_counters,
198 perf_max_counters - perf_reserved_percpu);
199 }
200
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100201 spin_unlock(&ctx->lock);
202 curr_rq_unlock_irq_restore(&flags);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100203}
204
205
206/*
207 * Remove the counter from a task's (or a CPU's) list of counters.
208 *
Paul Mackerrasd859e292009-01-17 18:10:22 +1100209 * Must be called with counter->mutex and ctx->mutex held.
Thomas Gleixner0793a612008-12-04 20:12:29 +0100210 *
211 * CPU counters are removed with a smp call. For task counters we only
212 * call when the task is on a CPU.
213 */
Ingo Molnar04289bb2008-12-11 08:38:42 +0100214static void perf_counter_remove_from_context(struct perf_counter *counter)
Thomas Gleixner0793a612008-12-04 20:12:29 +0100215{
216 struct perf_counter_context *ctx = counter->ctx;
217 struct task_struct *task = ctx->task;
218
219 if (!task) {
220 /*
221 * Per cpu counters are removed via an smp call and
222 * the removal is always sucessful.
223 */
224 smp_call_function_single(counter->cpu,
Ingo Molnar04289bb2008-12-11 08:38:42 +0100225 __perf_counter_remove_from_context,
Thomas Gleixner0793a612008-12-04 20:12:29 +0100226 counter, 1);
227 return;
228 }
229
230retry:
Ingo Molnar04289bb2008-12-11 08:38:42 +0100231 task_oncpu_function_call(task, __perf_counter_remove_from_context,
Thomas Gleixner0793a612008-12-04 20:12:29 +0100232 counter);
233
234 spin_lock_irq(&ctx->lock);
235 /*
236 * If the context is active we need to retry the smp call.
237 */
Ingo Molnar04289bb2008-12-11 08:38:42 +0100238 if (ctx->nr_active && !list_empty(&counter->list_entry)) {
Thomas Gleixner0793a612008-12-04 20:12:29 +0100239 spin_unlock_irq(&ctx->lock);
240 goto retry;
241 }
242
243 /*
244 * The lock prevents that this context is scheduled in so we
Ingo Molnar04289bb2008-12-11 08:38:42 +0100245 * can remove the counter safely, if the call above did not
Thomas Gleixner0793a612008-12-04 20:12:29 +0100246 * succeed.
247 */
Ingo Molnar04289bb2008-12-11 08:38:42 +0100248 if (!list_empty(&counter->list_entry)) {
Thomas Gleixner0793a612008-12-04 20:12:29 +0100249 ctx->nr_counters--;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100250 list_del_counter(counter, ctx);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100251 counter->task = NULL;
252 }
253 spin_unlock_irq(&ctx->lock);
254}
255
Paul Mackerrasd859e292009-01-17 18:10:22 +1100256/*
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100257 * Get the current time for this context.
258 * If this is a task context, we use the task's task clock,
259 * or for a per-cpu context, we use the cpu clock.
260 */
261static u64 get_context_time(struct perf_counter_context *ctx, int update)
262{
263 struct task_struct *curr = ctx->task;
264
265 if (!curr)
266 return cpu_clock(smp_processor_id());
267
268 return __task_delta_exec(curr, update) + curr->se.sum_exec_runtime;
269}
270
271/*
272 * Update the record of the current time in a context.
273 */
274static void update_context_time(struct perf_counter_context *ctx, int update)
275{
276 ctx->time_now = get_context_time(ctx, update) - ctx->time_lost;
277}
278
279/*
280 * Update the total_time_enabled and total_time_running fields for a counter.
281 */
282static void update_counter_times(struct perf_counter *counter)
283{
284 struct perf_counter_context *ctx = counter->ctx;
285 u64 run_end;
286
287 if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
288 counter->total_time_enabled = ctx->time_now -
289 counter->tstamp_enabled;
290 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
291 run_end = counter->tstamp_stopped;
292 else
293 run_end = ctx->time_now;
294 counter->total_time_running = run_end - counter->tstamp_running;
295 }
296}
297
298/*
299 * Update total_time_enabled and total_time_running for all counters in a group.
300 */
301static void update_group_times(struct perf_counter *leader)
302{
303 struct perf_counter *counter;
304
305 update_counter_times(leader);
306 list_for_each_entry(counter, &leader->sibling_list, list_entry)
307 update_counter_times(counter);
308}
309
310/*
Paul Mackerrasd859e292009-01-17 18:10:22 +1100311 * Cross CPU call to disable a performance counter
312 */
313static void __perf_counter_disable(void *info)
314{
315 struct perf_counter *counter = info;
316 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
317 struct perf_counter_context *ctx = counter->ctx;
318 unsigned long flags;
319
320 /*
321 * If this is a per-task counter, need to check whether this
322 * counter's task is the current task on this cpu.
323 */
324 if (ctx->task && cpuctx->task_ctx != ctx)
325 return;
326
327 curr_rq_lock_irq_save(&flags);
328 spin_lock(&ctx->lock);
329
330 /*
331 * If the counter is on, turn it off.
332 * If it is in error state, leave it in error state.
333 */
334 if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100335 update_context_time(ctx, 1);
336 update_counter_times(counter);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100337 if (counter == counter->group_leader)
338 group_sched_out(counter, cpuctx, ctx);
339 else
340 counter_sched_out(counter, cpuctx, ctx);
341 counter->state = PERF_COUNTER_STATE_OFF;
342 }
343
344 spin_unlock(&ctx->lock);
345 curr_rq_unlock_irq_restore(&flags);
346}
347
348/*
349 * Disable a counter.
350 */
351static void perf_counter_disable(struct perf_counter *counter)
352{
353 struct perf_counter_context *ctx = counter->ctx;
354 struct task_struct *task = ctx->task;
355
356 if (!task) {
357 /*
358 * Disable the counter on the cpu that it's on
359 */
360 smp_call_function_single(counter->cpu, __perf_counter_disable,
361 counter, 1);
362 return;
363 }
364
365 retry:
366 task_oncpu_function_call(task, __perf_counter_disable, counter);
367
368 spin_lock_irq(&ctx->lock);
369 /*
370 * If the counter is still active, we need to retry the cross-call.
371 */
372 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
373 spin_unlock_irq(&ctx->lock);
374 goto retry;
375 }
376
377 /*
378 * Since we have the lock this context can't be scheduled
379 * in, so we can change the state safely.
380 */
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100381 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
382 update_counter_times(counter);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100383 counter->state = PERF_COUNTER_STATE_OFF;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100384 }
Paul Mackerrasd859e292009-01-17 18:10:22 +1100385
386 spin_unlock_irq(&ctx->lock);
387}
388
389/*
390 * Disable a counter and all its children.
391 */
392static void perf_counter_disable_family(struct perf_counter *counter)
393{
394 struct perf_counter *child;
395
396 perf_counter_disable(counter);
397
398 /*
399 * Lock the mutex to protect the list of children
400 */
401 mutex_lock(&counter->mutex);
402 list_for_each_entry(child, &counter->child_list, child_list)
403 perf_counter_disable(child);
404 mutex_unlock(&counter->mutex);
405}
406
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100407static int
408counter_sched_in(struct perf_counter *counter,
409 struct perf_cpu_context *cpuctx,
410 struct perf_counter_context *ctx,
411 int cpu)
412{
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100413 if (counter->state <= PERF_COUNTER_STATE_OFF)
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100414 return 0;
415
416 counter->state = PERF_COUNTER_STATE_ACTIVE;
417 counter->oncpu = cpu; /* TODO: put 'cpu' into cpuctx->cpu */
418 /*
419 * The new state must be visible before we turn it on in the hardware:
420 */
421 smp_wmb();
422
423 if (counter->hw_ops->enable(counter)) {
424 counter->state = PERF_COUNTER_STATE_INACTIVE;
425 counter->oncpu = -1;
426 return -EAGAIN;
427 }
428
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100429 counter->tstamp_running += ctx->time_now - counter->tstamp_stopped;
430
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100431 if (!is_software_counter(counter))
432 cpuctx->active_oncpu++;
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100433 ctx->nr_active++;
434
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100435 if (counter->hw_event.exclusive)
436 cpuctx->exclusive = 1;
437
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100438 return 0;
439}
440
Thomas Gleixner0793a612008-12-04 20:12:29 +0100441/*
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100442 * Return 1 for a group consisting entirely of software counters,
443 * 0 if the group contains any hardware counters.
444 */
445static int is_software_only_group(struct perf_counter *leader)
446{
447 struct perf_counter *counter;
448
449 if (!is_software_counter(leader))
450 return 0;
Peter Zijlstra5c148192009-03-25 12:30:23 +0100451
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100452 list_for_each_entry(counter, &leader->sibling_list, list_entry)
453 if (!is_software_counter(counter))
454 return 0;
Peter Zijlstra5c148192009-03-25 12:30:23 +0100455
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100456 return 1;
457}
458
459/*
460 * Work out whether we can put this counter group on the CPU now.
461 */
462static int group_can_go_on(struct perf_counter *counter,
463 struct perf_cpu_context *cpuctx,
464 int can_add_hw)
465{
466 /*
467 * Groups consisting entirely of software counters can always go on.
468 */
469 if (is_software_only_group(counter))
470 return 1;
471 /*
472 * If an exclusive group is already on, no other hardware
473 * counters can go on.
474 */
475 if (cpuctx->exclusive)
476 return 0;
477 /*
478 * If this group is exclusive and there are already
479 * counters on the CPU, it can't go on.
480 */
481 if (counter->hw_event.exclusive && cpuctx->active_oncpu)
482 return 0;
483 /*
484 * Otherwise, try to add it if all previous groups were able
485 * to go on.
486 */
487 return can_add_hw;
488}
489
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100490static void add_counter_to_ctx(struct perf_counter *counter,
491 struct perf_counter_context *ctx)
492{
493 list_add_counter(counter, ctx);
494 ctx->nr_counters++;
495 counter->prev_state = PERF_COUNTER_STATE_OFF;
496 counter->tstamp_enabled = ctx->time_now;
497 counter->tstamp_running = ctx->time_now;
498 counter->tstamp_stopped = ctx->time_now;
499}
500
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100501/*
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100502 * Cross CPU call to install and enable a performance counter
Thomas Gleixner0793a612008-12-04 20:12:29 +0100503 */
504static void __perf_install_in_context(void *info)
505{
506 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
507 struct perf_counter *counter = info;
508 struct perf_counter_context *ctx = counter->ctx;
Paul Mackerrasd859e292009-01-17 18:10:22 +1100509 struct perf_counter *leader = counter->group_leader;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100510 int cpu = smp_processor_id();
Ingo Molnar9b51f662008-12-12 13:49:45 +0100511 unsigned long flags;
Ingo Molnar5c92d122008-12-11 13:21:10 +0100512 u64 perf_flags;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100513 int err;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100514
515 /*
516 * If this is a task context, we need to check whether it is
517 * the current task context of this cpu. If not it has been
518 * scheduled out before the smp call arrived.
519 */
520 if (ctx->task && cpuctx->task_ctx != ctx)
521 return;
522
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100523 curr_rq_lock_irq_save(&flags);
524 spin_lock(&ctx->lock);
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100525 update_context_time(ctx, 1);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100526
527 /*
528 * Protect the list operation against NMI by disabling the
529 * counters on a global level. NOP for non NMI based counters.
530 */
Ingo Molnar01b28382008-12-11 13:45:51 +0100531 perf_flags = hw_perf_save_disable();
Thomas Gleixner0793a612008-12-04 20:12:29 +0100532
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100533 add_counter_to_ctx(counter, ctx);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100534
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100535 /*
Paul Mackerrasd859e292009-01-17 18:10:22 +1100536 * Don't put the counter on if it is disabled or if
537 * it is in a group and the group isn't on.
538 */
539 if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
540 (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
541 goto unlock;
542
543 /*
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100544 * An exclusive counter can't go on if there are already active
545 * hardware counters, and no hardware counter can go on if there
546 * is already an exclusive counter on.
547 */
Paul Mackerrasd859e292009-01-17 18:10:22 +1100548 if (!group_can_go_on(counter, cpuctx, 1))
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100549 err = -EEXIST;
550 else
551 err = counter_sched_in(counter, cpuctx, ctx, cpu);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100552
Paul Mackerrasd859e292009-01-17 18:10:22 +1100553 if (err) {
554 /*
555 * This counter couldn't go on. If it is in a group
556 * then we have to pull the whole group off.
557 * If the counter group is pinned then put it in error state.
558 */
559 if (leader != counter)
560 group_sched_out(leader, cpuctx, ctx);
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100561 if (leader->hw_event.pinned) {
562 update_group_times(leader);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100563 leader->state = PERF_COUNTER_STATE_ERROR;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100564 }
Paul Mackerrasd859e292009-01-17 18:10:22 +1100565 }
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100566
567 if (!err && !ctx->task && cpuctx->max_pertask)
Thomas Gleixner0793a612008-12-04 20:12:29 +0100568 cpuctx->max_pertask--;
569
Paul Mackerrasd859e292009-01-17 18:10:22 +1100570 unlock:
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100571 hw_perf_restore(perf_flags);
572
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100573 spin_unlock(&ctx->lock);
574 curr_rq_unlock_irq_restore(&flags);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100575}
576
577/*
578 * Attach a performance counter to a context
579 *
580 * First we add the counter to the list with the hardware enable bit
581 * in counter->hw_config cleared.
582 *
583 * If the counter is attached to a task which is on a CPU we use a smp
584 * call to enable it in the task context. The task might have been
585 * scheduled away, but we check this in the smp call again.
Paul Mackerrasd859e292009-01-17 18:10:22 +1100586 *
587 * Must be called with ctx->mutex held.
Thomas Gleixner0793a612008-12-04 20:12:29 +0100588 */
589static void
590perf_install_in_context(struct perf_counter_context *ctx,
591 struct perf_counter *counter,
592 int cpu)
593{
594 struct task_struct *task = ctx->task;
595
Thomas Gleixner0793a612008-12-04 20:12:29 +0100596 if (!task) {
597 /*
598 * Per cpu counters are installed via an smp call and
599 * the install is always sucessful.
600 */
601 smp_call_function_single(cpu, __perf_install_in_context,
602 counter, 1);
603 return;
604 }
605
606 counter->task = task;
607retry:
608 task_oncpu_function_call(task, __perf_install_in_context,
609 counter);
610
611 spin_lock_irq(&ctx->lock);
612 /*
Thomas Gleixner0793a612008-12-04 20:12:29 +0100613 * we need to retry the smp call.
614 */
Paul Mackerrasd859e292009-01-17 18:10:22 +1100615 if (ctx->is_active && list_empty(&counter->list_entry)) {
Thomas Gleixner0793a612008-12-04 20:12:29 +0100616 spin_unlock_irq(&ctx->lock);
617 goto retry;
618 }
619
620 /*
621 * The lock prevents that this context is scheduled in so we
622 * can add the counter safely, if it the call above did not
623 * succeed.
624 */
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100625 if (list_empty(&counter->list_entry))
626 add_counter_to_ctx(counter, ctx);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100627 spin_unlock_irq(&ctx->lock);
628}
629
Paul Mackerrasd859e292009-01-17 18:10:22 +1100630/*
631 * Cross CPU call to enable a performance counter
632 */
633static void __perf_counter_enable(void *info)
Ingo Molnar04289bb2008-12-11 08:38:42 +0100634{
Paul Mackerrasd859e292009-01-17 18:10:22 +1100635 struct perf_counter *counter = info;
636 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
637 struct perf_counter_context *ctx = counter->ctx;
638 struct perf_counter *leader = counter->group_leader;
639 unsigned long flags;
640 int err;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100641
642 /*
Paul Mackerrasd859e292009-01-17 18:10:22 +1100643 * If this is a per-task counter, need to check whether this
644 * counter's task is the current task on this cpu.
Ingo Molnar04289bb2008-12-11 08:38:42 +0100645 */
Paul Mackerrasd859e292009-01-17 18:10:22 +1100646 if (ctx->task && cpuctx->task_ctx != ctx)
647 return;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100648
Paul Mackerrasd859e292009-01-17 18:10:22 +1100649 curr_rq_lock_irq_save(&flags);
650 spin_lock(&ctx->lock);
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100651 update_context_time(ctx, 1);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100652
Paul Mackerrasc07c99b2009-02-13 22:10:34 +1100653 counter->prev_state = counter->state;
Paul Mackerrasd859e292009-01-17 18:10:22 +1100654 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
655 goto unlock;
656 counter->state = PERF_COUNTER_STATE_INACTIVE;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100657 counter->tstamp_enabled = ctx->time_now - counter->total_time_enabled;
Paul Mackerrasd859e292009-01-17 18:10:22 +1100658
659 /*
660 * If the counter is in a group and isn't the group leader,
661 * then don't put it on unless the group is on.
662 */
663 if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
664 goto unlock;
665
666 if (!group_can_go_on(counter, cpuctx, 1))
667 err = -EEXIST;
668 else
669 err = counter_sched_in(counter, cpuctx, ctx,
670 smp_processor_id());
671
672 if (err) {
673 /*
674 * If this counter can't go on and it's part of a
675 * group, then the whole group has to come off.
676 */
677 if (leader != counter)
678 group_sched_out(leader, cpuctx, ctx);
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100679 if (leader->hw_event.pinned) {
680 update_group_times(leader);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100681 leader->state = PERF_COUNTER_STATE_ERROR;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100682 }
Paul Mackerrasd859e292009-01-17 18:10:22 +1100683 }
684
685 unlock:
686 spin_unlock(&ctx->lock);
687 curr_rq_unlock_irq_restore(&flags);
688}
689
690/*
691 * Enable a counter.
692 */
693static void perf_counter_enable(struct perf_counter *counter)
694{
695 struct perf_counter_context *ctx = counter->ctx;
696 struct task_struct *task = ctx->task;
697
698 if (!task) {
699 /*
700 * Enable the counter on the cpu that it's on
701 */
702 smp_call_function_single(counter->cpu, __perf_counter_enable,
703 counter, 1);
704 return;
705 }
706
707 spin_lock_irq(&ctx->lock);
708 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
709 goto out;
710
711 /*
712 * If the counter is in error state, clear that first.
713 * That way, if we see the counter in error state below, we
714 * know that it has gone back into error state, as distinct
715 * from the task having been scheduled away before the
716 * cross-call arrived.
717 */
718 if (counter->state == PERF_COUNTER_STATE_ERROR)
719 counter->state = PERF_COUNTER_STATE_OFF;
720
721 retry:
722 spin_unlock_irq(&ctx->lock);
723 task_oncpu_function_call(task, __perf_counter_enable, counter);
724
725 spin_lock_irq(&ctx->lock);
726
727 /*
728 * If the context is active and the counter is still off,
729 * we need to retry the cross-call.
730 */
731 if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
732 goto retry;
733
734 /*
735 * Since we have the lock this context can't be scheduled
736 * in, so we can change the state safely.
737 */
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100738 if (counter->state == PERF_COUNTER_STATE_OFF) {
Paul Mackerrasd859e292009-01-17 18:10:22 +1100739 counter->state = PERF_COUNTER_STATE_INACTIVE;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100740 counter->tstamp_enabled = ctx->time_now -
741 counter->total_time_enabled;
742 }
Paul Mackerrasd859e292009-01-17 18:10:22 +1100743 out:
744 spin_unlock_irq(&ctx->lock);
745}
746
Peter Zijlstra79f14642009-04-06 11:45:07 +0200747static void perf_counter_refresh(struct perf_counter *counter, int refresh)
748{
749 atomic_add(refresh, &counter->event_limit);
750 perf_counter_enable(counter);
751}
752
Paul Mackerrasd859e292009-01-17 18:10:22 +1100753/*
754 * Enable a counter and all its children.
755 */
756static void perf_counter_enable_family(struct perf_counter *counter)
757{
758 struct perf_counter *child;
759
760 perf_counter_enable(counter);
761
762 /*
763 * Lock the mutex to protect the list of children
764 */
765 mutex_lock(&counter->mutex);
766 list_for_each_entry(child, &counter->child_list, child_list)
767 perf_counter_enable(child);
768 mutex_unlock(&counter->mutex);
Ingo Molnar04289bb2008-12-11 08:38:42 +0100769}
770
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100771void __perf_counter_sched_out(struct perf_counter_context *ctx,
772 struct perf_cpu_context *cpuctx)
773{
774 struct perf_counter *counter;
Paul Mackerras3cbed422009-01-09 16:43:42 +1100775 u64 flags;
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100776
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100777 spin_lock(&ctx->lock);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100778 ctx->is_active = 0;
779 if (likely(!ctx->nr_counters))
780 goto out;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100781 update_context_time(ctx, 0);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100782
Paul Mackerras3cbed422009-01-09 16:43:42 +1100783 flags = hw_perf_save_disable();
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100784 if (ctx->nr_active) {
785 list_for_each_entry(counter, &ctx->counter_list, list_entry)
786 group_sched_out(counter, cpuctx, ctx);
787 }
Paul Mackerras3cbed422009-01-09 16:43:42 +1100788 hw_perf_restore(flags);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100789 out:
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100790 spin_unlock(&ctx->lock);
791}
792
Thomas Gleixner0793a612008-12-04 20:12:29 +0100793/*
794 * Called from scheduler to remove the counters of the current task,
795 * with interrupts disabled.
796 *
797 * We stop each counter and update the counter value in counter->count.
798 *
Ingo Molnar76715812008-12-17 14:20:28 +0100799 * This does not protect us against NMI, but disable()
Thomas Gleixner0793a612008-12-04 20:12:29 +0100800 * sets the disabled bit in the control field of counter _before_
801 * accessing the counter control register. If a NMI hits, then it will
802 * not restart the counter.
803 */
804void perf_counter_task_sched_out(struct task_struct *task, int cpu)
805{
806 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
807 struct perf_counter_context *ctx = &task->perf_counter_ctx;
Peter Zijlstra4a0deca2009-03-19 20:26:12 +0100808 struct pt_regs *regs;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100809
810 if (likely(!cpuctx->task_ctx))
811 return;
812
Peter Zijlstra4a0deca2009-03-19 20:26:12 +0100813 regs = task_pt_regs(task);
814 perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs);
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100815 __perf_counter_sched_out(ctx, cpuctx);
816
Thomas Gleixner0793a612008-12-04 20:12:29 +0100817 cpuctx->task_ctx = NULL;
818}
819
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100820static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
Ingo Molnar04289bb2008-12-11 08:38:42 +0100821{
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100822 __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
Ingo Molnar04289bb2008-12-11 08:38:42 +0100823}
824
Ingo Molnar79958882008-12-17 08:54:56 +0100825static int
Ingo Molnar04289bb2008-12-11 08:38:42 +0100826group_sched_in(struct perf_counter *group_counter,
827 struct perf_cpu_context *cpuctx,
828 struct perf_counter_context *ctx,
829 int cpu)
830{
Ingo Molnar95cdd2e2008-12-21 13:50:42 +0100831 struct perf_counter *counter, *partial_group;
Paul Mackerras3cbed422009-01-09 16:43:42 +1100832 int ret;
833
834 if (group_counter->state == PERF_COUNTER_STATE_OFF)
835 return 0;
836
837 ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
838 if (ret)
839 return ret < 0 ? ret : 0;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100840
Paul Mackerrasc07c99b2009-02-13 22:10:34 +1100841 group_counter->prev_state = group_counter->state;
Ingo Molnar95cdd2e2008-12-21 13:50:42 +0100842 if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
843 return -EAGAIN;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100844
845 /*
846 * Schedule in siblings as one group (if any):
847 */
Ingo Molnar79958882008-12-17 08:54:56 +0100848 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
Paul Mackerrasc07c99b2009-02-13 22:10:34 +1100849 counter->prev_state = counter->state;
Ingo Molnar95cdd2e2008-12-21 13:50:42 +0100850 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
851 partial_group = counter;
852 goto group_error;
853 }
Ingo Molnar79958882008-12-17 08:54:56 +0100854 }
855
Paul Mackerras3cbed422009-01-09 16:43:42 +1100856 return 0;
Ingo Molnar95cdd2e2008-12-21 13:50:42 +0100857
858group_error:
859 /*
860 * Groups can be scheduled in as one unit only, so undo any
861 * partial group before returning:
862 */
863 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
864 if (counter == partial_group)
865 break;
866 counter_sched_out(counter, cpuctx, ctx);
867 }
868 counter_sched_out(group_counter, cpuctx, ctx);
869
870 return -EAGAIN;
Ingo Molnar04289bb2008-12-11 08:38:42 +0100871}
872
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100873static void
874__perf_counter_sched_in(struct perf_counter_context *ctx,
875 struct perf_cpu_context *cpuctx, int cpu)
Thomas Gleixner0793a612008-12-04 20:12:29 +0100876{
Thomas Gleixner0793a612008-12-04 20:12:29 +0100877 struct perf_counter *counter;
Paul Mackerras3cbed422009-01-09 16:43:42 +1100878 u64 flags;
Paul Mackerrasdd0e6ba2009-01-12 15:11:00 +1100879 int can_add_hw = 1;
Thomas Gleixner0793a612008-12-04 20:12:29 +0100880
Thomas Gleixner0793a612008-12-04 20:12:29 +0100881 spin_lock(&ctx->lock);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100882 ctx->is_active = 1;
883 if (likely(!ctx->nr_counters))
884 goto out;
885
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100886 /*
887 * Add any time since the last sched_out to the lost time
888 * so it doesn't get included in the total_time_enabled and
889 * total_time_running measures for counters in the context.
890 */
891 ctx->time_lost = get_context_time(ctx, 0) - ctx->time_now;
892
Paul Mackerras3cbed422009-01-09 16:43:42 +1100893 flags = hw_perf_save_disable();
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100894
895 /*
896 * First go through the list and put on any pinned groups
897 * in order to give them the best chance of going on.
898 */
Ingo Molnar04289bb2008-12-11 08:38:42 +0100899 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100900 if (counter->state <= PERF_COUNTER_STATE_OFF ||
901 !counter->hw_event.pinned)
902 continue;
903 if (counter->cpu != -1 && counter->cpu != cpu)
904 continue;
905
906 if (group_can_go_on(counter, cpuctx, 1))
907 group_sched_in(counter, cpuctx, ctx, cpu);
908
909 /*
910 * If this pinned group hasn't been scheduled,
911 * put it in error state.
912 */
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100913 if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
914 update_group_times(counter);
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100915 counter->state = PERF_COUNTER_STATE_ERROR;
Paul Mackerras53cfbf52009-03-25 22:46:58 +1100916 }
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100917 }
918
919 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
920 /*
921 * Ignore counters in OFF or ERROR state, and
922 * ignore pinned counters since we did them already.
923 */
924 if (counter->state <= PERF_COUNTER_STATE_OFF ||
925 counter->hw_event.pinned)
926 continue;
927
Ingo Molnar04289bb2008-12-11 08:38:42 +0100928 /*
929 * Listen to the 'cpu' scheduling filter constraint
930 * of counters:
931 */
Thomas Gleixner0793a612008-12-04 20:12:29 +0100932 if (counter->cpu != -1 && counter->cpu != cpu)
933 continue;
934
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100935 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
Paul Mackerrasdd0e6ba2009-01-12 15:11:00 +1100936 if (group_sched_in(counter, cpuctx, ctx, cpu))
937 can_add_hw = 0;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100938 }
Thomas Gleixner0793a612008-12-04 20:12:29 +0100939 }
Paul Mackerras3cbed422009-01-09 16:43:42 +1100940 hw_perf_restore(flags);
Paul Mackerrasd859e292009-01-17 18:10:22 +1100941 out:
Thomas Gleixner0793a612008-12-04 20:12:29 +0100942 spin_unlock(&ctx->lock);
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100943}
Ingo Molnar04289bb2008-12-11 08:38:42 +0100944
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100945/*
946 * Called from scheduler to add the counters of the current task
947 * with interrupts disabled.
948 *
949 * We restore the counter value and then enable it.
950 *
951 * This does not protect us against NMI, but enable()
952 * sets the enabled bit in the control field of counter _before_
953 * accessing the counter control register. If a NMI hits, then it will
954 * keep the counter running.
955 */
956void perf_counter_task_sched_in(struct task_struct *task, int cpu)
957{
958 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
959 struct perf_counter_context *ctx = &task->perf_counter_ctx;
960
961 __perf_counter_sched_in(ctx, cpuctx, cpu);
Thomas Gleixner0793a612008-12-04 20:12:29 +0100962 cpuctx->task_ctx = ctx;
963}
964
Ingo Molnar235c7fc2008-12-21 14:43:25 +0100965static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
966{
967 struct perf_counter_context *ctx = &cpuctx->ctx;
968
969 __perf_counter_sched_in(ctx, cpuctx, cpu);
970}
971
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +0100972int perf_counter_task_disable(void)
973{
974 struct task_struct *curr = current;
975 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
976 struct perf_counter *counter;
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100977 unsigned long flags;
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +0100978 u64 perf_flags;
979 int cpu;
980
981 if (likely(!ctx->nr_counters))
982 return 0;
983
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100984 curr_rq_lock_irq_save(&flags);
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +0100985 cpu = smp_processor_id();
986
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100987 /* force the update of the task clock: */
988 __task_delta_exec(curr, 1);
989
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +0100990 perf_counter_task_sched_out(curr, cpu);
991
992 spin_lock(&ctx->lock);
993
994 /*
995 * Disable all the counters:
996 */
997 perf_flags = hw_perf_save_disable();
998
Paul Mackerras3b6f9e52009-01-14 21:00:30 +1100999 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001000 if (counter->state != PERF_COUNTER_STATE_ERROR) {
1001 update_group_times(counter);
Paul Mackerras3b6f9e52009-01-14 21:00:30 +11001002 counter->state = PERF_COUNTER_STATE_OFF;
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001003 }
Paul Mackerras3b6f9e52009-01-14 21:00:30 +11001004 }
Ingo Molnar9b51f662008-12-12 13:49:45 +01001005
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001006 hw_perf_restore(perf_flags);
1007
1008 spin_unlock(&ctx->lock);
1009
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001010 curr_rq_unlock_irq_restore(&flags);
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001011
1012 return 0;
1013}
1014
1015int perf_counter_task_enable(void)
1016{
1017 struct task_struct *curr = current;
1018 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1019 struct perf_counter *counter;
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001020 unsigned long flags;
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001021 u64 perf_flags;
1022 int cpu;
1023
1024 if (likely(!ctx->nr_counters))
1025 return 0;
1026
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001027 curr_rq_lock_irq_save(&flags);
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001028 cpu = smp_processor_id();
1029
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001030 /* force the update of the task clock: */
1031 __task_delta_exec(curr, 1);
1032
Ingo Molnar235c7fc2008-12-21 14:43:25 +01001033 perf_counter_task_sched_out(curr, cpu);
1034
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001035 spin_lock(&ctx->lock);
1036
1037 /*
1038 * Disable all the counters:
1039 */
1040 perf_flags = hw_perf_save_disable();
1041
1042 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
Paul Mackerras3b6f9e52009-01-14 21:00:30 +11001043 if (counter->state > PERF_COUNTER_STATE_OFF)
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001044 continue;
Ingo Molnar6a930702008-12-11 15:17:03 +01001045 counter->state = PERF_COUNTER_STATE_INACTIVE;
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001046 counter->tstamp_enabled = ctx->time_now -
1047 counter->total_time_enabled;
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001048 counter->hw_event.disabled = 0;
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001049 }
1050 hw_perf_restore(perf_flags);
1051
1052 spin_unlock(&ctx->lock);
1053
1054 perf_counter_task_sched_in(curr, cpu);
1055
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001056 curr_rq_unlock_irq_restore(&flags);
Ingo Molnar1d1c7dd2008-12-11 14:59:31 +01001057
1058 return 0;
1059}
1060
Ingo Molnar235c7fc2008-12-21 14:43:25 +01001061/*
1062 * Round-robin a context's counters:
1063 */
1064static void rotate_ctx(struct perf_counter_context *ctx)
Thomas Gleixner0793a612008-12-04 20:12:29 +01001065{
Thomas Gleixner0793a612008-12-04 20:12:29 +01001066 struct perf_counter *counter;
Ingo Molnar5c92d122008-12-11 13:21:10 +01001067 u64 perf_flags;
Thomas Gleixner0793a612008-12-04 20:12:29 +01001068
Ingo Molnar235c7fc2008-12-21 14:43:25 +01001069 if (!ctx->nr_counters)
Thomas Gleixner0793a612008-12-04 20:12:29 +01001070 return;
1071
Thomas Gleixner0793a612008-12-04 20:12:29 +01001072 spin_lock(&ctx->lock);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001073 /*
Ingo Molnar04289bb2008-12-11 08:38:42 +01001074 * Rotate the first entry last (works just fine for group counters too):
Thomas Gleixner0793a612008-12-04 20:12:29 +01001075 */
Ingo Molnar01b28382008-12-11 13:45:51 +01001076 perf_flags = hw_perf_save_disable();
Ingo Molnar04289bb2008-12-11 08:38:42 +01001077 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
Peter Zijlstra75564232009-03-13 12:21:29 +01001078 list_move_tail(&counter->list_entry, &ctx->counter_list);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001079 break;
1080 }
Ingo Molnar01b28382008-12-11 13:45:51 +01001081 hw_perf_restore(perf_flags);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001082
1083 spin_unlock(&ctx->lock);
Ingo Molnar235c7fc2008-12-21 14:43:25 +01001084}
Thomas Gleixner0793a612008-12-04 20:12:29 +01001085
Ingo Molnar235c7fc2008-12-21 14:43:25 +01001086void perf_counter_task_tick(struct task_struct *curr, int cpu)
1087{
1088 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
1089 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
1090 const int rotate_percpu = 0;
1091
1092 if (rotate_percpu)
1093 perf_counter_cpu_sched_out(cpuctx);
1094 perf_counter_task_sched_out(curr, cpu);
1095
1096 if (rotate_percpu)
1097 rotate_ctx(&cpuctx->ctx);
1098 rotate_ctx(ctx);
1099
1100 if (rotate_percpu)
1101 perf_counter_cpu_sched_in(cpuctx, cpu);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001102 perf_counter_task_sched_in(curr, cpu);
1103}
1104
1105/*
Thomas Gleixner0793a612008-12-04 20:12:29 +01001106 * Cross CPU call to read the hardware counter
1107 */
Ingo Molnar76715812008-12-17 14:20:28 +01001108static void __read(void *info)
Thomas Gleixner0793a612008-12-04 20:12:29 +01001109{
Ingo Molnar621a01e2008-12-11 12:46:46 +01001110 struct perf_counter *counter = info;
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001111 struct perf_counter_context *ctx = counter->ctx;
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001112 unsigned long flags;
Ingo Molnar621a01e2008-12-11 12:46:46 +01001113
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001114 curr_rq_lock_irq_save(&flags);
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001115 if (ctx->is_active)
1116 update_context_time(ctx, 1);
Ingo Molnar76715812008-12-17 14:20:28 +01001117 counter->hw_ops->read(counter);
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001118 update_counter_times(counter);
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01001119 curr_rq_unlock_irq_restore(&flags);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001120}
1121
Ingo Molnar04289bb2008-12-11 08:38:42 +01001122static u64 perf_counter_read(struct perf_counter *counter)
Thomas Gleixner0793a612008-12-04 20:12:29 +01001123{
1124 /*
1125 * If counter is enabled and currently active on a CPU, update the
1126 * value in the counter structure:
1127 */
Ingo Molnar6a930702008-12-11 15:17:03 +01001128 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
Thomas Gleixner0793a612008-12-04 20:12:29 +01001129 smp_call_function_single(counter->oncpu,
Ingo Molnar76715812008-12-17 14:20:28 +01001130 __read, counter, 1);
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001131 } else if (counter->state == PERF_COUNTER_STATE_INACTIVE) {
1132 update_counter_times(counter);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001133 }
1134
Ingo Molnaree060942008-12-13 09:00:03 +01001135 return atomic64_read(&counter->count);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001136}
1137
Thomas Gleixner0793a612008-12-04 20:12:29 +01001138static void put_context(struct perf_counter_context *ctx)
1139{
1140 if (ctx->task)
1141 put_task_struct(ctx->task);
1142}
1143
1144static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1145{
1146 struct perf_cpu_context *cpuctx;
1147 struct perf_counter_context *ctx;
1148 struct task_struct *task;
1149
1150 /*
1151 * If cpu is not a wildcard then this is a percpu counter:
1152 */
1153 if (cpu != -1) {
1154 /* Must be root to operate on a CPU counter: */
1155 if (!capable(CAP_SYS_ADMIN))
1156 return ERR_PTR(-EACCES);
1157
1158 if (cpu < 0 || cpu > num_possible_cpus())
1159 return ERR_PTR(-EINVAL);
1160
1161 /*
1162 * We could be clever and allow to attach a counter to an
1163 * offline CPU and activate it when the CPU comes up, but
1164 * that's for later.
1165 */
1166 if (!cpu_isset(cpu, cpu_online_map))
1167 return ERR_PTR(-ENODEV);
1168
1169 cpuctx = &per_cpu(perf_cpu_context, cpu);
1170 ctx = &cpuctx->ctx;
1171
Thomas Gleixner0793a612008-12-04 20:12:29 +01001172 return ctx;
1173 }
1174
1175 rcu_read_lock();
1176 if (!pid)
1177 task = current;
1178 else
1179 task = find_task_by_vpid(pid);
1180 if (task)
1181 get_task_struct(task);
1182 rcu_read_unlock();
1183
1184 if (!task)
1185 return ERR_PTR(-ESRCH);
1186
1187 ctx = &task->perf_counter_ctx;
1188 ctx->task = task;
1189
1190 /* Reuse ptrace permission checks for now. */
1191 if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1192 put_context(ctx);
1193 return ERR_PTR(-EACCES);
1194 }
1195
1196 return ctx;
1197}
1198
Peter Zijlstra592903c2009-03-13 12:21:36 +01001199static void free_counter_rcu(struct rcu_head *head)
1200{
1201 struct perf_counter *counter;
1202
1203 counter = container_of(head, struct perf_counter, rcu_head);
1204 kfree(counter);
1205}
1206
Peter Zijlstra925d5192009-03-30 19:07:02 +02001207static void perf_pending_sync(struct perf_counter *counter);
1208
Peter Zijlstraf1600952009-03-19 20:26:16 +01001209static void free_counter(struct perf_counter *counter)
1210{
Peter Zijlstra925d5192009-03-30 19:07:02 +02001211 perf_pending_sync(counter);
1212
Peter Zijlstrae077df42009-03-19 20:26:17 +01001213 if (counter->destroy)
1214 counter->destroy(counter);
1215
Peter Zijlstraf1600952009-03-19 20:26:16 +01001216 call_rcu(&counter->rcu_head, free_counter_rcu);
1217}
1218
Thomas Gleixner0793a612008-12-04 20:12:29 +01001219/*
1220 * Called when the last reference to the file is gone.
1221 */
1222static int perf_release(struct inode *inode, struct file *file)
1223{
1224 struct perf_counter *counter = file->private_data;
1225 struct perf_counter_context *ctx = counter->ctx;
1226
1227 file->private_data = NULL;
1228
Paul Mackerrasd859e292009-01-17 18:10:22 +11001229 mutex_lock(&ctx->mutex);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001230 mutex_lock(&counter->mutex);
1231
Ingo Molnar04289bb2008-12-11 08:38:42 +01001232 perf_counter_remove_from_context(counter);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001233
1234 mutex_unlock(&counter->mutex);
Paul Mackerrasd859e292009-01-17 18:10:22 +11001235 mutex_unlock(&ctx->mutex);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001236
Peter Zijlstraf1600952009-03-19 20:26:16 +01001237 free_counter(counter);
Mike Galbraith5af75912009-02-11 10:53:37 +01001238 put_context(ctx);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001239
1240 return 0;
1241}
1242
1243/*
1244 * Read the performance counter - simple non blocking version for now
1245 */
1246static ssize_t
1247perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1248{
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001249 u64 values[3];
1250 int n;
Thomas Gleixner0793a612008-12-04 20:12:29 +01001251
Paul Mackerras3b6f9e52009-01-14 21:00:30 +11001252 /*
1253 * Return end-of-file for a read on a counter that is in
1254 * error state (i.e. because it was pinned but it couldn't be
1255 * scheduled on to the CPU at some point).
1256 */
1257 if (counter->state == PERF_COUNTER_STATE_ERROR)
1258 return 0;
1259
Thomas Gleixner0793a612008-12-04 20:12:29 +01001260 mutex_lock(&counter->mutex);
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001261 values[0] = perf_counter_read(counter);
1262 n = 1;
1263 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1264 values[n++] = counter->total_time_enabled +
1265 atomic64_read(&counter->child_total_time_enabled);
1266 if (counter->hw_event.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1267 values[n++] = counter->total_time_running +
1268 atomic64_read(&counter->child_total_time_running);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001269 mutex_unlock(&counter->mutex);
1270
Paul Mackerras53cfbf52009-03-25 22:46:58 +11001271 if (count < n * sizeof(u64))
1272 return -EINVAL;
1273 count = n * sizeof(u64);
1274
1275 if (copy_to_user(buf, values, count))
1276 return -EFAULT;
1277
1278 return count;
Thomas Gleixner0793a612008-12-04 20:12:29 +01001279}
1280
1281static ssize_t
Thomas Gleixner0793a612008-12-04 20:12:29 +01001282perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1283{
1284 struct perf_counter *counter = file->private_data;
1285
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001286 return perf_read_hw(counter, buf, count);
Thomas Gleixner0793a612008-12-04 20:12:29 +01001287}
1288
1289static unsigned int perf_poll(struct file *file, poll_table *wait)
1290{
1291 struct perf_counter *counter = file->private_data;
Peter Zijlstrac7138f32009-03-24 13:18:16 +01001292 struct perf_mmap_data *data;
1293 unsigned int events;
1294
1295 rcu_read_lock();
1296 data = rcu_dereference(counter->data);
1297 if (data)
1298 events = atomic_xchg(&data->wakeup, 0);
1299 else
1300 events = POLL_HUP;
1301 rcu_read_unlock();
Thomas Gleixner0793a612008-12-04 20:12:29 +01001302
1303 poll_wait(file, &counter->waitq, wait);
1304
Thomas Gleixner0793a612008-12-04 20:12:29 +01001305 return events;
1306}
1307
Paul Mackerrasd859e292009-01-17 18:10:22 +11001308static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1309{
1310 struct perf_counter *counter = file->private_data;
1311 int err = 0;
1312
1313 switch (cmd) {
1314 case PERF_COUNTER_IOC_ENABLE:
1315 perf_counter_enable_family(counter);
1316 break;
1317 case PERF_COUNTER_IOC_DISABLE:
1318 perf_counter_disable_family(counter);
1319 break;
Peter Zijlstra79f14642009-04-06 11:45:07 +02001320 case PERF_COUNTER_IOC_REFRESH:
1321 perf_counter_refresh(counter, arg);
1322 break;
Paul Mackerrasd859e292009-01-17 18:10:22 +11001323 default:
1324 err = -ENOTTY;
1325 }
1326 return err;
1327}
1328
Peter Zijlstra38ff6672009-03-30 19:07:03 +02001329/*
1330 * Callers need to ensure there can be no nesting of this function, otherwise
1331 * the seqlock logic goes bad. We can not serialize this because the arch
1332 * code calls this from NMI context.
1333 */
1334void perf_counter_update_userpage(struct perf_counter *counter)
Paul Mackerras37d81822009-03-23 18:22:08 +01001335{
Peter Zijlstra38ff6672009-03-30 19:07:03 +02001336 struct perf_mmap_data *data;
1337 struct perf_counter_mmap_page *userpg;
1338
1339 rcu_read_lock();
1340 data = rcu_dereference(counter->data);
1341 if (!data)
1342 goto unlock;
1343
1344 userpg = data->user_page;
Paul Mackerras37d81822009-03-23 18:22:08 +01001345
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001346 /*
1347 * Disable preemption so as to not let the corresponding user-space
1348 * spin too long if we get preempted.
1349 */
1350 preempt_disable();
Paul Mackerras37d81822009-03-23 18:22:08 +01001351 ++userpg->lock;
Peter Zijlstra92f22a32009-04-02 11:12:04 +02001352 barrier();
Paul Mackerras37d81822009-03-23 18:22:08 +01001353 userpg->index = counter->hw.idx;
1354 userpg->offset = atomic64_read(&counter->count);
1355 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
1356 userpg->offset -= atomic64_read(&counter->hw.prev_count);
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001357
Peter Zijlstra92f22a32009-04-02 11:12:04 +02001358 barrier();
Paul Mackerras37d81822009-03-23 18:22:08 +01001359 ++userpg->lock;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001360 preempt_enable();
Peter Zijlstra38ff6672009-03-30 19:07:03 +02001361unlock:
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001362 rcu_read_unlock();
Paul Mackerras37d81822009-03-23 18:22:08 +01001363}
1364
1365static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1366{
1367 struct perf_counter *counter = vma->vm_file->private_data;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001368 struct perf_mmap_data *data;
1369 int ret = VM_FAULT_SIGBUS;
Paul Mackerras37d81822009-03-23 18:22:08 +01001370
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001371 rcu_read_lock();
1372 data = rcu_dereference(counter->data);
1373 if (!data)
1374 goto unlock;
Paul Mackerras37d81822009-03-23 18:22:08 +01001375
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001376 if (vmf->pgoff == 0) {
1377 vmf->page = virt_to_page(data->user_page);
1378 } else {
1379 int nr = vmf->pgoff - 1;
1380
1381 if ((unsigned)nr > data->nr_pages)
1382 goto unlock;
1383
1384 vmf->page = virt_to_page(data->data_pages[nr]);
1385 }
Paul Mackerras37d81822009-03-23 18:22:08 +01001386 get_page(vmf->page);
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001387 ret = 0;
1388unlock:
1389 rcu_read_unlock();
1390
1391 return ret;
1392}
1393
1394static int perf_mmap_data_alloc(struct perf_counter *counter, int nr_pages)
1395{
1396 struct perf_mmap_data *data;
1397 unsigned long size;
1398 int i;
1399
1400 WARN_ON(atomic_read(&counter->mmap_count));
1401
1402 size = sizeof(struct perf_mmap_data);
1403 size += nr_pages * sizeof(void *);
1404
1405 data = kzalloc(size, GFP_KERNEL);
1406 if (!data)
1407 goto fail;
1408
1409 data->user_page = (void *)get_zeroed_page(GFP_KERNEL);
1410 if (!data->user_page)
1411 goto fail_user_page;
1412
1413 for (i = 0; i < nr_pages; i++) {
1414 data->data_pages[i] = (void *)get_zeroed_page(GFP_KERNEL);
1415 if (!data->data_pages[i])
1416 goto fail_data_pages;
1417 }
1418
1419 data->nr_pages = nr_pages;
1420
1421 rcu_assign_pointer(counter->data, data);
1422
Paul Mackerras37d81822009-03-23 18:22:08 +01001423 return 0;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001424
1425fail_data_pages:
1426 for (i--; i >= 0; i--)
1427 free_page((unsigned long)data->data_pages[i]);
1428
1429 free_page((unsigned long)data->user_page);
1430
1431fail_user_page:
1432 kfree(data);
1433
1434fail:
1435 return -ENOMEM;
1436}
1437
1438static void __perf_mmap_data_free(struct rcu_head *rcu_head)
1439{
1440 struct perf_mmap_data *data = container_of(rcu_head,
1441 struct perf_mmap_data, rcu_head);
1442 int i;
1443
1444 free_page((unsigned long)data->user_page);
1445 for (i = 0; i < data->nr_pages; i++)
1446 free_page((unsigned long)data->data_pages[i]);
1447 kfree(data);
1448}
1449
1450static void perf_mmap_data_free(struct perf_counter *counter)
1451{
1452 struct perf_mmap_data *data = counter->data;
1453
1454 WARN_ON(atomic_read(&counter->mmap_count));
1455
1456 rcu_assign_pointer(counter->data, NULL);
1457 call_rcu(&data->rcu_head, __perf_mmap_data_free);
1458}
1459
1460static void perf_mmap_open(struct vm_area_struct *vma)
1461{
1462 struct perf_counter *counter = vma->vm_file->private_data;
1463
1464 atomic_inc(&counter->mmap_count);
1465}
1466
1467static void perf_mmap_close(struct vm_area_struct *vma)
1468{
1469 struct perf_counter *counter = vma->vm_file->private_data;
1470
1471 if (atomic_dec_and_mutex_lock(&counter->mmap_count,
1472 &counter->mmap_mutex)) {
Peter Zijlstraebb3c4c2009-04-06 11:45:05 +02001473 vma->vm_mm->locked_vm -= counter->data->nr_pages + 1;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001474 perf_mmap_data_free(counter);
1475 mutex_unlock(&counter->mmap_mutex);
1476 }
Paul Mackerras37d81822009-03-23 18:22:08 +01001477}
1478
1479static struct vm_operations_struct perf_mmap_vmops = {
Peter Zijlstraebb3c4c2009-04-06 11:45:05 +02001480 .open = perf_mmap_open,
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001481 .close = perf_mmap_close,
Paul Mackerras37d81822009-03-23 18:22:08 +01001482 .fault = perf_mmap_fault,
1483};
1484
1485static int perf_mmap(struct file *file, struct vm_area_struct *vma)
1486{
1487 struct perf_counter *counter = file->private_data;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001488 unsigned long vma_size;
1489 unsigned long nr_pages;
1490 unsigned long locked, lock_limit;
1491 int ret = 0;
Paul Mackerras37d81822009-03-23 18:22:08 +01001492
1493 if (!(vma->vm_flags & VM_SHARED) || (vma->vm_flags & VM_WRITE))
1494 return -EINVAL;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001495
1496 vma_size = vma->vm_end - vma->vm_start;
1497 nr_pages = (vma_size / PAGE_SIZE) - 1;
1498
Peter Zijlstra7730d862009-03-25 12:48:31 +01001499 /*
1500 * If we have data pages ensure they're a power-of-two number, so we
1501 * can do bitmasks instead of modulo.
1502 */
1503 if (nr_pages != 0 && !is_power_of_2(nr_pages))
Paul Mackerras37d81822009-03-23 18:22:08 +01001504 return -EINVAL;
1505
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001506 if (vma_size != PAGE_SIZE * (1 + nr_pages))
Paul Mackerras37d81822009-03-23 18:22:08 +01001507 return -EINVAL;
1508
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001509 if (vma->vm_pgoff != 0)
1510 return -EINVAL;
Paul Mackerras37d81822009-03-23 18:22:08 +01001511
Peter Zijlstraebb3c4c2009-04-06 11:45:05 +02001512 mutex_lock(&counter->mmap_mutex);
1513 if (atomic_inc_not_zero(&counter->mmap_count)) {
1514 if (nr_pages != counter->data->nr_pages)
1515 ret = -EINVAL;
1516 goto unlock;
1517 }
1518
1519 locked = vma->vm_mm->locked_vm;
1520 locked += nr_pages + 1;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001521
1522 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
1523 lock_limit >>= PAGE_SHIFT;
1524
Peter Zijlstraebb3c4c2009-04-06 11:45:05 +02001525 if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
1526 ret = -EPERM;
1527 goto unlock;
1528 }
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001529
1530 WARN_ON(counter->data);
1531 ret = perf_mmap_data_alloc(counter, nr_pages);
Peter Zijlstraebb3c4c2009-04-06 11:45:05 +02001532 if (ret)
1533 goto unlock;
1534
1535 atomic_set(&counter->mmap_count, 1);
1536 vma->vm_mm->locked_vm += nr_pages + 1;
1537unlock:
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001538 mutex_unlock(&counter->mmap_mutex);
Paul Mackerras37d81822009-03-23 18:22:08 +01001539
1540 vma->vm_flags &= ~VM_MAYWRITE;
1541 vma->vm_flags |= VM_RESERVED;
1542 vma->vm_ops = &perf_mmap_vmops;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001543
1544 return ret;
Paul Mackerras37d81822009-03-23 18:22:08 +01001545}
1546
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +02001547static int perf_fasync(int fd, struct file *filp, int on)
1548{
1549 struct perf_counter *counter = filp->private_data;
1550 struct inode *inode = filp->f_path.dentry->d_inode;
1551 int retval;
1552
1553 mutex_lock(&inode->i_mutex);
1554 retval = fasync_helper(fd, filp, on, &counter->fasync);
1555 mutex_unlock(&inode->i_mutex);
1556
1557 if (retval < 0)
1558 return retval;
1559
1560 return 0;
1561}
1562
Thomas Gleixner0793a612008-12-04 20:12:29 +01001563static const struct file_operations perf_fops = {
1564 .release = perf_release,
1565 .read = perf_read,
1566 .poll = perf_poll,
Paul Mackerrasd859e292009-01-17 18:10:22 +11001567 .unlocked_ioctl = perf_ioctl,
1568 .compat_ioctl = perf_ioctl,
Paul Mackerras37d81822009-03-23 18:22:08 +01001569 .mmap = perf_mmap,
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +02001570 .fasync = perf_fasync,
Thomas Gleixner0793a612008-12-04 20:12:29 +01001571};
1572
Peter Zijlstra15dbf272009-03-13 12:21:32 +01001573/*
Peter Zijlstra925d5192009-03-30 19:07:02 +02001574 * Perf counter wakeup
1575 *
1576 * If there's data, ensure we set the poll() state and publish everything
1577 * to user-space before waking everybody up.
1578 */
1579
1580void perf_counter_wakeup(struct perf_counter *counter)
1581{
1582 struct perf_mmap_data *data;
1583
1584 rcu_read_lock();
1585 data = rcu_dereference(counter->data);
1586 if (data) {
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +02001587 atomic_set(&data->wakeup, POLL_IN);
Peter Zijlstra38ff6672009-03-30 19:07:03 +02001588 /*
1589 * Ensure all data writes are issued before updating the
1590 * user-space data head information. The matching rmb()
1591 * will be in userspace after reading this value.
1592 */
1593 smp_wmb();
1594 data->user_page->data_head = atomic_read(&data->head);
Peter Zijlstra925d5192009-03-30 19:07:02 +02001595 }
1596 rcu_read_unlock();
1597
1598 wake_up_all(&counter->waitq);
Peter Zijlstra3c446b3d2009-04-06 11:45:01 +02001599 kill_fasync(&counter->fasync, SIGIO, POLL_IN);
Peter Zijlstra925d5192009-03-30 19:07:02 +02001600}
1601
1602/*
1603 * Pending wakeups
1604 *
1605 * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
1606 *
1607 * The NMI bit means we cannot possibly take locks. Therefore, maintain a
1608 * single linked list and use cmpxchg() to add entries lockless.
1609 */
1610
Peter Zijlstra79f14642009-04-06 11:45:07 +02001611static void perf_pending_counter(struct perf_pending_entry *entry)
1612{
1613 struct perf_counter *counter = container_of(entry,
1614 struct perf_counter, pending);
1615
1616 if (counter->pending_disable) {
1617 counter->pending_disable = 0;
1618 perf_counter_disable(counter);
1619 }
1620
1621 if (counter->pending_wakeup) {
1622 counter->pending_wakeup = 0;
1623 perf_counter_wakeup(counter);
1624 }
1625}
1626
Peter Zijlstra671dec52009-04-06 11:45:02 +02001627#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
Peter Zijlstra925d5192009-03-30 19:07:02 +02001628
Peter Zijlstra671dec52009-04-06 11:45:02 +02001629static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
Peter Zijlstra925d5192009-03-30 19:07:02 +02001630 PENDING_TAIL,
1631};
1632
Peter Zijlstra671dec52009-04-06 11:45:02 +02001633static void perf_pending_queue(struct perf_pending_entry *entry,
1634 void (*func)(struct perf_pending_entry *))
Peter Zijlstra925d5192009-03-30 19:07:02 +02001635{
Peter Zijlstra671dec52009-04-06 11:45:02 +02001636 struct perf_pending_entry **head;
Peter Zijlstra925d5192009-03-30 19:07:02 +02001637
Peter Zijlstra671dec52009-04-06 11:45:02 +02001638 if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
Peter Zijlstra925d5192009-03-30 19:07:02 +02001639 return;
1640
Peter Zijlstra671dec52009-04-06 11:45:02 +02001641 entry->func = func;
1642
1643 head = &get_cpu_var(perf_pending_head);
Peter Zijlstra925d5192009-03-30 19:07:02 +02001644
1645 do {
Peter Zijlstra671dec52009-04-06 11:45:02 +02001646 entry->next = *head;
1647 } while (cmpxchg(head, entry->next, entry) != entry->next);
Peter Zijlstra925d5192009-03-30 19:07:02 +02001648
1649 set_perf_counter_pending();
1650
Peter Zijlstra671dec52009-04-06 11:45:02 +02001651 put_cpu_var(perf_pending_head);
Peter Zijlstra925d5192009-03-30 19:07:02 +02001652}
1653
1654static int __perf_pending_run(void)
1655{
Peter Zijlstra671dec52009-04-06 11:45:02 +02001656 struct perf_pending_entry *list;
Peter Zijlstra925d5192009-03-30 19:07:02 +02001657 int nr = 0;
1658
Peter Zijlstra671dec52009-04-06 11:45:02 +02001659 list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
Peter Zijlstra925d5192009-03-30 19:07:02 +02001660 while (list != PENDING_TAIL) {
Peter Zijlstra671dec52009-04-06 11:45:02 +02001661 void (*func)(struct perf_pending_entry *);
1662 struct perf_pending_entry *entry = list;
Peter Zijlstra925d5192009-03-30 19:07:02 +02001663
1664 list = list->next;
1665
Peter Zijlstra671dec52009-04-06 11:45:02 +02001666 func = entry->func;
1667 entry->next = NULL;
Peter Zijlstra925d5192009-03-30 19:07:02 +02001668 /*
1669 * Ensure we observe the unqueue before we issue the wakeup,
1670 * so that we won't be waiting forever.
1671 * -- see perf_not_pending().
1672 */
1673 smp_wmb();
1674
Peter Zijlstra671dec52009-04-06 11:45:02 +02001675 func(entry);
Peter Zijlstra925d5192009-03-30 19:07:02 +02001676 nr++;
1677 }
1678
1679 return nr;
1680}
1681
1682static inline int perf_not_pending(struct perf_counter *counter)
1683{
1684 /*
1685 * If we flush on whatever cpu we run, there is a chance we don't
1686 * need to wait.
1687 */
1688 get_cpu();
1689 __perf_pending_run();
1690 put_cpu();
1691
1692 /*
1693 * Ensure we see the proper queue state before going to sleep
1694 * so that we do not miss the wakeup. -- see perf_pending_handle()
1695 */
1696 smp_rmb();
Peter Zijlstra671dec52009-04-06 11:45:02 +02001697 return counter->pending.next == NULL;
Peter Zijlstra925d5192009-03-30 19:07:02 +02001698}
1699
1700static void perf_pending_sync(struct perf_counter *counter)
1701{
1702 wait_event(counter->waitq, perf_not_pending(counter));
1703}
1704
1705void perf_counter_do_pending(void)
1706{
1707 __perf_pending_run();
1708}
1709
1710/*
Peter Zijlstra394ee072009-03-30 19:07:14 +02001711 * Callchain support -- arch specific
1712 */
1713
Peter Zijlstra9c03d882009-04-06 11:45:00 +02001714__weak struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
Peter Zijlstra394ee072009-03-30 19:07:14 +02001715{
1716 return NULL;
1717}
1718
1719/*
Peter Zijlstra0322cd62009-03-19 20:26:19 +01001720 * Output
1721 */
1722
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001723struct perf_output_handle {
1724 struct perf_counter *counter;
1725 struct perf_mmap_data *data;
1726 unsigned int offset;
Peter Zijlstra63e35b22009-03-25 12:30:24 +01001727 unsigned int head;
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001728 int wakeup;
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001729 int nmi;
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001730};
1731
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001732static inline void __perf_output_wakeup(struct perf_output_handle *handle)
1733{
Peter Zijlstra671dec52009-04-06 11:45:02 +02001734 if (handle->nmi) {
Peter Zijlstra79f14642009-04-06 11:45:07 +02001735 handle->counter->pending_wakeup = 1;
Peter Zijlstra671dec52009-04-06 11:45:02 +02001736 perf_pending_queue(&handle->counter->pending,
Peter Zijlstra79f14642009-04-06 11:45:07 +02001737 perf_pending_counter);
Peter Zijlstra671dec52009-04-06 11:45:02 +02001738 } else
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001739 perf_counter_wakeup(handle->counter);
1740}
1741
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001742static int perf_output_begin(struct perf_output_handle *handle,
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001743 struct perf_counter *counter, unsigned int size,
1744 int nmi)
Peter Zijlstra0322cd62009-03-19 20:26:19 +01001745{
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001746 struct perf_mmap_data *data;
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001747 unsigned int offset, head;
Peter Zijlstra0322cd62009-03-19 20:26:19 +01001748
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001749 rcu_read_lock();
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001750 data = rcu_dereference(counter->data);
1751 if (!data)
1752 goto out;
Peter Zijlstra0322cd62009-03-19 20:26:19 +01001753
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001754 handle->counter = counter;
1755 handle->nmi = nmi;
1756
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001757 if (!data->nr_pages)
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001758 goto fail;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001759
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001760 do {
1761 offset = head = atomic_read(&data->head);
Peter Zijlstrac7138f32009-03-24 13:18:16 +01001762 head += size;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001763 } while (atomic_cmpxchg(&data->head, offset, head) != offset);
1764
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001765 handle->data = data;
1766 handle->offset = offset;
Peter Zijlstra63e35b22009-03-25 12:30:24 +01001767 handle->head = head;
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001768 handle->wakeup = (offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT);
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001769
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001770 return 0;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001771
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001772fail:
1773 __perf_output_wakeup(handle);
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001774out:
1775 rcu_read_unlock();
1776
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001777 return -ENOSPC;
1778}
1779
1780static void perf_output_copy(struct perf_output_handle *handle,
1781 void *buf, unsigned int len)
1782{
1783 unsigned int pages_mask;
1784 unsigned int offset;
1785 unsigned int size;
1786 void **pages;
1787
1788 offset = handle->offset;
1789 pages_mask = handle->data->nr_pages - 1;
1790 pages = handle->data->data_pages;
1791
1792 do {
1793 unsigned int page_offset;
1794 int nr;
1795
1796 nr = (offset >> PAGE_SHIFT) & pages_mask;
1797 page_offset = offset & (PAGE_SIZE - 1);
1798 size = min_t(unsigned int, PAGE_SIZE - page_offset, len);
1799
1800 memcpy(pages[nr] + page_offset, buf, size);
1801
1802 len -= size;
1803 buf += size;
1804 offset += size;
1805 } while (len);
1806
1807 handle->offset = offset;
Peter Zijlstra63e35b22009-03-25 12:30:24 +01001808
1809 WARN_ON_ONCE(handle->offset > handle->head);
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001810}
1811
Peter Zijlstra5c148192009-03-25 12:30:23 +01001812#define perf_output_put(handle, x) \
1813 perf_output_copy((handle), &(x), sizeof(x))
1814
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001815static void perf_output_end(struct perf_output_handle *handle)
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001816{
Peter Zijlstrac4578102009-04-02 11:12:01 +02001817 int wakeup_events = handle->counter->hw_event.wakeup_events;
1818
1819 if (wakeup_events) {
1820 int events = atomic_inc_return(&handle->data->events);
1821 if (events >= wakeup_events) {
1822 atomic_sub(wakeup_events, &handle->data->events);
1823 __perf_output_wakeup(handle);
1824 }
1825 } else if (handle->wakeup)
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001826 __perf_output_wakeup(handle);
Peter Zijlstrab9cacc72009-03-25 12:30:22 +01001827 rcu_read_unlock();
1828}
1829
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02001830static void perf_counter_output(struct perf_counter *counter,
1831 int nmi, struct pt_regs *regs)
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001832{
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001833 int ret;
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001834 u64 record_type = counter->hw_event.record_type;
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001835 struct perf_output_handle handle;
1836 struct perf_event_header header;
1837 u64 ip;
Peter Zijlstra5c148192009-03-25 12:30:23 +01001838 struct {
Peter Zijlstraea5d20c2009-03-25 12:30:25 +01001839 u32 pid, tid;
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001840 } tid_entry;
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001841 struct {
1842 u64 event;
1843 u64 counter;
1844 } group_entry;
Peter Zijlstra394ee072009-03-30 19:07:14 +02001845 struct perf_callchain_entry *callchain = NULL;
1846 int callchain_size = 0;
Peter Zijlstra339f7c92009-04-06 11:45:06 +02001847 u64 time;
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001848
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001849 header.type = PERF_EVENT_COUNTER_OVERFLOW;
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001850 header.size = sizeof(header);
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001851
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001852 if (record_type & PERF_RECORD_IP) {
1853 ip = instruction_pointer(regs);
1854 header.type |= __PERF_EVENT_IP;
1855 header.size += sizeof(ip);
1856 }
Peter Zijlstraea5d20c2009-03-25 12:30:25 +01001857
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001858 if (record_type & PERF_RECORD_TID) {
Peter Zijlstraea5d20c2009-03-25 12:30:25 +01001859 /* namespace issues */
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001860 tid_entry.pid = current->group_leader->pid;
1861 tid_entry.tid = current->pid;
Peter Zijlstraea5d20c2009-03-25 12:30:25 +01001862
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001863 header.type |= __PERF_EVENT_TID;
1864 header.size += sizeof(tid_entry);
1865 }
Peter Zijlstraea5d20c2009-03-25 12:30:25 +01001866
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001867 if (record_type & PERF_RECORD_GROUP) {
1868 header.type |= __PERF_EVENT_GROUP;
1869 header.size += sizeof(u64) +
1870 counter->nr_siblings * sizeof(group_entry);
1871 }
1872
1873 if (record_type & PERF_RECORD_CALLCHAIN) {
Peter Zijlstra394ee072009-03-30 19:07:14 +02001874 callchain = perf_callchain(regs);
1875
1876 if (callchain) {
Peter Zijlstra9c03d882009-04-06 11:45:00 +02001877 callchain_size = (1 + callchain->nr) * sizeof(u64);
Peter Zijlstra394ee072009-03-30 19:07:14 +02001878
1879 header.type |= __PERF_EVENT_CALLCHAIN;
1880 header.size += callchain_size;
1881 }
1882 }
1883
Peter Zijlstra339f7c92009-04-06 11:45:06 +02001884 if (record_type & PERF_RECORD_TIME) {
1885 /*
1886 * Maybe do better on x86 and provide cpu_clock_nmi()
1887 */
1888 time = sched_clock();
1889
1890 header.type |= __PERF_EVENT_TIME;
1891 header.size += sizeof(u64);
1892 }
1893
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001894 ret = perf_output_begin(&handle, counter, header.size, nmi);
1895 if (ret)
1896 return;
Peter Zijlstraea5d20c2009-03-25 12:30:25 +01001897
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001898 perf_output_put(&handle, header);
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001899
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001900 if (record_type & PERF_RECORD_IP)
1901 perf_output_put(&handle, ip);
1902
1903 if (record_type & PERF_RECORD_TID)
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001904 perf_output_put(&handle, tid_entry);
1905
Peter Zijlstra8a057d82009-04-02 11:11:59 +02001906 if (record_type & PERF_RECORD_GROUP) {
1907 struct perf_counter *leader, *sub;
1908 u64 nr = counter->nr_siblings;
1909
1910 perf_output_put(&handle, nr);
1911
1912 leader = counter->group_leader;
1913 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1914 if (sub != counter)
1915 sub->hw_ops->read(sub);
1916
1917 group_entry.event = sub->hw_event.config;
1918 group_entry.counter = atomic64_read(&sub->count);
1919
1920 perf_output_put(&handle, group_entry);
1921 }
1922 }
1923
Peter Zijlstra394ee072009-03-30 19:07:14 +02001924 if (callchain)
1925 perf_output_copy(&handle, callchain, callchain_size);
1926
Peter Zijlstra339f7c92009-04-06 11:45:06 +02001927 if (record_type & PERF_RECORD_TIME)
1928 perf_output_put(&handle, time);
1929
Peter Zijlstra5ed00412009-03-30 19:07:12 +02001930 perf_output_end(&handle);
Peter Zijlstra7b732a72009-03-23 18:22:10 +01001931}
1932
Peter Zijlstra0322cd62009-03-19 20:26:19 +01001933/*
Peter Zijlstra0a4a9392009-03-30 19:07:05 +02001934 * mmap tracking
1935 */
1936
1937struct perf_mmap_event {
1938 struct file *file;
1939 char *file_name;
1940 int file_size;
1941
1942 struct {
1943 struct perf_event_header header;
1944
1945 u32 pid;
1946 u32 tid;
1947 u64 start;
1948 u64 len;
1949 u64 pgoff;
1950 } event;
1951};
1952
1953static void perf_counter_mmap_output(struct perf_counter *counter,
1954 struct perf_mmap_event *mmap_event)
1955{
1956 struct perf_output_handle handle;
1957 int size = mmap_event->event.header.size;
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001958 int ret = perf_output_begin(&handle, counter, size, 0);
Peter Zijlstra0a4a9392009-03-30 19:07:05 +02001959
1960 if (ret)
1961 return;
1962
1963 perf_output_put(&handle, mmap_event->event);
1964 perf_output_copy(&handle, mmap_event->file_name,
1965 mmap_event->file_size);
Peter Zijlstra78d613e2009-03-30 19:07:11 +02001966 perf_output_end(&handle);
Peter Zijlstra0a4a9392009-03-30 19:07:05 +02001967}
1968
1969static int perf_counter_mmap_match(struct perf_counter *counter,
1970 struct perf_mmap_event *mmap_event)
1971{
1972 if (counter->hw_event.mmap &&
1973 mmap_event->event.header.type == PERF_EVENT_MMAP)
1974 return 1;
1975
1976 if (counter->hw_event.munmap &&
1977 mmap_event->event.header.type == PERF_EVENT_MUNMAP)
1978 return 1;
1979
1980 return 0;
1981}
1982
1983static void perf_counter_mmap_ctx(struct perf_counter_context *ctx,
1984 struct perf_mmap_event *mmap_event)
1985{
1986 struct perf_counter *counter;
1987
1988 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
1989 return;
1990
1991 rcu_read_lock();
1992 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
1993 if (perf_counter_mmap_match(counter, mmap_event))
1994 perf_counter_mmap_output(counter, mmap_event);
1995 }
1996 rcu_read_unlock();
1997}
1998
1999static void perf_counter_mmap_event(struct perf_mmap_event *mmap_event)
2000{
2001 struct perf_cpu_context *cpuctx;
2002 struct file *file = mmap_event->file;
2003 unsigned int size;
2004 char tmp[16];
2005 char *buf = NULL;
2006 char *name;
2007
2008 if (file) {
2009 buf = kzalloc(PATH_MAX, GFP_KERNEL);
2010 if (!buf) {
2011 name = strncpy(tmp, "//enomem", sizeof(tmp));
2012 goto got_name;
2013 }
2014 name = dentry_path(file->f_dentry, buf, PATH_MAX);
2015 if (IS_ERR(name)) {
2016 name = strncpy(tmp, "//toolong", sizeof(tmp));
2017 goto got_name;
2018 }
2019 } else {
2020 name = strncpy(tmp, "//anon", sizeof(tmp));
2021 goto got_name;
2022 }
2023
2024got_name:
2025 size = ALIGN(strlen(name), sizeof(u64));
2026
2027 mmap_event->file_name = name;
2028 mmap_event->file_size = size;
2029
2030 mmap_event->event.header.size = sizeof(mmap_event->event) + size;
2031
2032 cpuctx = &get_cpu_var(perf_cpu_context);
2033 perf_counter_mmap_ctx(&cpuctx->ctx, mmap_event);
2034 put_cpu_var(perf_cpu_context);
2035
2036 perf_counter_mmap_ctx(&current->perf_counter_ctx, mmap_event);
2037
2038 kfree(buf);
2039}
2040
2041void perf_counter_mmap(unsigned long addr, unsigned long len,
2042 unsigned long pgoff, struct file *file)
2043{
2044 struct perf_mmap_event mmap_event = {
2045 .file = file,
2046 .event = {
2047 .header = { .type = PERF_EVENT_MMAP, },
2048 .pid = current->group_leader->pid,
2049 .tid = current->pid,
2050 .start = addr,
2051 .len = len,
2052 .pgoff = pgoff,
2053 },
2054 };
2055
2056 perf_counter_mmap_event(&mmap_event);
2057}
2058
2059void perf_counter_munmap(unsigned long addr, unsigned long len,
2060 unsigned long pgoff, struct file *file)
2061{
2062 struct perf_mmap_event mmap_event = {
2063 .file = file,
2064 .event = {
2065 .header = { .type = PERF_EVENT_MUNMAP, },
2066 .pid = current->group_leader->pid,
2067 .tid = current->pid,
2068 .start = addr,
2069 .len = len,
2070 .pgoff = pgoff,
2071 },
2072 };
2073
2074 perf_counter_mmap_event(&mmap_event);
2075}
2076
2077/*
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02002078 * Generic counter overflow handling.
2079 */
2080
2081int perf_counter_overflow(struct perf_counter *counter,
2082 int nmi, struct pt_regs *regs)
2083{
Peter Zijlstra79f14642009-04-06 11:45:07 +02002084 int events = atomic_read(&counter->event_limit);
2085 int ret = 0;
2086
2087 if (events && atomic_dec_and_test(&counter->event_limit)) {
2088 ret = 1;
2089 if (nmi) {
2090 counter->pending_disable = 1;
2091 perf_pending_queue(&counter->pending,
2092 perf_pending_counter);
2093 } else
2094 perf_counter_disable(counter);
2095 }
2096
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02002097 perf_counter_output(counter, nmi, regs);
Peter Zijlstra79f14642009-04-06 11:45:07 +02002098 return ret;
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02002099}
2100
2101/*
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002102 * Generic software counter infrastructure
2103 */
2104
2105static void perf_swcounter_update(struct perf_counter *counter)
2106{
2107 struct hw_perf_counter *hwc = &counter->hw;
2108 u64 prev, now;
2109 s64 delta;
2110
2111again:
2112 prev = atomic64_read(&hwc->prev_count);
2113 now = atomic64_read(&hwc->count);
2114 if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
2115 goto again;
2116
2117 delta = now - prev;
2118
2119 atomic64_add(delta, &counter->count);
2120 atomic64_sub(delta, &hwc->period_left);
2121}
2122
2123static void perf_swcounter_set_period(struct perf_counter *counter)
2124{
2125 struct hw_perf_counter *hwc = &counter->hw;
2126 s64 left = atomic64_read(&hwc->period_left);
2127 s64 period = hwc->irq_period;
2128
2129 if (unlikely(left <= -period)) {
2130 left = period;
2131 atomic64_set(&hwc->period_left, left);
2132 }
2133
2134 if (unlikely(left <= 0)) {
2135 left += period;
2136 atomic64_add(period, &hwc->period_left);
2137 }
2138
2139 atomic64_set(&hwc->prev_count, -left);
2140 atomic64_set(&hwc->count, -left);
2141}
2142
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002143static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
2144{
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02002145 enum hrtimer_restart ret = HRTIMER_RESTART;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002146 struct perf_counter *counter;
2147 struct pt_regs *regs;
2148
2149 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
2150 counter->hw_ops->read(counter);
2151
2152 regs = get_irq_regs();
2153 /*
2154 * In case we exclude kernel IPs or are somehow not in interrupt
2155 * context, provide the next best thing, the user IP.
2156 */
2157 if ((counter->hw_event.exclude_kernel || !regs) &&
2158 !counter->hw_event.exclude_user)
2159 regs = task_pt_regs(current);
2160
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02002161 if (regs) {
2162 if (perf_counter_overflow(counter, 0, regs))
2163 ret = HRTIMER_NORESTART;
2164 }
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002165
2166 hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
2167
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02002168 return ret;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002169}
2170
2171static void perf_swcounter_overflow(struct perf_counter *counter,
2172 int nmi, struct pt_regs *regs)
2173{
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002174 perf_swcounter_update(counter);
2175 perf_swcounter_set_period(counter);
Peter Zijlstraf6c7d5f2009-04-06 11:45:04 +02002176 if (perf_counter_overflow(counter, nmi, regs))
2177 /* soft-disable the counter */
2178 ;
2179
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002180}
2181
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002182static int perf_swcounter_match(struct perf_counter *counter,
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002183 enum perf_event_types type,
2184 u32 event, struct pt_regs *regs)
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002185{
2186 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
2187 return 0;
2188
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002189 if (perf_event_raw(&counter->hw_event))
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002190 return 0;
2191
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002192 if (perf_event_type(&counter->hw_event) != type)
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002193 return 0;
2194
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002195 if (perf_event_id(&counter->hw_event) != event)
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002196 return 0;
2197
2198 if (counter->hw_event.exclude_user && user_mode(regs))
2199 return 0;
2200
2201 if (counter->hw_event.exclude_kernel && !user_mode(regs))
2202 return 0;
2203
2204 return 1;
2205}
2206
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002207static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
2208 int nmi, struct pt_regs *regs)
2209{
2210 int neg = atomic64_add_negative(nr, &counter->hw.count);
2211 if (counter->hw.irq_period && !neg)
2212 perf_swcounter_overflow(counter, nmi, regs);
2213}
2214
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002215static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002216 enum perf_event_types type, u32 event,
2217 u64 nr, int nmi, struct pt_regs *regs)
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002218{
2219 struct perf_counter *counter;
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002220
Peter Zijlstra01ef09d2009-03-19 20:26:11 +01002221 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002222 return;
2223
Peter Zijlstra592903c2009-03-13 12:21:36 +01002224 rcu_read_lock();
2225 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002226 if (perf_swcounter_match(counter, type, event, regs))
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002227 perf_swcounter_add(counter, nr, nmi, regs);
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002228 }
Peter Zijlstra592903c2009-03-13 12:21:36 +01002229 rcu_read_unlock();
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002230}
2231
Peter Zijlstra96f6d442009-03-23 18:22:07 +01002232static int *perf_swcounter_recursion_context(struct perf_cpu_context *cpuctx)
2233{
2234 if (in_nmi())
2235 return &cpuctx->recursion[3];
2236
2237 if (in_irq())
2238 return &cpuctx->recursion[2];
2239
2240 if (in_softirq())
2241 return &cpuctx->recursion[1];
2242
2243 return &cpuctx->recursion[0];
2244}
2245
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002246static void __perf_swcounter_event(enum perf_event_types type, u32 event,
2247 u64 nr, int nmi, struct pt_regs *regs)
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002248{
2249 struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
Peter Zijlstra96f6d442009-03-23 18:22:07 +01002250 int *recursion = perf_swcounter_recursion_context(cpuctx);
2251
2252 if (*recursion)
2253 goto out;
2254
2255 (*recursion)++;
2256 barrier();
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002257
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002258 perf_swcounter_ctx_event(&cpuctx->ctx, type, event, nr, nmi, regs);
2259 if (cpuctx->task_ctx) {
2260 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
2261 nr, nmi, regs);
2262 }
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002263
Peter Zijlstra96f6d442009-03-23 18:22:07 +01002264 barrier();
2265 (*recursion)--;
2266
2267out:
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002268 put_cpu_var(perf_cpu_context);
2269}
2270
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002271void perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs)
2272{
2273 __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs);
2274}
2275
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002276static void perf_swcounter_read(struct perf_counter *counter)
2277{
2278 perf_swcounter_update(counter);
2279}
2280
2281static int perf_swcounter_enable(struct perf_counter *counter)
2282{
2283 perf_swcounter_set_period(counter);
2284 return 0;
2285}
2286
2287static void perf_swcounter_disable(struct perf_counter *counter)
2288{
2289 perf_swcounter_update(counter);
2290}
2291
Peter Zijlstraac17dc82009-03-13 12:21:34 +01002292static const struct hw_perf_counter_ops perf_ops_generic = {
2293 .enable = perf_swcounter_enable,
2294 .disable = perf_swcounter_disable,
2295 .read = perf_swcounter_read,
2296};
2297
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002298/*
2299 * Software counter: cpu wall time clock
2300 */
2301
Paul Mackerras9abf8a02009-01-09 16:26:43 +11002302static void cpu_clock_perf_counter_update(struct perf_counter *counter)
2303{
2304 int cpu = raw_smp_processor_id();
2305 s64 prev;
2306 u64 now;
2307
2308 now = cpu_clock(cpu);
2309 prev = atomic64_read(&counter->hw.prev_count);
2310 atomic64_set(&counter->hw.prev_count, now);
2311 atomic64_add(now - prev, &counter->count);
2312}
2313
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002314static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
2315{
2316 struct hw_perf_counter *hwc = &counter->hw;
2317 int cpu = raw_smp_processor_id();
2318
2319 atomic64_set(&hwc->prev_count, cpu_clock(cpu));
Peter Zijlstra039fc912009-03-13 16:43:47 +01002320 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2321 hwc->hrtimer.function = perf_swcounter_hrtimer;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002322 if (hwc->irq_period) {
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002323 __hrtimer_start_range_ns(&hwc->hrtimer,
2324 ns_to_ktime(hwc->irq_period), 0,
2325 HRTIMER_MODE_REL, 0);
2326 }
2327
2328 return 0;
2329}
2330
Ingo Molnar5c92d122008-12-11 13:21:10 +01002331static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
2332{
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002333 hrtimer_cancel(&counter->hw.hrtimer);
Paul Mackerras9abf8a02009-01-09 16:26:43 +11002334 cpu_clock_perf_counter_update(counter);
Ingo Molnar5c92d122008-12-11 13:21:10 +01002335}
2336
2337static void cpu_clock_perf_counter_read(struct perf_counter *counter)
2338{
Paul Mackerras9abf8a02009-01-09 16:26:43 +11002339 cpu_clock_perf_counter_update(counter);
Ingo Molnar5c92d122008-12-11 13:21:10 +01002340}
2341
2342static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
Ingo Molnar76715812008-12-17 14:20:28 +01002343 .enable = cpu_clock_perf_counter_enable,
2344 .disable = cpu_clock_perf_counter_disable,
2345 .read = cpu_clock_perf_counter_read,
Ingo Molnar5c92d122008-12-11 13:21:10 +01002346};
2347
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01002348/*
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002349 * Software counter: task time clock
2350 */
2351
2352/*
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01002353 * Called from within the scheduler:
2354 */
2355static u64 task_clock_perf_counter_val(struct perf_counter *counter, int update)
Ingo Molnarbae43c92008-12-11 14:03:20 +01002356{
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01002357 struct task_struct *curr = counter->task;
2358 u64 delta;
2359
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01002360 delta = __task_delta_exec(curr, update);
2361
2362 return curr->se.sum_exec_runtime + delta;
2363}
2364
2365static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
2366{
2367 u64 prev;
Ingo Molnar8cb391e2008-12-14 12:22:31 +01002368 s64 delta;
Ingo Molnarbae43c92008-12-11 14:03:20 +01002369
Ingo Molnar8cb391e2008-12-14 12:22:31 +01002370 prev = atomic64_read(&counter->hw.prev_count);
Ingo Molnar8cb391e2008-12-14 12:22:31 +01002371
2372 atomic64_set(&counter->hw.prev_count, now);
2373
2374 delta = now - prev;
Ingo Molnar8cb391e2008-12-14 12:22:31 +01002375
2376 atomic64_add(delta, &counter->count);
Ingo Molnarbae43c92008-12-11 14:03:20 +01002377}
2378
Ingo Molnar95cdd2e2008-12-21 13:50:42 +01002379static int task_clock_perf_counter_enable(struct perf_counter *counter)
Ingo Molnar8cb391e2008-12-14 12:22:31 +01002380{
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002381 struct hw_perf_counter *hwc = &counter->hw;
2382
2383 atomic64_set(&hwc->prev_count, task_clock_perf_counter_val(counter, 0));
Peter Zijlstra039fc912009-03-13 16:43:47 +01002384 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2385 hwc->hrtimer.function = perf_swcounter_hrtimer;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002386 if (hwc->irq_period) {
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002387 __hrtimer_start_range_ns(&hwc->hrtimer,
2388 ns_to_ktime(hwc->irq_period), 0,
2389 HRTIMER_MODE_REL, 0);
2390 }
Ingo Molnar95cdd2e2008-12-21 13:50:42 +01002391
2392 return 0;
Ingo Molnar8cb391e2008-12-14 12:22:31 +01002393}
2394
2395static void task_clock_perf_counter_disable(struct perf_counter *counter)
2396{
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002397 hrtimer_cancel(&counter->hw.hrtimer);
2398 task_clock_perf_counter_update(counter,
2399 task_clock_perf_counter_val(counter, 0));
2400}
Ingo Molnaraa9c4c02008-12-17 14:10:57 +01002401
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002402static void task_clock_perf_counter_read(struct perf_counter *counter)
2403{
2404 task_clock_perf_counter_update(counter,
2405 task_clock_perf_counter_val(counter, 1));
Ingo Molnarbae43c92008-12-11 14:03:20 +01002406}
2407
2408static const struct hw_perf_counter_ops perf_ops_task_clock = {
Ingo Molnar76715812008-12-17 14:20:28 +01002409 .enable = task_clock_perf_counter_enable,
2410 .disable = task_clock_perf_counter_disable,
2411 .read = task_clock_perf_counter_read,
Ingo Molnarbae43c92008-12-11 14:03:20 +01002412};
2413
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002414/*
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002415 * Software counter: cpu migrations
2416 */
2417
Paul Mackerras23a185c2009-02-09 22:42:47 +11002418static inline u64 get_cpu_migrations(struct perf_counter *counter)
Ingo Molnar6c594c22008-12-14 12:34:15 +01002419{
Paul Mackerras23a185c2009-02-09 22:42:47 +11002420 struct task_struct *curr = counter->ctx->task;
2421
2422 if (curr)
2423 return curr->se.nr_migrations;
2424 return cpu_nr_migrations(smp_processor_id());
Ingo Molnar6c594c22008-12-14 12:34:15 +01002425}
2426
2427static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
2428{
2429 u64 prev, now;
2430 s64 delta;
2431
2432 prev = atomic64_read(&counter->hw.prev_count);
Paul Mackerras23a185c2009-02-09 22:42:47 +11002433 now = get_cpu_migrations(counter);
Ingo Molnar6c594c22008-12-14 12:34:15 +01002434
2435 atomic64_set(&counter->hw.prev_count, now);
2436
2437 delta = now - prev;
Ingo Molnar6c594c22008-12-14 12:34:15 +01002438
2439 atomic64_add(delta, &counter->count);
2440}
2441
2442static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
2443{
2444 cpu_migrations_perf_counter_update(counter);
2445}
2446
Ingo Molnar95cdd2e2008-12-21 13:50:42 +01002447static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
Ingo Molnar6c594c22008-12-14 12:34:15 +01002448{
Paul Mackerrasc07c99b2009-02-13 22:10:34 +11002449 if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
2450 atomic64_set(&counter->hw.prev_count,
2451 get_cpu_migrations(counter));
Ingo Molnar95cdd2e2008-12-21 13:50:42 +01002452 return 0;
Ingo Molnar6c594c22008-12-14 12:34:15 +01002453}
2454
2455static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
2456{
2457 cpu_migrations_perf_counter_update(counter);
2458}
2459
2460static const struct hw_perf_counter_ops perf_ops_cpu_migrations = {
Ingo Molnar76715812008-12-17 14:20:28 +01002461 .enable = cpu_migrations_perf_counter_enable,
2462 .disable = cpu_migrations_perf_counter_disable,
2463 .read = cpu_migrations_perf_counter_read,
Ingo Molnar6c594c22008-12-14 12:34:15 +01002464};
2465
Peter Zijlstrae077df42009-03-19 20:26:17 +01002466#ifdef CONFIG_EVENT_PROFILE
2467void perf_tpcounter_event(int event_id)
2468{
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002469 struct pt_regs *regs = get_irq_regs();
2470
2471 if (!regs)
2472 regs = task_pt_regs(current);
2473
2474 __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs);
Peter Zijlstrae077df42009-03-19 20:26:17 +01002475}
2476
2477extern int ftrace_profile_enable(int);
2478extern void ftrace_profile_disable(int);
2479
2480static void tp_perf_counter_destroy(struct perf_counter *counter)
2481{
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002482 ftrace_profile_disable(perf_event_id(&counter->hw_event));
Peter Zijlstrae077df42009-03-19 20:26:17 +01002483}
2484
2485static const struct hw_perf_counter_ops *
2486tp_perf_counter_init(struct perf_counter *counter)
2487{
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002488 int event_id = perf_event_id(&counter->hw_event);
Peter Zijlstrae077df42009-03-19 20:26:17 +01002489 int ret;
2490
2491 ret = ftrace_profile_enable(event_id);
2492 if (ret)
2493 return NULL;
2494
2495 counter->destroy = tp_perf_counter_destroy;
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002496 counter->hw.irq_period = counter->hw_event.irq_period;
Peter Zijlstrae077df42009-03-19 20:26:17 +01002497
2498 return &perf_ops_generic;
2499}
2500#else
2501static const struct hw_perf_counter_ops *
2502tp_perf_counter_init(struct perf_counter *counter)
2503{
2504 return NULL;
2505}
2506#endif
2507
Ingo Molnar5c92d122008-12-11 13:21:10 +01002508static const struct hw_perf_counter_ops *
2509sw_perf_counter_init(struct perf_counter *counter)
2510{
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002511 struct perf_counter_hw_event *hw_event = &counter->hw_event;
Ingo Molnar5c92d122008-12-11 13:21:10 +01002512 const struct hw_perf_counter_ops *hw_ops = NULL;
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002513 struct hw_perf_counter *hwc = &counter->hw;
Ingo Molnar5c92d122008-12-11 13:21:10 +01002514
Paul Mackerras0475f9e2009-02-11 14:35:35 +11002515 /*
2516 * Software counters (currently) can't in general distinguish
2517 * between user, kernel and hypervisor events.
2518 * However, context switches and cpu migrations are considered
2519 * to be kernel events, and page faults are never hypervisor
2520 * events.
2521 */
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002522 switch (perf_event_id(&counter->hw_event)) {
Ingo Molnar5c92d122008-12-11 13:21:10 +01002523 case PERF_COUNT_CPU_CLOCK:
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002524 hw_ops = &perf_ops_cpu_clock;
2525
2526 if (hw_event->irq_period && hw_event->irq_period < 10000)
2527 hw_event->irq_period = 10000;
Ingo Molnar5c92d122008-12-11 13:21:10 +01002528 break;
Ingo Molnarbae43c92008-12-11 14:03:20 +01002529 case PERF_COUNT_TASK_CLOCK:
Paul Mackerras23a185c2009-02-09 22:42:47 +11002530 /*
2531 * If the user instantiates this as a per-cpu counter,
2532 * use the cpu_clock counter instead.
2533 */
2534 if (counter->ctx->task)
2535 hw_ops = &perf_ops_task_clock;
2536 else
2537 hw_ops = &perf_ops_cpu_clock;
Peter Zijlstrad6d020e2009-03-13 12:21:35 +01002538
2539 if (hw_event->irq_period && hw_event->irq_period < 10000)
2540 hw_event->irq_period = 10000;
Ingo Molnarbae43c92008-12-11 14:03:20 +01002541 break;
Ingo Molnare06c61a2008-12-14 14:44:31 +01002542 case PERF_COUNT_PAGE_FAULTS:
Peter Zijlstraac17dc82009-03-13 12:21:34 +01002543 case PERF_COUNT_PAGE_FAULTS_MIN:
2544 case PERF_COUNT_PAGE_FAULTS_MAJ:
Ingo Molnar5d6a27d2008-12-14 12:28:33 +01002545 case PERF_COUNT_CONTEXT_SWITCHES:
Peter Zijlstra4a0deca2009-03-19 20:26:12 +01002546 hw_ops = &perf_ops_generic;
Ingo Molnar5d6a27d2008-12-14 12:28:33 +01002547 break;
Ingo Molnar6c594c22008-12-14 12:34:15 +01002548 case PERF_COUNT_CPU_MIGRATIONS:
Paul Mackerras0475f9e2009-02-11 14:35:35 +11002549 if (!counter->hw_event.exclude_kernel)
2550 hw_ops = &perf_ops_cpu_migrations;
Ingo Molnar6c594c22008-12-14 12:34:15 +01002551 break;
Ingo Molnar5c92d122008-12-11 13:21:10 +01002552 }
Peter Zijlstra15dbf272009-03-13 12:21:32 +01002553
2554 if (hw_ops)
2555 hwc->irq_period = hw_event->irq_period;
2556
Ingo Molnar5c92d122008-12-11 13:21:10 +01002557 return hw_ops;
2558}
2559
Thomas Gleixner0793a612008-12-04 20:12:29 +01002560/*
2561 * Allocate and initialize a counter structure
2562 */
2563static struct perf_counter *
Ingo Molnar04289bb2008-12-11 08:38:42 +01002564perf_counter_alloc(struct perf_counter_hw_event *hw_event,
2565 int cpu,
Paul Mackerras23a185c2009-02-09 22:42:47 +11002566 struct perf_counter_context *ctx,
Ingo Molnar9b51f662008-12-12 13:49:45 +01002567 struct perf_counter *group_leader,
2568 gfp_t gfpflags)
Thomas Gleixner0793a612008-12-04 20:12:29 +01002569{
Ingo Molnar5c92d122008-12-11 13:21:10 +01002570 const struct hw_perf_counter_ops *hw_ops;
Ingo Molnar621a01e2008-12-11 12:46:46 +01002571 struct perf_counter *counter;
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002572 long err;
Thomas Gleixner0793a612008-12-04 20:12:29 +01002573
Ingo Molnar9b51f662008-12-12 13:49:45 +01002574 counter = kzalloc(sizeof(*counter), gfpflags);
Thomas Gleixner0793a612008-12-04 20:12:29 +01002575 if (!counter)
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002576 return ERR_PTR(-ENOMEM);
Thomas Gleixner0793a612008-12-04 20:12:29 +01002577
Ingo Molnar04289bb2008-12-11 08:38:42 +01002578 /*
2579 * Single counters are their own group leaders, with an
2580 * empty sibling list:
2581 */
2582 if (!group_leader)
2583 group_leader = counter;
2584
Thomas Gleixner0793a612008-12-04 20:12:29 +01002585 mutex_init(&counter->mutex);
Ingo Molnar04289bb2008-12-11 08:38:42 +01002586 INIT_LIST_HEAD(&counter->list_entry);
Peter Zijlstra592903c2009-03-13 12:21:36 +01002587 INIT_LIST_HEAD(&counter->event_entry);
Ingo Molnar04289bb2008-12-11 08:38:42 +01002588 INIT_LIST_HEAD(&counter->sibling_list);
Thomas Gleixner0793a612008-12-04 20:12:29 +01002589 init_waitqueue_head(&counter->waitq);
2590
Peter Zijlstra7b732a72009-03-23 18:22:10 +01002591 mutex_init(&counter->mmap_mutex);
2592
Paul Mackerrasd859e292009-01-17 18:10:22 +11002593 INIT_LIST_HEAD(&counter->child_list);
2594
Ingo Molnar9f66a382008-12-10 12:33:23 +01002595 counter->cpu = cpu;
2596 counter->hw_event = *hw_event;
Ingo Molnar04289bb2008-12-11 08:38:42 +01002597 counter->group_leader = group_leader;
Ingo Molnar621a01e2008-12-11 12:46:46 +01002598 counter->hw_ops = NULL;
Paul Mackerras23a185c2009-02-09 22:42:47 +11002599 counter->ctx = ctx;
Ingo Molnar621a01e2008-12-11 12:46:46 +01002600
Ingo Molnar235c7fc2008-12-21 14:43:25 +01002601 counter->state = PERF_COUNTER_STATE_INACTIVE;
Ingo Molnara86ed502008-12-17 00:43:10 +01002602 if (hw_event->disabled)
2603 counter->state = PERF_COUNTER_STATE_OFF;
2604
Ingo Molnar5c92d122008-12-11 13:21:10 +01002605 hw_ops = NULL;
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002606
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002607 if (perf_event_raw(hw_event)) {
Ingo Molnar5c92d122008-12-11 13:21:10 +01002608 hw_ops = hw_perf_counter_init(counter);
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002609 goto done;
2610 }
2611
2612 switch (perf_event_type(hw_event)) {
Peter Zijlstrab8e83512009-03-19 20:26:18 +01002613 case PERF_TYPE_HARDWARE:
2614 hw_ops = hw_perf_counter_init(counter);
2615 break;
2616
2617 case PERF_TYPE_SOFTWARE:
2618 hw_ops = sw_perf_counter_init(counter);
2619 break;
2620
2621 case PERF_TYPE_TRACEPOINT:
2622 hw_ops = tp_perf_counter_init(counter);
2623 break;
2624 }
Peter Zijlstraf4a2deb42009-03-23 18:22:06 +01002625done:
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002626 err = 0;
2627 if (!hw_ops)
2628 err = -EINVAL;
2629 else if (IS_ERR(hw_ops))
2630 err = PTR_ERR(hw_ops);
2631
2632 if (err) {
2633 kfree(counter);
2634 return ERR_PTR(err);
2635 }
2636
Ingo Molnar621a01e2008-12-11 12:46:46 +01002637 counter->hw_ops = hw_ops;
Thomas Gleixner0793a612008-12-04 20:12:29 +01002638
2639 return counter;
2640}
2641
2642/**
Paul Mackerras2743a5b2009-03-04 20:36:51 +11002643 * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
Ingo Molnar9f66a382008-12-10 12:33:23 +01002644 *
2645 * @hw_event_uptr: event type attributes for monitoring/sampling
Thomas Gleixner0793a612008-12-04 20:12:29 +01002646 * @pid: target pid
Ingo Molnar9f66a382008-12-10 12:33:23 +01002647 * @cpu: target cpu
2648 * @group_fd: group leader counter fd
Thomas Gleixner0793a612008-12-04 20:12:29 +01002649 */
Paul Mackerras2743a5b2009-03-04 20:36:51 +11002650SYSCALL_DEFINE5(perf_counter_open,
Paul Mackerrasf3dfd262009-02-26 22:43:46 +11002651 const struct perf_counter_hw_event __user *, hw_event_uptr,
Paul Mackerras2743a5b2009-03-04 20:36:51 +11002652 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
Thomas Gleixner0793a612008-12-04 20:12:29 +01002653{
Ingo Molnar04289bb2008-12-11 08:38:42 +01002654 struct perf_counter *counter, *group_leader;
Ingo Molnar9f66a382008-12-10 12:33:23 +01002655 struct perf_counter_hw_event hw_event;
Ingo Molnar04289bb2008-12-11 08:38:42 +01002656 struct perf_counter_context *ctx;
Ingo Molnar9b51f662008-12-12 13:49:45 +01002657 struct file *counter_file = NULL;
Ingo Molnar04289bb2008-12-11 08:38:42 +01002658 struct file *group_file = NULL;
2659 int fput_needed = 0;
Ingo Molnar9b51f662008-12-12 13:49:45 +01002660 int fput_needed2 = 0;
Thomas Gleixner0793a612008-12-04 20:12:29 +01002661 int ret;
2662
Paul Mackerras2743a5b2009-03-04 20:36:51 +11002663 /* for future expandability... */
2664 if (flags)
2665 return -EINVAL;
2666
Ingo Molnar9f66a382008-12-10 12:33:23 +01002667 if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
Thomas Gleixnereab656a2008-12-08 19:26:59 +01002668 return -EFAULT;
2669
Ingo Molnar04289bb2008-12-11 08:38:42 +01002670 /*
Ingo Molnarccff2862008-12-11 11:26:29 +01002671 * Get the target context (task or percpu):
2672 */
2673 ctx = find_get_context(pid, cpu);
2674 if (IS_ERR(ctx))
2675 return PTR_ERR(ctx);
2676
2677 /*
2678 * Look up the group leader (we will attach this counter to it):
Ingo Molnar04289bb2008-12-11 08:38:42 +01002679 */
2680 group_leader = NULL;
2681 if (group_fd != -1) {
2682 ret = -EINVAL;
2683 group_file = fget_light(group_fd, &fput_needed);
2684 if (!group_file)
Ingo Molnarccff2862008-12-11 11:26:29 +01002685 goto err_put_context;
Ingo Molnar04289bb2008-12-11 08:38:42 +01002686 if (group_file->f_op != &perf_fops)
Ingo Molnarccff2862008-12-11 11:26:29 +01002687 goto err_put_context;
Ingo Molnar04289bb2008-12-11 08:38:42 +01002688
2689 group_leader = group_file->private_data;
2690 /*
Ingo Molnarccff2862008-12-11 11:26:29 +01002691 * Do not allow a recursive hierarchy (this new sibling
2692 * becoming part of another group-sibling):
Ingo Molnar04289bb2008-12-11 08:38:42 +01002693 */
Ingo Molnarccff2862008-12-11 11:26:29 +01002694 if (group_leader->group_leader != group_leader)
2695 goto err_put_context;
2696 /*
2697 * Do not allow to attach to a group in a different
2698 * task or CPU context:
2699 */
2700 if (group_leader->ctx != ctx)
2701 goto err_put_context;
Paul Mackerras3b6f9e52009-01-14 21:00:30 +11002702 /*
2703 * Only a group leader can be exclusive or pinned
2704 */
2705 if (hw_event.exclusive || hw_event.pinned)
2706 goto err_put_context;
Ingo Molnar04289bb2008-12-11 08:38:42 +01002707 }
2708
Paul Mackerras23a185c2009-02-09 22:42:47 +11002709 counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
2710 GFP_KERNEL);
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002711 ret = PTR_ERR(counter);
2712 if (IS_ERR(counter))
Thomas Gleixner0793a612008-12-04 20:12:29 +01002713 goto err_put_context;
2714
Thomas Gleixner0793a612008-12-04 20:12:29 +01002715 ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
2716 if (ret < 0)
Ingo Molnar9b51f662008-12-12 13:49:45 +01002717 goto err_free_put_context;
2718
2719 counter_file = fget_light(ret, &fput_needed2);
2720 if (!counter_file)
2721 goto err_free_put_context;
2722
2723 counter->filp = counter_file;
Paul Mackerrasd859e292009-01-17 18:10:22 +11002724 mutex_lock(&ctx->mutex);
Ingo Molnar9b51f662008-12-12 13:49:45 +01002725 perf_install_in_context(ctx, counter, cpu);
Paul Mackerrasd859e292009-01-17 18:10:22 +11002726 mutex_unlock(&ctx->mutex);
Ingo Molnar9b51f662008-12-12 13:49:45 +01002727
2728 fput_light(counter_file, fput_needed2);
Thomas Gleixner0793a612008-12-04 20:12:29 +01002729
Ingo Molnar04289bb2008-12-11 08:38:42 +01002730out_fput:
2731 fput_light(group_file, fput_needed);
2732
Thomas Gleixner0793a612008-12-04 20:12:29 +01002733 return ret;
2734
Ingo Molnar9b51f662008-12-12 13:49:45 +01002735err_free_put_context:
Thomas Gleixner0793a612008-12-04 20:12:29 +01002736 kfree(counter);
2737
2738err_put_context:
2739 put_context(ctx);
2740
Ingo Molnar04289bb2008-12-11 08:38:42 +01002741 goto out_fput;
Thomas Gleixner0793a612008-12-04 20:12:29 +01002742}
2743
Ingo Molnar9b51f662008-12-12 13:49:45 +01002744/*
2745 * Initialize the perf_counter context in a task_struct:
2746 */
2747static void
2748__perf_counter_init_context(struct perf_counter_context *ctx,
2749 struct task_struct *task)
2750{
2751 memset(ctx, 0, sizeof(*ctx));
2752 spin_lock_init(&ctx->lock);
Paul Mackerrasd859e292009-01-17 18:10:22 +11002753 mutex_init(&ctx->mutex);
Ingo Molnar9b51f662008-12-12 13:49:45 +01002754 INIT_LIST_HEAD(&ctx->counter_list);
Peter Zijlstra592903c2009-03-13 12:21:36 +01002755 INIT_LIST_HEAD(&ctx->event_list);
Ingo Molnar9b51f662008-12-12 13:49:45 +01002756 ctx->task = task;
2757}
2758
2759/*
2760 * inherit a counter from parent task to child task:
2761 */
Paul Mackerrasd859e292009-01-17 18:10:22 +11002762static struct perf_counter *
Ingo Molnar9b51f662008-12-12 13:49:45 +01002763inherit_counter(struct perf_counter *parent_counter,
2764 struct task_struct *parent,
2765 struct perf_counter_context *parent_ctx,
2766 struct task_struct *child,
Paul Mackerrasd859e292009-01-17 18:10:22 +11002767 struct perf_counter *group_leader,
Ingo Molnar9b51f662008-12-12 13:49:45 +01002768 struct perf_counter_context *child_ctx)
2769{
2770 struct perf_counter *child_counter;
2771
Paul Mackerrasd859e292009-01-17 18:10:22 +11002772 /*
2773 * Instead of creating recursive hierarchies of counters,
2774 * we link inherited counters back to the original parent,
2775 * which has a filp for sure, which we use as the reference
2776 * count:
2777 */
2778 if (parent_counter->parent)
2779 parent_counter = parent_counter->parent;
2780
Ingo Molnar9b51f662008-12-12 13:49:45 +01002781 child_counter = perf_counter_alloc(&parent_counter->hw_event,
Paul Mackerras23a185c2009-02-09 22:42:47 +11002782 parent_counter->cpu, child_ctx,
2783 group_leader, GFP_KERNEL);
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002784 if (IS_ERR(child_counter))
2785 return child_counter;
Ingo Molnar9b51f662008-12-12 13:49:45 +01002786
2787 /*
2788 * Link it up in the child's context:
2789 */
Ingo Molnar9b51f662008-12-12 13:49:45 +01002790 child_counter->task = child;
Paul Mackerras53cfbf52009-03-25 22:46:58 +11002791 add_counter_to_ctx(child_counter, child_ctx);
Ingo Molnar9b51f662008-12-12 13:49:45 +01002792
2793 child_counter->parent = parent_counter;
Ingo Molnar9b51f662008-12-12 13:49:45 +01002794 /*
2795 * inherit into child's child as well:
2796 */
2797 child_counter->hw_event.inherit = 1;
2798
2799 /*
2800 * Get a reference to the parent filp - we will fput it
2801 * when the child counter exits. This is safe to do because
2802 * we are in the parent and we know that the filp still
2803 * exists and has a nonzero count:
2804 */
2805 atomic_long_inc(&parent_counter->filp->f_count);
2806
Paul Mackerrasd859e292009-01-17 18:10:22 +11002807 /*
2808 * Link this into the parent counter's child list
2809 */
2810 mutex_lock(&parent_counter->mutex);
2811 list_add_tail(&child_counter->child_list, &parent_counter->child_list);
2812
2813 /*
2814 * Make the child state follow the state of the parent counter,
2815 * not its hw_event.disabled bit. We hold the parent's mutex,
2816 * so we won't race with perf_counter_{en,dis}able_family.
2817 */
2818 if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
2819 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
2820 else
2821 child_counter->state = PERF_COUNTER_STATE_OFF;
2822
2823 mutex_unlock(&parent_counter->mutex);
2824
2825 return child_counter;
2826}
2827
2828static int inherit_group(struct perf_counter *parent_counter,
2829 struct task_struct *parent,
2830 struct perf_counter_context *parent_ctx,
2831 struct task_struct *child,
2832 struct perf_counter_context *child_ctx)
2833{
2834 struct perf_counter *leader;
2835 struct perf_counter *sub;
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002836 struct perf_counter *child_ctr;
Paul Mackerrasd859e292009-01-17 18:10:22 +11002837
2838 leader = inherit_counter(parent_counter, parent, parent_ctx,
2839 child, NULL, child_ctx);
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002840 if (IS_ERR(leader))
2841 return PTR_ERR(leader);
Paul Mackerrasd859e292009-01-17 18:10:22 +11002842 list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
Paul Mackerrasd5d2bc0d2009-03-30 19:07:08 +02002843 child_ctr = inherit_counter(sub, parent, parent_ctx,
2844 child, leader, child_ctx);
2845 if (IS_ERR(child_ctr))
2846 return PTR_ERR(child_ctr);
Paul Mackerrasd859e292009-01-17 18:10:22 +11002847 }
Ingo Molnar9b51f662008-12-12 13:49:45 +01002848 return 0;
2849}
2850
Paul Mackerrasd859e292009-01-17 18:10:22 +11002851static void sync_child_counter(struct perf_counter *child_counter,
2852 struct perf_counter *parent_counter)
2853{
2854 u64 parent_val, child_val;
2855
2856 parent_val = atomic64_read(&parent_counter->count);
2857 child_val = atomic64_read(&child_counter->count);
2858
2859 /*
2860 * Add back the child's count to the parent's count:
2861 */
2862 atomic64_add(child_val, &parent_counter->count);
Paul Mackerras53cfbf52009-03-25 22:46:58 +11002863 atomic64_add(child_counter->total_time_enabled,
2864 &parent_counter->child_total_time_enabled);
2865 atomic64_add(child_counter->total_time_running,
2866 &parent_counter->child_total_time_running);
Paul Mackerrasd859e292009-01-17 18:10:22 +11002867
2868 /*
2869 * Remove this counter from the parent's list
2870 */
2871 mutex_lock(&parent_counter->mutex);
2872 list_del_init(&child_counter->child_list);
2873 mutex_unlock(&parent_counter->mutex);
2874
2875 /*
2876 * Release the parent counter, if this was the last
2877 * reference to it.
2878 */
2879 fput(parent_counter->filp);
2880}
2881
Ingo Molnar9b51f662008-12-12 13:49:45 +01002882static void
2883__perf_counter_exit_task(struct task_struct *child,
2884 struct perf_counter *child_counter,
2885 struct perf_counter_context *child_ctx)
2886{
2887 struct perf_counter *parent_counter;
Paul Mackerrasd859e292009-01-17 18:10:22 +11002888 struct perf_counter *sub, *tmp;
Ingo Molnar9b51f662008-12-12 13:49:45 +01002889
2890 /*
Ingo Molnar235c7fc2008-12-21 14:43:25 +01002891 * If we do not self-reap then we have to wait for the
2892 * child task to unschedule (it will happen for sure),
2893 * so that its counter is at its final count. (This
2894 * condition triggers rarely - child tasks usually get
2895 * off their CPU before the parent has a chance to
2896 * get this far into the reaping action)
Ingo Molnar9b51f662008-12-12 13:49:45 +01002897 */
Ingo Molnar235c7fc2008-12-21 14:43:25 +01002898 if (child != current) {
2899 wait_task_inactive(child, 0);
2900 list_del_init(&child_counter->list_entry);
Paul Mackerras53cfbf52009-03-25 22:46:58 +11002901 update_counter_times(child_counter);
Ingo Molnar235c7fc2008-12-21 14:43:25 +01002902 } else {
Ingo Molnar0cc0c022008-12-14 23:20:36 +01002903 struct perf_cpu_context *cpuctx;
Ingo Molnar235c7fc2008-12-21 14:43:25 +01002904 unsigned long flags;
2905 u64 perf_flags;
2906
2907 /*
2908 * Disable and unlink this counter.
2909 *
2910 * Be careful about zapping the list - IRQ/NMI context
2911 * could still be processing it:
2912 */
2913 curr_rq_lock_irq_save(&flags);
2914 perf_flags = hw_perf_save_disable();
Ingo Molnar0cc0c022008-12-14 23:20:36 +01002915
2916 cpuctx = &__get_cpu_var(perf_cpu_context);
2917
Paul Mackerrasd859e292009-01-17 18:10:22 +11002918 group_sched_out(child_counter, cpuctx, child_ctx);
Paul Mackerras53cfbf52009-03-25 22:46:58 +11002919 update_counter_times(child_counter);
Ingo Molnar0cc0c022008-12-14 23:20:36 +01002920
Ingo Molnar235c7fc2008-12-21 14:43:25 +01002921 list_del_init(&child_counter->list_entry);
2922
2923 child_ctx->nr_counters--;
2924
2925 hw_perf_restore(perf_flags);
2926 curr_rq_unlock_irq_restore(&flags);
Ingo Molnar0cc0c022008-12-14 23:20:36 +01002927 }
2928
Ingo Molnar9b51f662008-12-12 13:49:45 +01002929 parent_counter = child_counter->parent;
2930 /*
2931 * It can happen that parent exits first, and has counters
2932 * that are still around due to the child reference. These
2933 * counters need to be zapped - but otherwise linger.
2934 */
Paul Mackerrasd859e292009-01-17 18:10:22 +11002935 if (parent_counter) {
2936 sync_child_counter(child_counter, parent_counter);
2937 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
2938 list_entry) {
Paul Mackerras4bcf3492009-02-11 13:53:19 +01002939 if (sub->parent) {
Paul Mackerrasd859e292009-01-17 18:10:22 +11002940 sync_child_counter(sub, sub->parent);
Peter Zijlstraf1600952009-03-19 20:26:16 +01002941 free_counter(sub);
Paul Mackerras4bcf3492009-02-11 13:53:19 +01002942 }
Paul Mackerrasd859e292009-01-17 18:10:22 +11002943 }
Peter Zijlstraf1600952009-03-19 20:26:16 +01002944 free_counter(child_counter);
Paul Mackerras4bcf3492009-02-11 13:53:19 +01002945 }
Ingo Molnar9b51f662008-12-12 13:49:45 +01002946}
2947
2948/*
Paul Mackerrasd859e292009-01-17 18:10:22 +11002949 * When a child task exits, feed back counter values to parent counters.
Ingo Molnar9b51f662008-12-12 13:49:45 +01002950 *
Paul Mackerrasd859e292009-01-17 18:10:22 +11002951 * Note: we may be running in child context, but the PID is not hashed
Ingo Molnar9b51f662008-12-12 13:49:45 +01002952 * anymore so new counters will not be added.
2953 */
2954void perf_counter_exit_task(struct task_struct *child)
2955{
2956 struct perf_counter *child_counter, *tmp;
2957 struct perf_counter_context *child_ctx;
2958
2959 child_ctx = &child->perf_counter_ctx;
2960
2961 if (likely(!child_ctx->nr_counters))
2962 return;
2963
2964 list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
2965 list_entry)
2966 __perf_counter_exit_task(child, child_counter, child_ctx);
2967}
2968
2969/*
2970 * Initialize the perf_counter context in task_struct
2971 */
2972void perf_counter_init_task(struct task_struct *child)
2973{
2974 struct perf_counter_context *child_ctx, *parent_ctx;
Paul Mackerrasd859e292009-01-17 18:10:22 +11002975 struct perf_counter *counter;
Ingo Molnar9b51f662008-12-12 13:49:45 +01002976 struct task_struct *parent = current;
Ingo Molnar9b51f662008-12-12 13:49:45 +01002977
2978 child_ctx = &child->perf_counter_ctx;
2979 parent_ctx = &parent->perf_counter_ctx;
2980
2981 __perf_counter_init_context(child_ctx, child);
2982
2983 /*
2984 * This is executed from the parent task context, so inherit
2985 * counters that have been marked for cloning:
2986 */
2987
2988 if (likely(!parent_ctx->nr_counters))
2989 return;
2990
2991 /*
2992 * Lock the parent list. No need to lock the child - not PID
2993 * hashed yet and not running, so nobody can access it.
2994 */
Paul Mackerrasd859e292009-01-17 18:10:22 +11002995 mutex_lock(&parent_ctx->mutex);
Ingo Molnar9b51f662008-12-12 13:49:45 +01002996
2997 /*
2998 * We dont have to disable NMIs - we are only looking at
2999 * the list, not manipulating it:
3000 */
3001 list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
Paul Mackerrasd859e292009-01-17 18:10:22 +11003002 if (!counter->hw_event.inherit)
Ingo Molnar9b51f662008-12-12 13:49:45 +01003003 continue;
3004
Paul Mackerrasd859e292009-01-17 18:10:22 +11003005 if (inherit_group(counter, parent,
Ingo Molnar9b51f662008-12-12 13:49:45 +01003006 parent_ctx, child, child_ctx))
3007 break;
3008 }
3009
Paul Mackerrasd859e292009-01-17 18:10:22 +11003010 mutex_unlock(&parent_ctx->mutex);
Ingo Molnar9b51f662008-12-12 13:49:45 +01003011}
3012
Ingo Molnar04289bb2008-12-11 08:38:42 +01003013static void __cpuinit perf_counter_init_cpu(int cpu)
Thomas Gleixner0793a612008-12-04 20:12:29 +01003014{
Ingo Molnar04289bb2008-12-11 08:38:42 +01003015 struct perf_cpu_context *cpuctx;
Thomas Gleixner0793a612008-12-04 20:12:29 +01003016
Ingo Molnar04289bb2008-12-11 08:38:42 +01003017 cpuctx = &per_cpu(perf_cpu_context, cpu);
3018 __perf_counter_init_context(&cpuctx->ctx, NULL);
Thomas Gleixner0793a612008-12-04 20:12:29 +01003019
3020 mutex_lock(&perf_resource_mutex);
Ingo Molnar04289bb2008-12-11 08:38:42 +01003021 cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
Thomas Gleixner0793a612008-12-04 20:12:29 +01003022 mutex_unlock(&perf_resource_mutex);
Ingo Molnar04289bb2008-12-11 08:38:42 +01003023
Paul Mackerras01d02872009-01-14 13:44:19 +11003024 hw_perf_counter_setup(cpu);
Thomas Gleixner0793a612008-12-04 20:12:29 +01003025}
3026
3027#ifdef CONFIG_HOTPLUG_CPU
Ingo Molnar04289bb2008-12-11 08:38:42 +01003028static void __perf_counter_exit_cpu(void *info)
Thomas Gleixner0793a612008-12-04 20:12:29 +01003029{
3030 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
3031 struct perf_counter_context *ctx = &cpuctx->ctx;
3032 struct perf_counter *counter, *tmp;
3033
Ingo Molnar04289bb2008-12-11 08:38:42 +01003034 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
3035 __perf_counter_remove_from_context(counter);
Thomas Gleixner0793a612008-12-04 20:12:29 +01003036}
Ingo Molnar04289bb2008-12-11 08:38:42 +01003037static void perf_counter_exit_cpu(int cpu)
Thomas Gleixner0793a612008-12-04 20:12:29 +01003038{
Paul Mackerrasd859e292009-01-17 18:10:22 +11003039 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
3040 struct perf_counter_context *ctx = &cpuctx->ctx;
3041
3042 mutex_lock(&ctx->mutex);
Ingo Molnar04289bb2008-12-11 08:38:42 +01003043 smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
Paul Mackerrasd859e292009-01-17 18:10:22 +11003044 mutex_unlock(&ctx->mutex);
Thomas Gleixner0793a612008-12-04 20:12:29 +01003045}
3046#else
Ingo Molnar04289bb2008-12-11 08:38:42 +01003047static inline void perf_counter_exit_cpu(int cpu) { }
Thomas Gleixner0793a612008-12-04 20:12:29 +01003048#endif
3049
3050static int __cpuinit
3051perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
3052{
3053 unsigned int cpu = (long)hcpu;
3054
3055 switch (action) {
3056
3057 case CPU_UP_PREPARE:
3058 case CPU_UP_PREPARE_FROZEN:
Ingo Molnar04289bb2008-12-11 08:38:42 +01003059 perf_counter_init_cpu(cpu);
Thomas Gleixner0793a612008-12-04 20:12:29 +01003060 break;
3061
3062 case CPU_DOWN_PREPARE:
3063 case CPU_DOWN_PREPARE_FROZEN:
Ingo Molnar04289bb2008-12-11 08:38:42 +01003064 perf_counter_exit_cpu(cpu);
Thomas Gleixner0793a612008-12-04 20:12:29 +01003065 break;
3066
3067 default:
3068 break;
3069 }
3070
3071 return NOTIFY_OK;
3072}
3073
3074static struct notifier_block __cpuinitdata perf_cpu_nb = {
3075 .notifier_call = perf_cpu_notify,
3076};
3077
3078static int __init perf_counter_init(void)
3079{
3080 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
3081 (void *)(long)smp_processor_id());
3082 register_cpu_notifier(&perf_cpu_nb);
3083
3084 return 0;
3085}
3086early_initcall(perf_counter_init);
3087
3088static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
3089{
3090 return sprintf(buf, "%d\n", perf_reserved_percpu);
3091}
3092
3093static ssize_t
3094perf_set_reserve_percpu(struct sysdev_class *class,
3095 const char *buf,
3096 size_t count)
3097{
3098 struct perf_cpu_context *cpuctx;
3099 unsigned long val;
3100 int err, cpu, mpt;
3101
3102 err = strict_strtoul(buf, 10, &val);
3103 if (err)
3104 return err;
3105 if (val > perf_max_counters)
3106 return -EINVAL;
3107
3108 mutex_lock(&perf_resource_mutex);
3109 perf_reserved_percpu = val;
3110 for_each_online_cpu(cpu) {
3111 cpuctx = &per_cpu(perf_cpu_context, cpu);
3112 spin_lock_irq(&cpuctx->ctx.lock);
3113 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
3114 perf_max_counters - perf_reserved_percpu);
3115 cpuctx->max_pertask = mpt;
3116 spin_unlock_irq(&cpuctx->ctx.lock);
3117 }
3118 mutex_unlock(&perf_resource_mutex);
3119
3120 return count;
3121}
3122
3123static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
3124{
3125 return sprintf(buf, "%d\n", perf_overcommit);
3126}
3127
3128static ssize_t
3129perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
3130{
3131 unsigned long val;
3132 int err;
3133
3134 err = strict_strtoul(buf, 10, &val);
3135 if (err)
3136 return err;
3137 if (val > 1)
3138 return -EINVAL;
3139
3140 mutex_lock(&perf_resource_mutex);
3141 perf_overcommit = val;
3142 mutex_unlock(&perf_resource_mutex);
3143
3144 return count;
3145}
3146
3147static SYSDEV_CLASS_ATTR(
3148 reserve_percpu,
3149 0644,
3150 perf_show_reserve_percpu,
3151 perf_set_reserve_percpu
3152 );
3153
3154static SYSDEV_CLASS_ATTR(
3155 overcommit,
3156 0644,
3157 perf_show_overcommit,
3158 perf_set_overcommit
3159 );
3160
3161static struct attribute *perfclass_attrs[] = {
3162 &attr_reserve_percpu.attr,
3163 &attr_overcommit.attr,
3164 NULL
3165};
3166
3167static struct attribute_group perfclass_attr_group = {
3168 .attrs = perfclass_attrs,
3169 .name = "perf_counters",
3170};
3171
3172static int __init perf_counter_sysfs_init(void)
3173{
3174 return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
3175 &perfclass_attr_group);
3176}
3177device_initcall(perf_counter_sysfs_init);