perf_counter: uncouple data_head updates from wakeups
Keep data_head up-to-date irrespective of notifications. This fixes
the case where you disable a counter and don't get a notification for
the last few pending events, and it also allows polling usage.
[ Impact: increase precision of perfcounter mmap-ed fields ]
Suggested-by: Corey Ashford <cjashfor@linux.vnet.ibm.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
LKML-Reference: <20090505155436.925084300@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index 5f86a11..ba5e921 100644
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -1696,7 +1696,6 @@
struct perf_mmap_data *data;
unsigned int offset;
unsigned int head;
- int wakeup;
int nmi;
int overflow;
int locked;
@@ -1752,8 +1751,7 @@
struct perf_mmap_data *data = handle->data;
int head, cpu;
- if (handle->wakeup)
- data->wakeup_head = data->head;
+ data->done_head = data->head;
if (!handle->locked)
goto out;
@@ -1764,13 +1762,11 @@
* before we publish the new head, matched by a rmb() in userspace when
* reading this position.
*/
- while ((head = atomic_xchg(&data->wakeup_head, 0))) {
+ while ((head = atomic_xchg(&data->done_head, 0)))
data->user_page->data_head = head;
- handle->wakeup = 1;
- }
/*
- * NMI can happen here, which means we can miss a wakeup_head update.
+ * NMI can happen here, which means we can miss a done_head update.
*/
cpu = atomic_xchg(&data->lock, 0);
@@ -1779,7 +1775,7 @@
/*
* Therefore we have to validate we did not indeed do so.
*/
- if (unlikely(atomic_read(&data->wakeup_head))) {
+ if (unlikely(atomic_read(&data->done_head))) {
/*
* Since we had it locked, we can lock it again.
*/
@@ -1789,7 +1785,7 @@
goto again;
}
- if (handle->wakeup)
+ if (atomic_xchg(&data->wakeup, 0))
perf_output_wakeup(handle);
out:
local_irq_restore(handle->flags);
@@ -1824,7 +1820,9 @@
handle->offset = offset;
handle->head = head;
- handle->wakeup = (offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT);
+
+ if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
+ atomic_set(&data->wakeup, 1);
return 0;
@@ -1882,7 +1880,7 @@
int events = atomic_inc_return(&data->events);
if (events >= wakeup_events) {
atomic_sub(wakeup_events, &data->events);
- handle->wakeup = 1;
+ atomic_set(&data->wakeup, 1);
}
}