[tip:perfcounters/core] perf_counter: uncouple data_head updates from wakeups

From: tip-bot for Peter Zijlstra
Date: Tue May 05 2009 - 14:35:01 EST


Commit-ID: c66de4a5be7913247bd83d79168f8e4420c9cfbc
Gitweb: http://git.kernel.org/tip/c66de4a5be7913247bd83d79168f8e4420c9cfbc
Author: Peter Zijlstra <a.p.zijlstra@xxxxxxxxx>
AuthorDate: Tue, 5 May 2009 17:50:22 +0200
Committer: Ingo Molnar <mingo@xxxxxxx>
CommitDate: Tue, 5 May 2009 20:18:30 +0200

perf_counter: uncouple data_head updates from wakeups

Keep data_head up-to-date irrespective of notifications. This fixes
the case where you disable a counter and don't get a notification for
the last few pending events, and it also allows polling usage.

[ Impact: increase precision of perfcounter mmap-ed fields ]

Suggested-by: Corey Ashford <cjashfor@xxxxxxxxxxxxxxxxxx>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@xxxxxxxxx>
Cc: Paul Mackerras <paulus@xxxxxxxxx>
LKML-Reference: <20090505155436.925084300@xxxxxxxxx>
Signed-off-by: Ingo Molnar <mingo@xxxxxxx>


---
include/linux/perf_counter.h | 4 +++-
kernel/perf_counter.c | 20 +++++++++-----------
2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/include/linux/perf_counter.h b/include/linux/perf_counter.h
index a356fa6..17b6310 100644
--- a/include/linux/perf_counter.h
+++ b/include/linux/perf_counter.h
@@ -362,9 +362,11 @@ struct perf_mmap_data {
atomic_t head; /* write position */
atomic_t events; /* event limit */

- atomic_t wakeup_head; /* completed head */
+ atomic_t done_head; /* completed head */
atomic_t lock; /* concurrent writes */

+ atomic_t wakeup; /* needs a wakeup */
+
struct perf_counter_mmap_page *user_page;
void *data_pages[0];
};
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index 5f86a11..ba5e921 100644
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -1696,7 +1696,6 @@ struct perf_output_handle {
struct perf_mmap_data *data;
unsigned int offset;
unsigned int head;
- int wakeup;
int nmi;
int overflow;
int locked;
@@ -1752,8 +1751,7 @@ static void perf_output_unlock(struct perf_output_handle *handle)
struct perf_mmap_data *data = handle->data;
int head, cpu;

- if (handle->wakeup)
- data->wakeup_head = data->head;
+ data->done_head = data->head;

if (!handle->locked)
goto out;
@@ -1764,13 +1762,11 @@ again:
* before we publish the new head, matched by a rmb() in userspace when
* reading this position.
*/
- while ((head = atomic_xchg(&data->wakeup_head, 0))) {
+ while ((head = atomic_xchg(&data->done_head, 0)))
data->user_page->data_head = head;
- handle->wakeup = 1;
- }

/*
- * NMI can happen here, which means we can miss a wakeup_head update.
+ * NMI can happen here, which means we can miss a done_head update.
*/

cpu = atomic_xchg(&data->lock, 0);
@@ -1779,7 +1775,7 @@ again:
/*
* Therefore we have to validate we did not indeed do so.
*/
- if (unlikely(atomic_read(&data->wakeup_head))) {
+ if (unlikely(atomic_read(&data->done_head))) {
/*
* Since we had it locked, we can lock it again.
*/
@@ -1789,7 +1785,7 @@ again:
goto again;
}

- if (handle->wakeup)
+ if (atomic_xchg(&data->wakeup, 0))
perf_output_wakeup(handle);
out:
local_irq_restore(handle->flags);
@@ -1824,7 +1820,9 @@ static int perf_output_begin(struct perf_output_handle *handle,

handle->offset = offset;
handle->head = head;
- handle->wakeup = (offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT);
+
+ if ((offset >> PAGE_SHIFT) != (head >> PAGE_SHIFT))
+ atomic_set(&data->wakeup, 1);

return 0;

@@ -1882,7 +1880,7 @@ static void perf_output_end(struct perf_output_handle *handle)
int events = atomic_inc_return(&data->events);
if (events >= wakeup_events) {
atomic_sub(wakeup_events, &data->events);
- handle->wakeup = 1;
+ atomic_set(&data->wakeup, 1);
}
}

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/