Perf: keep events across hotplug
Keep event list alive across a CPU hotplug so that perf
can resume when the CPU comes back online. Bring a CPU
online when exiting a perf session so it can be cleaned
up properly.
Change-Id: Ie0e4a43f751beb77afdc84e9d52b21780f279d80
Signed-off-by: Neil Leeder <nleeder@codeaurora.org>
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index 5b99903..4c0aac5 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -661,6 +661,7 @@
armpmu->pmu.start = armpmu_start;
armpmu->pmu.stop = armpmu_stop;
armpmu->pmu.read = armpmu_read;
+ armpmu->pmu.events_across_hotplug = 1;
}
int armpmu_register(struct arm_pmu *armpmu, char *name, int type)
@@ -776,62 +777,6 @@
disable_percpu_irq(irq);
}
-/*
- * PMU hardware loses all context when a CPU goes offline.
- * When a CPU is hotplugged back in, since some hardware registers are
- * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading
- * junk values out of them.
- */
-static int __cpuinit pmu_cpu_notify(struct notifier_block *b,
- unsigned long action, void *hcpu)
-{
- int irq;
-
- if (cpu_has_active_perf((int)hcpu)) {
- switch ((action & ~CPU_TASKS_FROZEN)) {
-
- case CPU_DOWN_PREPARE:
- /*
- * If this is on a multicore CPU, we need
- * to disarm the PMU IRQ before disappearing.
- */
- if (cpu_pmu &&
- cpu_pmu->plat_device->dev.platform_data) {
- irq = platform_get_irq(cpu_pmu->plat_device, 0);
- smp_call_function_single((int)hcpu,
- disable_irq_callback, &irq, 1);
- }
- return NOTIFY_DONE;
-
- case CPU_UP_PREPARE:
- /*
- * If this is on a multicore CPU, we need
- * to arm the PMU IRQ before appearing.
- */
- if (cpu_pmu &&
- cpu_pmu->plat_device->dev.platform_data) {
- irq = platform_get_irq(cpu_pmu->plat_device, 0);
- smp_call_function_single((int)hcpu,
- enable_irq_callback, &irq, 1);
- }
- return NOTIFY_DONE;
-
- case CPU_STARTING:
- if (cpu_pmu && cpu_pmu->reset) {
- cpu_pmu->reset(NULL);
- return NOTIFY_OK;
- }
- default:
- return NOTIFY_DONE;
- }
- }
-
- if ((action & ~CPU_TASKS_FROZEN) != CPU_STARTING)
- return NOTIFY_DONE;
-
- return NOTIFY_OK;
-}
-
static void armpmu_update_counters(void)
{
struct pmu_hw_events *hw_events;
@@ -852,6 +797,64 @@
}
}
+/*
+ * PMU hardware loses all context when a CPU goes offline.
+ * When a CPU is hotplugged back in, since some hardware registers are
+ * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading
+ * junk values out of them.
+ */
+static int __cpuinit pmu_cpu_notify(struct notifier_block *b,
+ unsigned long action, void *hcpu)
+{
+ int irq;
+ struct pmu *pmu;
+
+ if (cpu_has_active_perf((int)hcpu)) {
+ switch ((action & ~CPU_TASKS_FROZEN)) {
+
+ case CPU_DOWN_PREPARE:
+ armpmu_update_counters();
+ /*
+ * If this is on a multicore CPU, we need
+ * to disarm the PMU IRQ before disappearing.
+ */
+ if (cpu_pmu &&
+ cpu_pmu->plat_device->dev.platform_data) {
+ irq = platform_get_irq(cpu_pmu->plat_device, 0);
+ smp_call_function_single((int)hcpu,
+ disable_irq_callback, &irq, 1);
+ }
+ return NOTIFY_DONE;
+
+ case CPU_STARTING:
+ /*
+ * If this is on a multicore CPU, we need
+ * to arm the PMU IRQ before appearing.
+ */
+ if (cpu_pmu &&
+ cpu_pmu->plat_device->dev.platform_data) {
+ irq = platform_get_irq(cpu_pmu->plat_device, 0);
+ enable_irq_callback(&irq);
+ }
+
+ if (cpu_pmu && cpu_pmu->reset) {
+ __get_cpu_var(from_idle) = 1;
+ cpu_pmu->reset(NULL);
+ pmu = &cpu_pmu->pmu;
+ pmu->pmu_enable(pmu);
+ return NOTIFY_OK;
+ }
+ default:
+ return NOTIFY_DONE;
+ }
+ }
+
+ if ((action & ~CPU_TASKS_FROZEN) != CPU_STARTING)
+ return NOTIFY_DONE;
+
+ return NOTIFY_OK;
+}
+
static struct notifier_block __cpuinitdata pmu_cpu_notifier = {
.notifier_call = pmu_cpu_notify,
};
diff --git a/arch/arm/mach-msm/perf_debug.c b/arch/arm/mach-msm/perf_debug.c
index 70c8c68..478848d 100644
--- a/arch/arm/mach-msm/perf_debug.c
+++ b/arch/arm/mach-msm/perf_debug.c
@@ -31,6 +31,7 @@
"10 Perf: Fix counts across power collapse\n"
"12 Perf: Make per-process counters configurable\n"
"13 msm: perf: Add L2 support for tracecounters\n"
+ "14 Perf: keep events across hotplug\n"
;
static ssize_t desc_read(struct file *fp, char __user *buf,
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index ddbb6a9..bf99faa 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -740,6 +740,8 @@
int * __percpu pmu_disable_count;
struct perf_cpu_context * __percpu pmu_cpu_context;
int task_ctx_nr;
+ u32 events_across_hotplug:1,
+ reserved:31;
/*
* Fully disable/enable this PMU, can be used to protect from the PMI
diff --git a/kernel/events/core.c b/kernel/events/core.c
index aafa4c1..c9dc1ac 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -1194,6 +1194,28 @@
return 0;
}
+#ifdef CONFIG_SMP
+static void perf_retry_remove(struct perf_event *event)
+{
+ int up_ret;
+ /*
+ * CPU was offline. Bring it online so we can
+ * gracefully exit a perf context.
+ */
+ up_ret = cpu_up(event->cpu);
+ if (!up_ret)
+ /* Try the remove call once again. */
+ cpu_function_call(event->cpu, __perf_remove_from_context,
+ event);
+ else
+ pr_err("Failed to bring up CPU: %d, ret: %d\n",
+ event->cpu, up_ret);
+}
+#else
+static void perf_retry_remove(struct perf_event *event)
+{
+}
+#endif
/*
* Remove the event from a task's (or a CPU's) list of events.
@@ -1208,19 +1230,22 @@
* When called from perf_event_exit_task, it's OK because the
* context has been detached from its task.
*/
-static void perf_remove_from_context(struct perf_event *event)
+static void __ref perf_remove_from_context(struct perf_event *event)
{
struct perf_event_context *ctx = event->ctx;
struct task_struct *task = ctx->task;
+ int ret;
lockdep_assert_held(&ctx->mutex);
if (!task) {
/*
- * Per cpu events are removed via an smp call and
- * the removal is always successful.
+ * Per cpu events are removed via an smp call
*/
- cpu_function_call(event->cpu, __perf_remove_from_context, event);
+ ret = cpu_function_call(event->cpu, __perf_remove_from_context,
+ event);
+ if (ret == -ENXIO)
+ perf_retry_remove(event);
return;
}
@@ -7029,11 +7054,20 @@
idx = srcu_read_lock(&pmus_srcu);
list_for_each_entry_rcu(pmu, &pmus, entry) {
- ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
+ /*
+ * If keeping events across hotplugging is supported, do not
+ * remove the event list, but keep it alive across CPU hotplug.
+ * The context is exited via an fd close path when userspace
+ * is done and the target CPU is online.
+ */
+ if (!pmu->events_across_hotplug) {
+ ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
- mutex_lock(&ctx->mutex);
- smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
- mutex_unlock(&ctx->mutex);
+ mutex_lock(&ctx->mutex);
+ smp_call_function_single(cpu, __perf_event_exit_context,
+ ctx, 1);
+ mutex_unlock(&ctx->mutex);
+ }
}
srcu_read_unlock(&pmus_srcu, idx);
}