core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
__get_cpu_var() can be replaced with this_cpu_read and will then use a
single read instruction with implied address calculation to access the
correct per cpu instance.
However, the address of a per cpu variable passed to __this_cpu_read()
cannot be determined (since it's an implied address conversion through
segment prefixes). Therefore apply this only to uses of __get_cpu_var
where the address of the variable is not used.
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Hugh Dickins <hughd@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@zytor.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index 4fd978e..4d85797 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -195,15 +195,9 @@
/*
* io context count accounting
*/
-#define elv_ioc_count_mod(name, __val) \
- do { \
- preempt_disable(); \
- __get_cpu_var(name) += (__val); \
- preempt_enable(); \
- } while (0)
-
-#define elv_ioc_count_inc(name) elv_ioc_count_mod(name, 1)
-#define elv_ioc_count_dec(name) elv_ioc_count_mod(name, -1)
+#define elv_ioc_count_mod(name, __val) this_cpu_add(name, __val)
+#define elv_ioc_count_inc(name) this_cpu_inc(name)
+#define elv_ioc_count_dec(name) this_cpu_dec(name)
#define elv_ioc_count_read(name) \
({ \
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index ad54c84..44e83ba 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -47,7 +47,7 @@
#ifndef CONFIG_GENERIC_HARDIRQS
#define kstat_irqs_this_cpu(irq) \
- (kstat_this_cpu.irqs[irq])
+ (this_cpu_read(kstat.irqs[irq])
struct irq_desc;