include/linux/lglock.h

/*
 * Specialised local-global spinlock. Can only be declared as global variables
 * to avoid overhead and keep things simple (and we don't want to start using
 * these inside dynamically allocated structures).
 *
 * "local/global locks" (lglocks) can be used to:
 *
 * - Provide fast exclusive access to per-CPU data, with exclusive access to
 *   another CPU's data allowed but possibly subject to contention, and to
 *   provide very slow exclusive access to all per-CPU data.
 * - Or to provide very fast and scalable read serialisation, and to provide
 *   very slow exclusive serialisation of data (not necessarily per-CPU data).
 *
 * Brlocks are also implemented as a short-hand notation for the latter use
 * case.
 *
 * Copyright 2009, 2010, Nick Piggin, Novell Inc.
 */
#ifndef __LINUX_LGLOCK_H
#define __LINUX_LGLOCK_H

#include <linux/spinlock.h>
#include <linux/lockdep.h>
#include <linux/percpu.h>
#include <linux/cpu.h>

#define br_lock_init(name)	name##_lock_init()
#define br_read_lock(name)	name##_local_lock()
#define br_read_unlock(name)	name##_local_unlock()
#define br_write_lock(name)	name##_global_lock_online()
#define br_write_unlock(name)	name##_global_unlock_online()

#define DECLARE_BRLOCK(name)	DECLARE_LGLOCK(name)
#define DEFINE_BRLOCK(name)	DEFINE_LGLOCK(name)


#define lg_lock_init(name)	name##_lock_init()
#define lg_local_lock(name)	name##_local_lock()
#define lg_local_unlock(name)	name##_local_unlock()
#define lg_local_lock_cpu(name, cpu)	name##_local_lock_cpu(cpu)
#define lg_local_unlock_cpu(name, cpu)	name##_local_unlock_cpu(cpu)
#define lg_global_lock(name)	name##_global_lock()
#define lg_global_unlock(name)	name##_global_unlock()
#define lg_global_lock_online(name) name##_global_lock_online()
#define lg_global_unlock_online(name) name##_global_unlock_online()

#ifdef CONFIG_DEBUG_LOCK_ALLOC
#define LOCKDEP_INIT_MAP lockdep_init_map

#define DEFINE_LGLOCK_LOCKDEP(name)					\
 struct lock_class_key name##_lock_key;					\
 struct lockdep_map name##_lock_dep_map;				\
 EXPORT_SYMBOL(name##_lock_dep_map)

#else
#define LOCKDEP_INIT_MAP(a, b, c, d)

#define DEFINE_LGLOCK_LOCKDEP(name)
#endif


#define DECLARE_LGLOCK(name)						\
 extern void name##_lock_init(void);					\
 extern void name##_local_lock(void);					\
 extern void name##_local_unlock(void);					\
 extern void name##_local_lock_cpu(int cpu);				\
 extern void name##_local_unlock_cpu(int cpu);				\
 extern void name##_global_lock(void);					\
 extern void name##_global_unlock(void);				\
 extern void name##_global_lock_online(void);				\
 extern void name##_global_unlock_online(void);				\

#define DEFINE_LGLOCK(name)						\
									\
 DEFINE_SPINLOCK(name##_cpu_lock);					\
 cpumask_t name##_cpus __read_mostly;					\
 DEFINE_PER_CPU(arch_spinlock_t, name##_lock);				\
 DEFINE_LGLOCK_LOCKDEP(name);						\
									\
 static int								\
 name##_lg_cpu_callback(struct notifier_block *nb,			\
				unsigned long action, void *hcpu)	\
 {									\
	switch (action & ~CPU_TASKS_FROZEN) {				\
	case CPU_UP_PREPARE:						\
		spin_lock(&name##_cpu_lock);				\
		cpu_set((unsigned long)hcpu, name##_cpus);		\
		spin_unlock(&name##_cpu_lock);				\
		break;							\
	case CPU_UP_CANCELED: case CPU_DEAD:				\
		spin_lock(&name##_cpu_lock);				\
		cpu_clear((unsigned long)hcpu, name##_cpus);		\
		spin_unlock(&name##_cpu_lock);				\
	}								\
	return NOTIFY_OK;						\
 }									\
 static struct notifier_block name##_lg_cpu_notifier = {		\
	.notifier_call = name##_lg_cpu_callback,			\
 };									\
 void name##_lock_init(void) {						\
	int i;								\
	LOCKDEP_INIT_MAP(&name##_lock_dep_map, #name, &name##_lock_key, 0); \
	for_each_possible_cpu(i) {					\
		arch_spinlock_t *lock;					\
		lock = &per_cpu(name##_lock, i);			\
		*lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;	\
	}								\
	register_hotcpu_notifier(&name##_lg_cpu_notifier);		\
	get_online_cpus();						\
	for_each_online_cpu(i)						\
		cpu_set(i, name##_cpus);				\
	put_online_cpus();						\
 }									\
 EXPORT_SYMBOL(name##_lock_init);					\
									\
 void name##_local_lock(void) {						\
	arch_spinlock_t *lock;						\
	preempt_disable();						\
	rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_);	\
	lock = &__get_cpu_var(name##_lock);				\
	arch_spin_lock(lock);						\
 }									\
 EXPORT_SYMBOL(name##_local_lock);					\
									\
 void name##_local_unlock(void) {					\
	arch_spinlock_t *lock;						\
	rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_);		\
	lock = &__get_cpu_var(name##_lock);				\
	arch_spin_unlock(lock);						\
	preempt_enable();						\
 }									\
 EXPORT_SYMBOL(name##_local_unlock);					\
									\
 void name##_local_lock_cpu(int cpu) {					\
	arch_spinlock_t *lock;						\
	preempt_disable();						\
	rwlock_acquire_read(&name##_lock_dep_map, 0, 0, _THIS_IP_);	\
	lock = &per_cpu(name##_lock, cpu);				\
	arch_spin_lock(lock);						\
 }									\
 EXPORT_SYMBOL(name##_local_lock_cpu);					\
									\
 void name##_local_unlock_cpu(int cpu) {				\
	arch_spinlock_t *lock;						\
	rwlock_release(&name##_lock_dep_map, 1, _THIS_IP_);		\
	lock = &per_cpu(name##_lock, cpu);				\
	arch_spin_unlock(lock);						\
	preempt_enable();						\
 }									\
 EXPORT_SYMBOL(name##_local_unlock_cpu);				\
									\
 void name##_global_lock_online(void) {					\
	int i;								\
	spin_lock(&name##_cpu_lock);					\
	rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_);		\
	for_each_cpu(i, &name##_cpus) {					\
		arch_spinlock_t *lock;					\
		lock = &per_cpu(name##_lock, i);			\
		arch_spin_lock(lock);					\
	}								\
 }									\
 EXPORT_SYMBOL(name##_global_lock_online);				\
									\
 void name##_global_unlock_online(void) {				\
	int i;								\
	rwlock_release(&name##_lock_dep_map, 1, _RET_IP_);		\
	for_each_cpu(i, &name##_cpus) {					\
		arch_spinlock_t *lock;					\
		lock = &per_cpu(name##_lock, i);			\
		arch_spin_unlock(lock);					\
	}								\
	spin_unlock(&name##_cpu_lock);					\
 }									\
 EXPORT_SYMBOL(name##_global_unlock_online);				\
									\
 void name##_global_lock(void) {					\
	int i;								\
	preempt_disable();						\
	rwlock_acquire(&name##_lock_dep_map, 0, 0, _RET_IP_);		\
	for_each_possible_cpu(i) {					\
		arch_spinlock_t *lock;					\
		lock = &per_cpu(name##_lock, i);			\
		arch_spin_lock(lock);					\
	}								\
 }									\
 EXPORT_SYMBOL(name##_global_lock);					\
									\
 void name##_global_unlock(void) {					\
	int i;								\
	rwlock_release(&name##_lock_dep_map, 1, _RET_IP_);		\
	for_each_possible_cpu(i) {					\
		arch_spinlock_t *lock;					\
		lock = &per_cpu(name##_lock, i);			\
		arch_spin_unlock(lock);					\
	}								\
	preempt_enable();						\
 }									\
 EXPORT_SYMBOL(name##_global_unlock);
#endif