blob: d9c0368eeb2163815fd8b23ce08e22ce2ba90138 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * kernel/sched.c
3 *
4 * Kernel scheduler and related syscalls
5 *
6 * Copyright (C) 1991-2002 Linus Torvalds
7 *
8 * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
9 * make semaphores SMP safe
10 * 1998-11-19 Implemented schedule_timeout() and related stuff
11 * by Andrea Arcangeli
12 * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
13 * hybrid priority-list and round-robin design with
14 * an array-switch method of distributing timeslices
15 * and per-CPU runqueues. Cleanups and useful suggestions
16 * by Davide Libenzi, preemptible kernel bits by Robert Love.
17 * 2003-09-03 Interactivity tuning by Con Kolivas.
18 * 2004-04-02 Scheduler domains code by Nick Piggin
Ingo Molnarc31f2e82007-07-09 18:52:01 +020019 * 2007-04-15 Work begun on replacing all interactivity tuning with a
20 * fair scheduling design by Con Kolivas.
21 * 2007-05-05 Load balancing (smp-nice) and other improvements
22 * by Peter Williams
23 * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
24 * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
Ingo Molnarb9131762008-01-25 21:08:19 +010025 * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
26 * Thomas Gleixner, Mike Kravetz
Linus Torvalds1da177e2005-04-16 15:20:36 -070027 */
28
29#include <linux/mm.h>
30#include <linux/module.h>
31#include <linux/nmi.h>
32#include <linux/init.h>
Ingo Molnardff06c12007-07-09 18:52:00 +020033#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <linux/highmem.h>
35#include <linux/smp_lock.h>
36#include <asm/mmu_context.h>
37#include <linux/interrupt.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080038#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/completion.h>
40#include <linux/kernel_stat.h>
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070041#include <linux/debug_locks.h>
Ingo Molnarcdd6c482009-09-21 12:02:48 +020042#include <linux/perf_event.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#include <linux/security.h>
44#include <linux/notifier.h>
45#include <linux/profile.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080046#include <linux/freezer.h>
akpm@osdl.org198e2f12006-01-12 01:05:30 -080047#include <linux/vmalloc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#include <linux/blkdev.h>
49#include <linux/delay.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070050#include <linux/pid_namespace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070051#include <linux/smp.h>
52#include <linux/threads.h>
53#include <linux/timer.h>
54#include <linux/rcupdate.h>
55#include <linux/cpu.h>
56#include <linux/cpuset.h>
57#include <linux/percpu.h>
Alexey Dobriyanb5aadf72008-10-06 13:23:43 +040058#include <linux/proc_fs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070059#include <linux/seq_file.h>
Tejun Heo969c7922010-05-06 18:49:21 +020060#include <linux/stop_machine.h>
Nick Piggine692ab52007-07-26 13:40:43 +020061#include <linux/sysctl.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070062#include <linux/syscalls.h>
63#include <linux/times.h>
Jay Lan8f0ab512006-09-30 23:28:59 -070064#include <linux/tsacct_kern.h>
bibo maoc6fd91f2006-03-26 01:38:20 -080065#include <linux/kprobes.h>
Shailabh Nagar0ff92242006-07-14 00:24:37 -070066#include <linux/delayacct.h>
Ingo Molnardff06c12007-07-09 18:52:00 +020067#include <linux/unistd.h>
Jens Axboef5ff8422007-09-21 09:19:54 +020068#include <linux/pagemap.h>
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +010069#include <linux/hrtimer.h>
Reynes Philippe30914a52008-03-17 16:19:05 -070070#include <linux/tick.h>
Peter Zijlstraf00b45c2008-04-19 19:45:00 +020071#include <linux/debugfs.h>
72#include <linux/ctype.h>
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +020073#include <linux/ftrace.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090074#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070075
Eric Dumazet5517d862007-05-08 00:32:57 -070076#include <asm/tlb.h>
Satyam Sharma838225b2007-10-24 18:23:50 +020077#include <asm/irq_regs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070078
Gregory Haskins6e0534f2008-05-12 21:21:01 +020079#include "sched_cpupri.h"
80
Steven Rostedta8d154b2009-04-10 09:36:00 -040081#define CREATE_TRACE_POINTS
Steven Rostedtad8d75f2009-04-14 19:39:12 -040082#include <trace/events/sched.h>
Steven Rostedta8d154b2009-04-10 09:36:00 -040083
Linus Torvalds1da177e2005-04-16 15:20:36 -070084/*
85 * Convert user-nice values [ -20 ... 0 ... 19 ]
86 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
87 * and back.
88 */
89#define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20)
90#define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20)
91#define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio)
92
93/*
94 * 'User priority' is the nice value converted to something we
95 * can work with better when scaling various scheduler parameters,
96 * it's a [ 0 ... 39 ] range.
97 */
98#define USER_PRIO(p) ((p)-MAX_RT_PRIO)
99#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
100#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
101
102/*
Ingo Molnard7876a02008-01-25 21:08:19 +0100103 * Helpers for converting nanosecond timing to jiffy resolution
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 */
Eric Dumazetd6322fa2007-11-09 22:39:38 +0100105#define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200107#define NICE_0_LOAD SCHED_LOAD_SCALE
108#define NICE_0_SHIFT SCHED_LOAD_SHIFT
109
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110/*
111 * These are the 'tuning knobs' of the scheduler:
112 *
Dmitry Adamushkoa4ec24b2007-10-15 17:00:13 +0200113 * default timeslice is 100 msecs (used only for SCHED_RR tasks).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 * Timeslices get refilled after they expire.
115 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116#define DEF_TIMESLICE (100 * HZ / 1000)
Peter Williams2dd73a42006-06-27 02:54:34 -0700117
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200118/*
119 * single value that denotes runtime == period, ie unlimited time.
120 */
121#define RUNTIME_INF ((u64)~0ULL)
122
Ingo Molnare05606d2007-07-09 18:51:59 +0200123static inline int rt_policy(int policy)
124{
Roel Kluin3f33a7c2008-05-13 23:44:11 +0200125 if (unlikely(policy == SCHED_FIFO || policy == SCHED_RR))
Ingo Molnare05606d2007-07-09 18:51:59 +0200126 return 1;
127 return 0;
128}
129
130static inline int task_has_rt_policy(struct task_struct *p)
131{
132 return rt_policy(p->policy);
133}
134
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135/*
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200136 * This is the priority-queue data structure of the RT scheduling class:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 */
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200138struct rt_prio_array {
139 DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
140 struct list_head queue[MAX_RT_PRIO];
141};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200143struct rt_bandwidth {
Ingo Molnarea736ed2008-03-25 13:51:45 +0100144 /* nests inside the rq lock: */
Thomas Gleixner0986b112009-11-17 15:32:06 +0100145 raw_spinlock_t rt_runtime_lock;
Ingo Molnarea736ed2008-03-25 13:51:45 +0100146 ktime_t rt_period;
147 u64 rt_runtime;
148 struct hrtimer rt_period_timer;
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200149};
150
151static struct rt_bandwidth def_rt_bandwidth;
152
153static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
154
155static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
156{
157 struct rt_bandwidth *rt_b =
158 container_of(timer, struct rt_bandwidth, rt_period_timer);
159 ktime_t now;
160 int overrun;
161 int idle = 0;
162
163 for (;;) {
164 now = hrtimer_cb_get_time(timer);
165 overrun = hrtimer_forward(timer, now, rt_b->rt_period);
166
167 if (!overrun)
168 break;
169
170 idle = do_sched_rt_period_timer(rt_b, overrun);
171 }
172
173 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
174}
175
176static
177void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
178{
179 rt_b->rt_period = ns_to_ktime(period);
180 rt_b->rt_runtime = runtime;
181
Thomas Gleixner0986b112009-11-17 15:32:06 +0100182 raw_spin_lock_init(&rt_b->rt_runtime_lock);
Peter Zijlstraac086bc2008-04-19 19:44:58 +0200183
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200184 hrtimer_init(&rt_b->rt_period_timer,
185 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
186 rt_b->rt_period_timer.function = sched_rt_period_timer;
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200187}
188
Krzysztof Heltc8bfff62008-09-05 23:46:19 +0200189static inline int rt_bandwidth_enabled(void)
190{
191 return sysctl_sched_rt_runtime >= 0;
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200192}
193
194static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
195{
196 ktime_t now;
197
Hiroshi Shimamotocac64d02009-02-25 09:59:26 -0800198 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200199 return;
200
201 if (hrtimer_active(&rt_b->rt_period_timer))
202 return;
203
Thomas Gleixner0986b112009-11-17 15:32:06 +0100204 raw_spin_lock(&rt_b->rt_runtime_lock);
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200205 for (;;) {
Peter Zijlstra7f1e2ca2009-03-13 12:21:27 +0100206 unsigned long delta;
207 ktime_t soft, hard;
208
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200209 if (hrtimer_active(&rt_b->rt_period_timer))
210 break;
211
212 now = hrtimer_cb_get_time(&rt_b->rt_period_timer);
213 hrtimer_forward(&rt_b->rt_period_timer, now, rt_b->rt_period);
Peter Zijlstra7f1e2ca2009-03-13 12:21:27 +0100214
215 soft = hrtimer_get_softexpires(&rt_b->rt_period_timer);
216 hard = hrtimer_get_expires(&rt_b->rt_period_timer);
217 delta = ktime_to_ns(ktime_sub(hard, soft));
218 __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
Arun R Bharadwaj5c333862009-04-16 12:14:37 +0530219 HRTIMER_MODE_ABS_PINNED, 0);
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200220 }
Thomas Gleixner0986b112009-11-17 15:32:06 +0100221 raw_spin_unlock(&rt_b->rt_runtime_lock);
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200222}
223
224#ifdef CONFIG_RT_GROUP_SCHED
225static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
226{
227 hrtimer_cancel(&rt_b->rt_period_timer);
228}
229#endif
230
Heiko Carstens712555e2008-04-28 11:33:07 +0200231/*
232 * sched_domains_mutex serializes calls to arch_init_sched_domains,
233 * detach_destroy_domains and partition_sched_domains.
234 */
235static DEFINE_MUTEX(sched_domains_mutex);
236
Dhaval Giani7c941432010-01-20 13:26:18 +0100237#ifdef CONFIG_CGROUP_SCHED
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200238
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -0700239#include <linux/cgroup.h>
240
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200241struct cfs_rq;
242
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100243static LIST_HEAD(task_groups);
244
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200245/* task group related information */
Ingo Molnar4cf86d72007-10-15 17:00:14 +0200246struct task_group {
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -0700247 struct cgroup_subsys_state css;
Arun R Bharadwaj6c415b92008-12-01 20:49:05 +0530248
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100249#ifdef CONFIG_FAIR_GROUP_SCHED
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200250 /* schedulable entities of this group on each cpu */
251 struct sched_entity **se;
252 /* runqueue "owned" by this group on each cpu */
253 struct cfs_rq **cfs_rq;
254 unsigned long shares;
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100255#endif
256
257#ifdef CONFIG_RT_GROUP_SCHED
258 struct sched_rt_entity **rt_se;
259 struct rt_rq **rt_rq;
260
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200261 struct rt_bandwidth rt_bandwidth;
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100262#endif
Srivatsa Vaddagiri6b2d7702008-01-25 21:08:00 +0100263
Srivatsa Vaddagiriae8393e2007-10-29 21:18:11 +0100264 struct rcu_head rcu;
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100265 struct list_head list;
Peter Zijlstraf473aa52008-04-19 19:45:00 +0200266
267 struct task_group *parent;
268 struct list_head siblings;
269 struct list_head children;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200270};
271
Peter Zijlstraeff766a2008-04-19 19:45:00 +0200272#define root_task_group init_task_group
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100273
Peter Zijlstra8ed36992008-02-13 15:45:39 +0100274/* task_group_lock serializes add/remove of task groups and also changes to
Srivatsa Vaddagiriec2c5072008-01-25 21:07:59 +0100275 * a task group's cpu shares.
276 */
Peter Zijlstra8ed36992008-02-13 15:45:39 +0100277static DEFINE_SPINLOCK(task_group_lock);
Srivatsa Vaddagiriec2c5072008-01-25 21:07:59 +0100278
Cyrill Gorcunove9036b32009-10-26 22:24:14 +0300279#ifdef CONFIG_FAIR_GROUP_SCHED
280
Peter Zijlstra57310a92009-03-09 13:56:21 +0100281#ifdef CONFIG_SMP
282static int root_task_group_empty(void)
283{
284 return list_empty(&root_task_group.children);
285}
286#endif
287
Srivatsa Vaddagiri93f992c2008-01-25 21:07:59 +0100288# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
Srivatsa Vaddagiri24e377a2007-10-15 17:00:09 +0200289
Miao Xiecb4ad1f2008-04-28 12:54:56 +0800290/*
Lai Jiangshan2e084782008-06-12 16:42:58 +0800291 * A weight of 0 or 1 can cause arithmetics problems.
292 * A weight of a cfs_rq is the sum of weights of which entities
293 * are queued on this cfs_rq, so a weight of a entity should not be
294 * too large, so as the shares value of a task group.
Miao Xiecb4ad1f2008-04-28 12:54:56 +0800295 * (The default weight is 1024 - so there's no practical
296 * limitation from this.)
297 */
Peter Zijlstra18d95a22008-04-19 19:45:00 +0200298#define MIN_SHARES 2
Lai Jiangshan2e084782008-06-12 16:42:58 +0800299#define MAX_SHARES (1UL << 18)
Peter Zijlstra18d95a22008-04-19 19:45:00 +0200300
Srivatsa Vaddagiri93f992c2008-01-25 21:07:59 +0100301static int init_task_group_load = INIT_TASK_GROUP_LOAD;
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100302#endif
303
304/* Default task group.
305 * Every task in system belong to this group at bootup.
306 */
Mike Travis434d53b2008-04-04 18:11:04 -0700307struct task_group init_task_group;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200308
309/* return group to which a task belongs */
Ingo Molnar4cf86d72007-10-15 17:00:14 +0200310static inline struct task_group *task_group(struct task_struct *p)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200311{
Ingo Molnar4cf86d72007-10-15 17:00:14 +0200312 struct task_group *tg;
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +0200313
Dhaval Giani7c941432010-01-20 13:26:18 +0100314#ifdef CONFIG_CGROUP_SCHED
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -0700315 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
316 struct task_group, css);
Srivatsa Vaddagiri24e377a2007-10-15 17:00:09 +0200317#else
Ingo Molnar41a2d6c2007-12-05 15:46:09 +0100318 tg = &init_task_group;
Srivatsa Vaddagiri24e377a2007-10-15 17:00:09 +0200319#endif
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +0200320 return tg;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200321}
322
323/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100324static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200325{
Peter Zijlstra8b08ca52010-04-21 13:02:07 -0700326 /*
327 * Strictly speaking this rcu_read_lock() is not needed since the
328 * task_group is tied to the cgroup, which in turn can never go away
329 * as long as there are tasks attached to it.
330 *
331 * However since task_group() uses task_subsys_state() which is an
332 * rcu_dereference() user, this quiets CONFIG_PROVE_RCU.
333 */
334 rcu_read_lock();
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100335#ifdef CONFIG_FAIR_GROUP_SCHED
Dmitry Adamushkoce96b5a2007-11-15 20:57:40 +0100336 p->se.cfs_rq = task_group(p)->cfs_rq[cpu];
337 p->se.parent = task_group(p)->se[cpu];
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100338#endif
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100339
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100340#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100341 p->rt.rt_rq = task_group(p)->rt_rq[cpu];
342 p->rt.parent = task_group(p)->rt_se[cpu];
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100343#endif
Peter Zijlstra8b08ca52010-04-21 13:02:07 -0700344 rcu_read_unlock();
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200345}
346
347#else
348
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100349static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
Peter Zijlstra83378262008-06-27 13:41:37 +0200350static inline struct task_group *task_group(struct task_struct *p)
351{
352 return NULL;
353}
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200354
Dhaval Giani7c941432010-01-20 13:26:18 +0100355#endif /* CONFIG_CGROUP_SCHED */
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +0200356
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200357/* CFS-related fields in a runqueue */
358struct cfs_rq {
359 struct load_weight load;
360 unsigned long nr_running;
361
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200362 u64 exec_clock;
Ingo Molnare9acbff2007-10-15 17:00:04 +0200363 u64 min_vruntime;
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200364
365 struct rb_root tasks_timeline;
366 struct rb_node *rb_leftmost;
Peter Zijlstra4a55bd52008-04-19 19:45:00 +0200367
368 struct list_head tasks;
369 struct list_head *balance_iterator;
370
371 /*
372 * 'curr' points to currently running entity on this cfs_rq.
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200373 * It is set to NULL otherwise (i.e when none are currently running).
374 */
Peter Zijlstra47932412008-11-04 21:25:09 +0100375 struct sched_entity *curr, *next, *last;
Peter Zijlstraddc97292007-10-15 17:00:10 +0200376
Peter Zijlstra5ac5c4d2008-11-10 10:46:32 +0100377 unsigned int nr_spread_over;
Peter Zijlstraddc97292007-10-15 17:00:10 +0200378
Ingo Molnar62160e32007-10-15 17:00:03 +0200379#ifdef CONFIG_FAIR_GROUP_SCHED
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200380 struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */
381
Ingo Molnar41a2d6c2007-12-05 15:46:09 +0100382 /*
383 * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200384 * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
385 * (like users, containers etc.)
386 *
387 * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
388 * list is used during load balance.
389 */
Ingo Molnar41a2d6c2007-12-05 15:46:09 +0100390 struct list_head leaf_cfs_rq_list;
391 struct task_group *tg; /* group that "owns" this runqueue */
Peter Zijlstrac09595f2008-06-27 13:41:14 +0200392
393#ifdef CONFIG_SMP
Peter Zijlstrac09595f2008-06-27 13:41:14 +0200394 /*
Peter Zijlstrac8cba852008-06-27 13:41:23 +0200395 * the part of load.weight contributed by tasks
Peter Zijlstrac09595f2008-06-27 13:41:14 +0200396 */
Peter Zijlstrac8cba852008-06-27 13:41:23 +0200397 unsigned long task_weight;
Peter Zijlstrac09595f2008-06-27 13:41:14 +0200398
Peter Zijlstrac8cba852008-06-27 13:41:23 +0200399 /*
400 * h_load = weight * f(tg)
401 *
402 * Where f(tg) is the recursive weight fraction assigned to
403 * this group.
404 */
405 unsigned long h_load;
Peter Zijlstrac09595f2008-06-27 13:41:14 +0200406
Peter Zijlstrac8cba852008-06-27 13:41:23 +0200407 /*
408 * this cpu's part of tg->shares
409 */
410 unsigned long shares;
Peter Zijlstraf1d239f2008-06-27 13:41:38 +0200411
412 /*
413 * load.weight at the time we set shares
414 */
415 unsigned long rq_weight;
Peter Zijlstrac09595f2008-06-27 13:41:14 +0200416#endif
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200417#endif
418};
419
420/* Real-Time classes' related field in a runqueue: */
421struct rt_rq {
422 struct rt_prio_array active;
Steven Rostedt63489e42008-01-25 21:08:03 +0100423 unsigned long rt_nr_running;
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100424#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
Gregory Haskinse864c492008-12-29 09:39:49 -0500425 struct {
426 int curr; /* highest queued rt task prio */
Gregory Haskins398a1532009-01-14 09:10:04 -0500427#ifdef CONFIG_SMP
Gregory Haskinse864c492008-12-29 09:39:49 -0500428 int next; /* next highest */
Gregory Haskins398a1532009-01-14 09:10:04 -0500429#endif
Gregory Haskinse864c492008-12-29 09:39:49 -0500430 } highest_prio;
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100431#endif
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100432#ifdef CONFIG_SMP
Gregory Haskins73fe6aa2008-01-25 21:08:07 +0100433 unsigned long rt_nr_migratory;
Peter Zijlstraa1ba4d82009-04-01 18:40:15 +0200434 unsigned long rt_nr_total;
Gregory Haskinsa22d7fc2008-01-25 21:08:12 +0100435 int overloaded;
Gregory Haskins917b6272008-12-29 09:39:53 -0500436 struct plist_head pushable_tasks;
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100437#endif
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100438 int rt_throttled;
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100439 u64 rt_time;
Peter Zijlstraac086bc2008-04-19 19:44:58 +0200440 u64 rt_runtime;
Ingo Molnarea736ed2008-03-25 13:51:45 +0100441 /* Nests inside the rq lock: */
Thomas Gleixner0986b112009-11-17 15:32:06 +0100442 raw_spinlock_t rt_runtime_lock;
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100443
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100444#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra23b0fdf2008-02-13 15:45:39 +0100445 unsigned long rt_nr_boosted;
446
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100447 struct rq *rq;
448 struct list_head leaf_rt_rq_list;
449 struct task_group *tg;
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100450#endif
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200451};
452
Gregory Haskins57d885f2008-01-25 21:08:18 +0100453#ifdef CONFIG_SMP
454
455/*
456 * We add the notion of a root-domain which will be used to define per-domain
Ingo Molnar0eab9142008-01-25 21:08:19 +0100457 * variables. Each exclusive cpuset essentially defines an island domain by
458 * fully partitioning the member cpus from any other cpuset. Whenever a new
Gregory Haskins57d885f2008-01-25 21:08:18 +0100459 * exclusive cpuset is created, we also create and attach a new root-domain
460 * object.
461 *
Gregory Haskins57d885f2008-01-25 21:08:18 +0100462 */
463struct root_domain {
464 atomic_t refcount;
Rusty Russellc6c49272008-11-25 02:35:05 +1030465 cpumask_var_t span;
466 cpumask_var_t online;
Gregory Haskins637f5082008-01-25 21:08:18 +0100467
Ingo Molnar0eab9142008-01-25 21:08:19 +0100468 /*
Gregory Haskins637f5082008-01-25 21:08:18 +0100469 * The "RT overload" flag: it gets set if a CPU has more than
470 * one runnable RT task.
471 */
Rusty Russellc6c49272008-11-25 02:35:05 +1030472 cpumask_var_t rto_mask;
Ingo Molnar0eab9142008-01-25 21:08:19 +0100473 atomic_t rto_count;
Gregory Haskins6e0534f2008-05-12 21:21:01 +0200474#ifdef CONFIG_SMP
475 struct cpupri cpupri;
476#endif
Gregory Haskins57d885f2008-01-25 21:08:18 +0100477};
478
Gregory Haskinsdc938522008-01-25 21:08:26 +0100479/*
480 * By default the system creates a single root-domain with all cpus as
481 * members (mimicking the global state we have today).
482 */
Gregory Haskins57d885f2008-01-25 21:08:18 +0100483static struct root_domain def_root_domain;
484
485#endif
486
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200487/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 * This is the main, per-CPU runqueue data structure.
489 *
490 * Locking rule: those places that want to lock multiple runqueues
491 * (such as the load balancing or the thread migration code), lock
492 * acquire operations must be ordered by ascending &runqueue.
493 */
Ingo Molnar70b97a72006-07-03 00:25:42 -0700494struct rq {
Ingo Molnard8016492007-10-18 21:32:55 +0200495 /* runqueue lock: */
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100496 raw_spinlock_t lock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497
498 /*
499 * nr_running and cpu_load should be in the same cacheline because
500 * remote CPUs use both these fields when doing load calculation.
501 */
502 unsigned long nr_running;
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200503 #define CPU_LOAD_IDX_MAX 5
504 unsigned long cpu_load[CPU_LOAD_IDX_MAX];
Siddha, Suresh B46cb4b72007-05-08 00:32:51 -0700505#ifdef CONFIG_NO_HZ
Mike Galbraith39c0cbe2010-03-11 17:17:13 +0100506 u64 nohz_stamp;
Siddha, Suresh B46cb4b72007-05-08 00:32:51 -0700507 unsigned char in_nohz_recently;
508#endif
Mike Galbraitha64692a2010-03-11 17:16:20 +0100509 unsigned int skip_clock_update;
510
Ingo Molnard8016492007-10-18 21:32:55 +0200511 /* capture load from *all* tasks on this cpu: */
512 struct load_weight load;
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200513 unsigned long nr_load_updates;
514 u64 nr_switches;
515
516 struct cfs_rq cfs;
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100517 struct rt_rq rt;
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100518
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200519#ifdef CONFIG_FAIR_GROUP_SCHED
Ingo Molnard8016492007-10-18 21:32:55 +0200520 /* list of leaf cfs_rq on this cpu: */
521 struct list_head leaf_cfs_rq_list;
Peter Zijlstra052f1dc2008-02-13 15:45:40 +0100522#endif
523#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra6f505b12008-01-25 21:08:30 +0100524 struct list_head leaf_rt_rq_list;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526
527 /*
528 * This is part of a global counter where only the total sum
529 * over all CPUs matters. A task can increase this counter on
530 * one CPU and if it got migrated afterwards it may decrease
531 * it on another CPU. Always updated under the runqueue lock:
532 */
533 unsigned long nr_uninterruptible;
534
Ingo Molnar36c8b582006-07-03 00:25:41 -0700535 struct task_struct *curr, *idle;
Christoph Lameterc9819f42006-12-10 02:20:25 -0800536 unsigned long next_balance;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 struct mm_struct *prev_mm;
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200538
Peter Zijlstra3e51f332008-05-03 18:29:28 +0200539 u64 clock;
Ingo Molnar6aa645e2007-07-09 18:51:58 +0200540
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541 atomic_t nr_iowait;
542
543#ifdef CONFIG_SMP
Ingo Molnar0eab9142008-01-25 21:08:19 +0100544 struct root_domain *rd;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 struct sched_domain *sd;
546
Henrik Austada0a522c2009-02-13 20:35:45 +0100547 unsigned char idle_at_tick;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548 /* For active balancing */
Gregory Haskins3f029d32009-07-29 11:08:47 -0400549 int post_schedule;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 int active_balance;
551 int push_cpu;
Tejun Heo969c7922010-05-06 18:49:21 +0200552 struct cpu_stop_work active_balance_work;
Ingo Molnard8016492007-10-18 21:32:55 +0200553 /* cpu of this runqueue: */
554 int cpu;
Gregory Haskins1f11eb62008-06-04 15:04:05 -0400555 int online;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556
Peter Zijlstraa8a51d52008-06-27 13:41:26 +0200557 unsigned long avg_load_per_task;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558
Peter Zijlstrae9e92502009-09-01 10:34:37 +0200559 u64 rt_avg;
560 u64 age_stamp;
Mike Galbraith1b9508f2009-11-04 17:53:50 +0100561 u64 idle_stamp;
562 u64 avg_idle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563#endif
564
Thomas Gleixnerdce48a82009-04-11 10:43:41 +0200565 /* calc_load related fields */
566 unsigned long calc_load_update;
567 long calc_load_active;
568
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +0100569#ifdef CONFIG_SCHED_HRTICK
Peter Zijlstra31656512008-07-18 18:01:23 +0200570#ifdef CONFIG_SMP
571 int hrtick_csd_pending;
572 struct call_single_data hrtick_csd;
573#endif
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +0100574 struct hrtimer hrtick_timer;
575#endif
576
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577#ifdef CONFIG_SCHEDSTATS
578 /* latency stats */
579 struct sched_info rq_sched_info;
Ken Chen9c2c4802008-12-16 23:41:22 -0800580 unsigned long long rq_cpu_time;
581 /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582
583 /* sys_sched_yield() stats */
Ken Chen480b9432007-10-18 21:32:56 +0200584 unsigned int yld_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
586 /* schedule() stats */
Ken Chen480b9432007-10-18 21:32:56 +0200587 unsigned int sched_switch;
588 unsigned int sched_count;
589 unsigned int sched_goidle;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590
591 /* try_to_wake_up() stats */
Ken Chen480b9432007-10-18 21:32:56 +0200592 unsigned int ttwu_count;
593 unsigned int ttwu_local;
Ingo Molnarb8efb562007-10-15 17:00:10 +0200594
595 /* BKL stats */
Ken Chen480b9432007-10-18 21:32:56 +0200596 unsigned int bkl_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597#endif
598};
599
Fenghua Yuf34e3b62007-07-19 01:48:13 -0700600static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601
Peter Zijlstra7d478722009-09-14 19:55:44 +0200602static inline
603void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
Ingo Molnardd41f592007-07-09 18:51:59 +0200604{
Peter Zijlstra7d478722009-09-14 19:55:44 +0200605 rq->curr->sched_class->check_preempt_curr(rq, p, flags);
Mike Galbraitha64692a2010-03-11 17:16:20 +0100606
607 /*
608 * A queue event has occurred, and we're going to schedule. In
609 * this case, we can save a useless back to back clock update.
610 */
611 if (test_tsk_need_resched(p))
612 rq->skip_clock_update = 1;
Ingo Molnardd41f592007-07-09 18:51:59 +0200613}
614
Christoph Lameter0a2966b2006-09-25 23:30:51 -0700615static inline int cpu_of(struct rq *rq)
616{
617#ifdef CONFIG_SMP
618 return rq->cpu;
619#else
620 return 0;
621#endif
622}
623
Paul E. McKenney497f0ab2010-02-22 17:04:51 -0800624#define rcu_dereference_check_sched_domain(p) \
Paul E. McKenneyd11c5632010-02-22 17:04:50 -0800625 rcu_dereference_check((p), \
626 rcu_read_lock_sched_held() || \
627 lockdep_is_held(&sched_domains_mutex))
628
Ingo Molnar20d315d2007-07-09 18:51:58 +0200629/*
Nick Piggin674311d2005-06-25 14:57:27 -0700630 * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -0700631 * See detach_destroy_domains: synchronize_sched for details.
Nick Piggin674311d2005-06-25 14:57:27 -0700632 *
633 * The domain tree of any CPU may only be accessed from within
634 * preempt-disabled sections.
635 */
Ingo Molnar48f24c42006-07-03 00:25:40 -0700636#define for_each_domain(cpu, __sd) \
Paul E. McKenney497f0ab2010-02-22 17:04:51 -0800637 for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
639#define cpu_rq(cpu) (&per_cpu(runqueues, (cpu)))
640#define this_rq() (&__get_cpu_var(runqueues))
641#define task_rq(p) cpu_rq(task_cpu(p))
642#define cpu_curr(cpu) (cpu_rq(cpu)->curr)
Hitoshi Mitake54d35f22009-06-29 14:44:57 +0900643#define raw_rq() (&__raw_get_cpu_var(runqueues))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644
Ingo Molnaraa9c4c02008-12-17 14:10:57 +0100645inline void update_rq_clock(struct rq *rq)
Peter Zijlstra3e51f332008-05-03 18:29:28 +0200646{
Mike Galbraitha64692a2010-03-11 17:16:20 +0100647 if (!rq->skip_clock_update)
648 rq->clock = sched_clock_cpu(cpu_of(rq));
Peter Zijlstra3e51f332008-05-03 18:29:28 +0200649}
650
Ingo Molnare436d802007-07-19 21:28:35 +0200651/*
Ingo Molnarbf5c91b2007-10-15 17:00:04 +0200652 * Tunables that become constants when CONFIG_SCHED_DEBUG is off:
653 */
654#ifdef CONFIG_SCHED_DEBUG
655# define const_debug __read_mostly
656#else
657# define const_debug static const
658#endif
659
Ingo Molnar017730c2008-05-12 21:20:52 +0200660/**
661 * runqueue_is_locked
Randy Dunlape17b38b2009-10-11 19:12:00 -0700662 * @cpu: the processor in question.
Ingo Molnar017730c2008-05-12 21:20:52 +0200663 *
664 * Returns true if the current cpu runqueue is locked.
665 * This interface allows printk to be called with the runqueue lock
666 * held and know whether or not it is OK to wake up the klogd.
667 */
Andrew Morton89f19f02009-09-19 11:55:44 -0700668int runqueue_is_locked(int cpu)
Ingo Molnar017730c2008-05-12 21:20:52 +0200669{
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100670 return raw_spin_is_locked(&cpu_rq(cpu)->lock);
Ingo Molnar017730c2008-05-12 21:20:52 +0200671}
672
Ingo Molnarbf5c91b2007-10-15 17:00:04 +0200673/*
674 * Debugging: various feature bits
675 */
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200676
677#define SCHED_FEAT(name, enabled) \
678 __SCHED_FEAT_##name ,
679
Ingo Molnarbf5c91b2007-10-15 17:00:04 +0200680enum {
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200681#include "sched_features.h"
Ingo Molnarbf5c91b2007-10-15 17:00:04 +0200682};
683
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200684#undef SCHED_FEAT
Ingo Molnarbf5c91b2007-10-15 17:00:04 +0200685
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200686#define SCHED_FEAT(name, enabled) \
687 (1UL << __SCHED_FEAT_##name) * enabled |
688
689const_debug unsigned int sysctl_sched_features =
690#include "sched_features.h"
691 0;
692
693#undef SCHED_FEAT
694
695#ifdef CONFIG_SCHED_DEBUG
696#define SCHED_FEAT(name, enabled) \
697 #name ,
698
Harvey Harrison983ed7a2008-04-24 18:17:55 -0700699static __read_mostly char *sched_feat_names[] = {
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200700#include "sched_features.h"
701 NULL
702};
703
704#undef SCHED_FEAT
705
Li Zefan34f3a812008-10-30 15:23:32 +0800706static int sched_feat_show(struct seq_file *m, void *v)
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200707{
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200708 int i;
709
710 for (i = 0; sched_feat_names[i]; i++) {
Li Zefan34f3a812008-10-30 15:23:32 +0800711 if (!(sysctl_sched_features & (1UL << i)))
712 seq_puts(m, "NO_");
713 seq_printf(m, "%s ", sched_feat_names[i]);
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200714 }
Li Zefan34f3a812008-10-30 15:23:32 +0800715 seq_puts(m, "\n");
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200716
Li Zefan34f3a812008-10-30 15:23:32 +0800717 return 0;
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200718}
719
720static ssize_t
721sched_feat_write(struct file *filp, const char __user *ubuf,
722 size_t cnt, loff_t *ppos)
723{
724 char buf[64];
725 char *cmp = buf;
726 int neg = 0;
727 int i;
728
729 if (cnt > 63)
730 cnt = 63;
731
732 if (copy_from_user(&buf, ubuf, cnt))
733 return -EFAULT;
734
735 buf[cnt] = 0;
736
Ingo Molnarc24b7c52008-04-18 10:55:34 +0200737 if (strncmp(buf, "NO_", 3) == 0) {
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200738 neg = 1;
739 cmp += 3;
740 }
741
742 for (i = 0; sched_feat_names[i]; i++) {
743 int len = strlen(sched_feat_names[i]);
744
745 if (strncmp(cmp, sched_feat_names[i], len) == 0) {
746 if (neg)
747 sysctl_sched_features &= ~(1UL << i);
748 else
749 sysctl_sched_features |= (1UL << i);
750 break;
751 }
752 }
753
754 if (!sched_feat_names[i])
755 return -EINVAL;
756
Jan Blunck42994722009-11-20 17:40:37 +0100757 *ppos += cnt;
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200758
759 return cnt;
760}
761
Li Zefan34f3a812008-10-30 15:23:32 +0800762static int sched_feat_open(struct inode *inode, struct file *filp)
763{
764 return single_open(filp, sched_feat_show, NULL);
765}
766
Alexey Dobriyan828c0952009-10-01 15:43:56 -0700767static const struct file_operations sched_feat_fops = {
Li Zefan34f3a812008-10-30 15:23:32 +0800768 .open = sched_feat_open,
769 .write = sched_feat_write,
770 .read = seq_read,
771 .llseek = seq_lseek,
772 .release = single_release,
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200773};
774
775static __init int sched_init_debug(void)
776{
Peter Zijlstraf00b45c2008-04-19 19:45:00 +0200777 debugfs_create_file("sched_features", 0644, NULL, NULL,
778 &sched_feat_fops);
779
780 return 0;
781}
782late_initcall(sched_init_debug);
783
784#endif
785
786#define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x))
Ingo Molnarbf5c91b2007-10-15 17:00:04 +0200787
788/*
Peter Zijlstrab82d9fd2007-11-09 22:39:39 +0100789 * Number of tasks to iterate in a single balance run.
790 * Limited because this is done with IRQs disabled.
791 */
792const_debug unsigned int sysctl_sched_nr_migrate = 32;
793
794/*
Peter Zijlstra2398f2c2008-06-27 13:41:35 +0200795 * ratelimit for updating the group shares.
Peter Zijlstra55cd5342008-08-04 08:54:26 +0200796 * default: 0.25ms
Peter Zijlstra2398f2c2008-06-27 13:41:35 +0200797 */
Peter Zijlstra55cd5342008-08-04 08:54:26 +0200798unsigned int sysctl_sched_shares_ratelimit = 250000;
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +0100799unsigned int normalized_sysctl_sched_shares_ratelimit = 250000;
Peter Zijlstra2398f2c2008-06-27 13:41:35 +0200800
801/*
Peter Zijlstraffda12a2008-10-17 19:27:02 +0200802 * Inject some fuzzyness into changing the per-cpu group shares
803 * this avoids remote rq-locks at the expense of fairness.
804 * default: 4
805 */
806unsigned int sysctl_sched_shares_thresh = 4;
807
808/*
Peter Zijlstrae9e92502009-09-01 10:34:37 +0200809 * period over which we average the RT time consumption, measured
810 * in ms.
811 *
812 * default: 1s
813 */
814const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
815
816/*
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +0100817 * period over which we measure -rt task cpu usage in us.
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100818 * default: 1s
819 */
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +0100820unsigned int sysctl_sched_rt_period = 1000000;
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100821
Ingo Molnar6892b752008-02-13 14:02:36 +0100822static __read_mostly int scheduler_running;
823
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100824/*
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +0100825 * part of the period that we allow rt tasks to run in us.
826 * default: 0.95s
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100827 */
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +0100828int sysctl_sched_rt_runtime = 950000;
829
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200830static inline u64 global_rt_period(void)
831{
832 return (u64)sysctl_sched_rt_period * NSEC_PER_USEC;
833}
834
835static inline u64 global_rt_runtime(void)
836{
roel kluine26873b2008-07-22 16:51:15 -0400837 if (sysctl_sched_rt_runtime < 0)
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +0200838 return RUNTIME_INF;
839
840 return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
841}
Peter Zijlstrafa85ae22008-01-25 21:08:29 +0100842
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843#ifndef prepare_arch_switch
Nick Piggin4866cde2005-06-25 14:57:23 -0700844# define prepare_arch_switch(next) do { } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845#endif
Nick Piggin4866cde2005-06-25 14:57:23 -0700846#ifndef finish_arch_switch
847# define finish_arch_switch(prev) do { } while (0)
848#endif
849
Dmitry Adamushko051a1d12007-12-18 15:21:13 +0100850static inline int task_current(struct rq *rq, struct task_struct *p)
851{
852 return rq->curr == p;
853}
854
Nick Piggin4866cde2005-06-25 14:57:23 -0700855#ifndef __ARCH_WANT_UNLOCKED_CTXSW
Ingo Molnar70b97a72006-07-03 00:25:42 -0700856static inline int task_running(struct rq *rq, struct task_struct *p)
Nick Piggin4866cde2005-06-25 14:57:23 -0700857{
Dmitry Adamushko051a1d12007-12-18 15:21:13 +0100858 return task_current(rq, p);
Nick Piggin4866cde2005-06-25 14:57:23 -0700859}
860
Ingo Molnar70b97a72006-07-03 00:25:42 -0700861static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
Nick Piggin4866cde2005-06-25 14:57:23 -0700862{
863}
864
Ingo Molnar70b97a72006-07-03 00:25:42 -0700865static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
Nick Piggin4866cde2005-06-25 14:57:23 -0700866{
Ingo Molnarda04c032005-09-13 11:17:59 +0200867#ifdef CONFIG_DEBUG_SPINLOCK
868 /* this is a valid case when another task releases the spinlock */
869 rq->lock.owner = current;
870#endif
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700871 /*
872 * If we are tracking spinlock dependencies then we have to
873 * fix up the runqueue lock - which gets 'carried over' from
874 * prev into current:
875 */
876 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
877
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100878 raw_spin_unlock_irq(&rq->lock);
Nick Piggin4866cde2005-06-25 14:57:23 -0700879}
880
881#else /* __ARCH_WANT_UNLOCKED_CTXSW */
Ingo Molnar70b97a72006-07-03 00:25:42 -0700882static inline int task_running(struct rq *rq, struct task_struct *p)
Nick Piggin4866cde2005-06-25 14:57:23 -0700883{
884#ifdef CONFIG_SMP
885 return p->oncpu;
886#else
Dmitry Adamushko051a1d12007-12-18 15:21:13 +0100887 return task_current(rq, p);
Nick Piggin4866cde2005-06-25 14:57:23 -0700888#endif
889}
890
Ingo Molnar70b97a72006-07-03 00:25:42 -0700891static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
Nick Piggin4866cde2005-06-25 14:57:23 -0700892{
893#ifdef CONFIG_SMP
894 /*
895 * We can optimise this out completely for !SMP, because the
896 * SMP rebalancing from interrupt is the only thing that cares
897 * here.
898 */
899 next->oncpu = 1;
900#endif
901#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100902 raw_spin_unlock_irq(&rq->lock);
Nick Piggin4866cde2005-06-25 14:57:23 -0700903#else
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100904 raw_spin_unlock(&rq->lock);
Nick Piggin4866cde2005-06-25 14:57:23 -0700905#endif
906}
907
Ingo Molnar70b97a72006-07-03 00:25:42 -0700908static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
Nick Piggin4866cde2005-06-25 14:57:23 -0700909{
910#ifdef CONFIG_SMP
911 /*
912 * After ->oncpu is cleared, the task can be moved to a different CPU.
913 * We must ensure this doesn't happen until the switch is completely
914 * finished.
915 */
916 smp_wmb();
917 prev->oncpu = 0;
918#endif
919#ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW
920 local_irq_enable();
921#endif
922}
923#endif /* __ARCH_WANT_UNLOCKED_CTXSW */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924
925/*
Peter Zijlstra65cc8e42010-03-25 21:05:16 +0100926 * Check whether the task is waking, we use this to synchronize ->cpus_allowed
927 * against ttwu().
Peter Zijlstra0970d292010-02-15 14:45:54 +0100928 */
929static inline int task_is_waking(struct task_struct *p)
930{
Peter Zijlstra0017d732010-03-24 18:34:10 +0100931 return unlikely(p->state == TASK_WAKING);
Peter Zijlstra0970d292010-02-15 14:45:54 +0100932}
933
934/*
Ingo Molnarb29739f2006-06-27 02:54:51 -0700935 * __task_rq_lock - lock the runqueue a given task resides on.
936 * Must be called interrupts disabled.
937 */
Ingo Molnar70b97a72006-07-03 00:25:42 -0700938static inline struct rq *__task_rq_lock(struct task_struct *p)
Ingo Molnarb29739f2006-06-27 02:54:51 -0700939 __acquires(rq->lock)
940{
Peter Zijlstra0970d292010-02-15 14:45:54 +0100941 struct rq *rq;
942
Andi Kleen3a5c3592007-10-15 17:00:14 +0200943 for (;;) {
Peter Zijlstra0970d292010-02-15 14:45:54 +0100944 rq = task_rq(p);
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100945 raw_spin_lock(&rq->lock);
Peter Zijlstra65cc8e42010-03-25 21:05:16 +0100946 if (likely(rq == task_rq(p)))
Andi Kleen3a5c3592007-10-15 17:00:14 +0200947 return rq;
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100948 raw_spin_unlock(&rq->lock);
Ingo Molnarb29739f2006-06-27 02:54:51 -0700949 }
Ingo Molnarb29739f2006-06-27 02:54:51 -0700950}
951
952/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700953 * task_rq_lock - lock the runqueue a given task resides on and disable
Ingo Molnar41a2d6c2007-12-05 15:46:09 +0100954 * interrupts. Note the ordering: we can safely lookup the task_rq without
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955 * explicitly disabling preemption.
956 */
Ingo Molnar70b97a72006-07-03 00:25:42 -0700957static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 __acquires(rq->lock)
959{
Ingo Molnar70b97a72006-07-03 00:25:42 -0700960 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961
Andi Kleen3a5c3592007-10-15 17:00:14 +0200962 for (;;) {
963 local_irq_save(*flags);
964 rq = task_rq(p);
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100965 raw_spin_lock(&rq->lock);
Peter Zijlstra65cc8e42010-03-25 21:05:16 +0100966 if (likely(rq == task_rq(p)))
Andi Kleen3a5c3592007-10-15 17:00:14 +0200967 return rq;
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100968 raw_spin_unlock_irqrestore(&rq->lock, *flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970}
971
Oleg Nesterovad474ca2008-11-10 15:39:30 +0100972void task_rq_unlock_wait(struct task_struct *p)
973{
974 struct rq *rq = task_rq(p);
975
976 smp_mb(); /* spin-unlock-wait is not a full memory barrier */
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100977 raw_spin_unlock_wait(&rq->lock);
Oleg Nesterovad474ca2008-11-10 15:39:30 +0100978}
979
Alexey Dobriyana9957442007-10-15 17:00:13 +0200980static void __task_rq_unlock(struct rq *rq)
Ingo Molnarb29739f2006-06-27 02:54:51 -0700981 __releases(rq->lock)
982{
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100983 raw_spin_unlock(&rq->lock);
Ingo Molnarb29739f2006-06-27 02:54:51 -0700984}
985
Ingo Molnar70b97a72006-07-03 00:25:42 -0700986static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987 __releases(rq->lock)
988{
Thomas Gleixner05fa7852009-11-17 14:28:38 +0100989 raw_spin_unlock_irqrestore(&rq->lock, *flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990}
991
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992/*
Robert P. J. Daycc2a73b2006-12-10 02:20:00 -0800993 * this_rq_lock - lock this runqueue and disable interrupts.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994 */
Alexey Dobriyana9957442007-10-15 17:00:13 +0200995static struct rq *this_rq_lock(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996 __acquires(rq->lock)
997{
Ingo Molnar70b97a72006-07-03 00:25:42 -0700998 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999
1000 local_irq_disable();
1001 rq = this_rq();
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001002 raw_spin_lock(&rq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003
1004 return rq;
1005}
1006
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001007#ifdef CONFIG_SCHED_HRTICK
1008/*
1009 * Use HR-timers to deliver accurate preemption points.
1010 *
1011 * Its all a bit involved since we cannot program an hrt while holding the
1012 * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a
1013 * reschedule event.
1014 *
1015 * When we get rescheduled we reprogram the hrtick_timer outside of the
1016 * rq->lock.
1017 */
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001018
1019/*
1020 * Use hrtick when:
1021 * - enabled by features
1022 * - hrtimer is actually high res
1023 */
1024static inline int hrtick_enabled(struct rq *rq)
1025{
1026 if (!sched_feat(HRTICK))
1027 return 0;
Ingo Molnarba420592008-07-20 11:02:06 +02001028 if (!cpu_active(cpu_of(rq)))
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001029 return 0;
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001030 return hrtimer_is_hres_active(&rq->hrtick_timer);
1031}
1032
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001033static void hrtick_clear(struct rq *rq)
1034{
1035 if (hrtimer_active(&rq->hrtick_timer))
1036 hrtimer_cancel(&rq->hrtick_timer);
1037}
1038
1039/*
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001040 * High-resolution timer tick.
1041 * Runs from hardirq context with interrupts disabled.
1042 */
1043static enum hrtimer_restart hrtick(struct hrtimer *timer)
1044{
1045 struct rq *rq = container_of(timer, struct rq, hrtick_timer);
1046
1047 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
1048
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001049 raw_spin_lock(&rq->lock);
Peter Zijlstra3e51f332008-05-03 18:29:28 +02001050 update_rq_clock(rq);
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001051 rq->curr->sched_class->task_tick(rq, rq->curr, 1);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001052 raw_spin_unlock(&rq->lock);
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001053
1054 return HRTIMER_NORESTART;
1055}
1056
Rabin Vincent95e904c2008-05-11 05:55:33 +05301057#ifdef CONFIG_SMP
Peter Zijlstra31656512008-07-18 18:01:23 +02001058/*
1059 * called from hardirq (IPI) context
1060 */
1061static void __hrtick_start(void *arg)
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001062{
Peter Zijlstra31656512008-07-18 18:01:23 +02001063 struct rq *rq = arg;
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001064
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001065 raw_spin_lock(&rq->lock);
Peter Zijlstra31656512008-07-18 18:01:23 +02001066 hrtimer_restart(&rq->hrtick_timer);
1067 rq->hrtick_csd_pending = 0;
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001068 raw_spin_unlock(&rq->lock);
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001069}
1070
Peter Zijlstra31656512008-07-18 18:01:23 +02001071/*
1072 * Called to set the hrtick timer state.
1073 *
1074 * called with rq->lock held and irqs disabled
1075 */
1076static void hrtick_start(struct rq *rq, u64 delay)
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001077{
Peter Zijlstra31656512008-07-18 18:01:23 +02001078 struct hrtimer *timer = &rq->hrtick_timer;
1079 ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001080
Arjan van de Vencc584b22008-09-01 15:02:30 -07001081 hrtimer_set_expires(timer, time);
Peter Zijlstra31656512008-07-18 18:01:23 +02001082
1083 if (rq == this_rq()) {
1084 hrtimer_restart(timer);
1085 } else if (!rq->hrtick_csd_pending) {
Peter Zijlstra6e275632009-02-25 13:59:48 +01001086 __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0);
Peter Zijlstra31656512008-07-18 18:01:23 +02001087 rq->hrtick_csd_pending = 1;
1088 }
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001089}
1090
1091static int
1092hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
1093{
1094 int cpu = (int)(long)hcpu;
1095
1096 switch (action) {
1097 case CPU_UP_CANCELED:
1098 case CPU_UP_CANCELED_FROZEN:
1099 case CPU_DOWN_PREPARE:
1100 case CPU_DOWN_PREPARE_FROZEN:
1101 case CPU_DEAD:
1102 case CPU_DEAD_FROZEN:
Peter Zijlstra31656512008-07-18 18:01:23 +02001103 hrtick_clear(cpu_rq(cpu));
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001104 return NOTIFY_OK;
1105 }
1106
1107 return NOTIFY_DONE;
1108}
1109
Rakib Mullickfa748202008-09-22 14:55:45 -07001110static __init void init_hrtick(void)
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001111{
1112 hotcpu_notifier(hotplug_hrtick, 0);
1113}
Peter Zijlstra31656512008-07-18 18:01:23 +02001114#else
1115/*
1116 * Called to set the hrtick timer state.
1117 *
1118 * called with rq->lock held and irqs disabled
1119 */
1120static void hrtick_start(struct rq *rq, u64 delay)
1121{
Peter Zijlstra7f1e2ca2009-03-13 12:21:27 +01001122 __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
Arun R Bharadwaj5c333862009-04-16 12:14:37 +05301123 HRTIMER_MODE_REL_PINNED, 0);
Peter Zijlstra31656512008-07-18 18:01:23 +02001124}
1125
Andrew Morton006c75f2008-09-22 14:55:46 -07001126static inline void init_hrtick(void)
Peter Zijlstra31656512008-07-18 18:01:23 +02001127{
1128}
Rabin Vincent95e904c2008-05-11 05:55:33 +05301129#endif /* CONFIG_SMP */
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001130
1131static void init_rq_hrtick(struct rq *rq)
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001132{
Peter Zijlstra31656512008-07-18 18:01:23 +02001133#ifdef CONFIG_SMP
1134 rq->hrtick_csd_pending = 0;
1135
1136 rq->hrtick_csd.flags = 0;
1137 rq->hrtick_csd.func = __hrtick_start;
1138 rq->hrtick_csd.info = rq;
1139#endif
1140
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001141 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1142 rq->hrtick_timer.function = hrtick;
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001143}
Andrew Morton006c75f2008-09-22 14:55:46 -07001144#else /* CONFIG_SCHED_HRTICK */
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001145static inline void hrtick_clear(struct rq *rq)
1146{
1147}
1148
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001149static inline void init_rq_hrtick(struct rq *rq)
1150{
1151}
1152
Peter Zijlstrab328ca12008-04-29 10:02:46 +02001153static inline void init_hrtick(void)
1154{
1155}
Andrew Morton006c75f2008-09-22 14:55:46 -07001156#endif /* CONFIG_SCHED_HRTICK */
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01001157
Ingo Molnar1b9f19c2007-07-09 18:51:59 +02001158/*
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001159 * resched_task - mark a task 'to be rescheduled now'.
1160 *
1161 * On UP this means the setting of the need_resched flag, on SMP it
1162 * might also involve a cross-CPU call to trigger the scheduler on
1163 * the target CPU.
1164 */
1165#ifdef CONFIG_SMP
1166
1167#ifndef tsk_is_polling
1168#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
1169#endif
1170
Peter Zijlstra31656512008-07-18 18:01:23 +02001171static void resched_task(struct task_struct *p)
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001172{
1173 int cpu;
1174
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001175 assert_raw_spin_locked(&task_rq(p)->lock);
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001176
Lai Jiangshan5ed0cec2009-03-06 19:40:20 +08001177 if (test_tsk_need_resched(p))
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001178 return;
1179
Lai Jiangshan5ed0cec2009-03-06 19:40:20 +08001180 set_tsk_need_resched(p);
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001181
1182 cpu = task_cpu(p);
1183 if (cpu == smp_processor_id())
1184 return;
1185
1186 /* NEED_RESCHED must be visible before we test polling */
1187 smp_mb();
1188 if (!tsk_is_polling(p))
1189 smp_send_reschedule(cpu);
1190}
1191
1192static void resched_cpu(int cpu)
1193{
1194 struct rq *rq = cpu_rq(cpu);
1195 unsigned long flags;
1196
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001197 if (!raw_spin_trylock_irqsave(&rq->lock, flags))
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001198 return;
1199 resched_task(cpu_curr(cpu));
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001200 raw_spin_unlock_irqrestore(&rq->lock, flags);
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001201}
Thomas Gleixner06d83082008-03-22 09:20:24 +01001202
1203#ifdef CONFIG_NO_HZ
1204/*
1205 * When add_timer_on() enqueues a timer into the timer wheel of an
1206 * idle CPU then this timer might expire before the next timer event
1207 * which is scheduled to wake up that CPU. In case of a completely
1208 * idle system the next event might even be infinite time into the
1209 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
1210 * leaves the inner idle loop so the newly added timer is taken into
1211 * account when the CPU goes back to idle and evaluates the timer
1212 * wheel for the next timer event.
1213 */
1214void wake_up_idle_cpu(int cpu)
1215{
1216 struct rq *rq = cpu_rq(cpu);
1217
1218 if (cpu == smp_processor_id())
1219 return;
1220
1221 /*
1222 * This is safe, as this function is called with the timer
1223 * wheel base lock of (cpu) held. When the CPU is on the way
1224 * to idle and has not yet set rq->curr to idle then it will
1225 * be serialized on the timer wheel base lock and take the new
1226 * timer into account automatically.
1227 */
1228 if (rq->curr != rq->idle)
1229 return;
1230
1231 /*
1232 * We can set TIF_RESCHED on the idle task of the other CPU
1233 * lockless. The worst case is that the other CPU runs the
1234 * idle task through an additional NOOP schedule()
1235 */
Lai Jiangshan5ed0cec2009-03-06 19:40:20 +08001236 set_tsk_need_resched(rq->idle);
Thomas Gleixner06d83082008-03-22 09:20:24 +01001237
1238 /* NEED_RESCHED must be visible before we test polling */
1239 smp_mb();
1240 if (!tsk_is_polling(rq->idle))
1241 smp_send_reschedule(cpu);
1242}
Mike Galbraith39c0cbe2010-03-11 17:17:13 +01001243
1244int nohz_ratelimit(int cpu)
1245{
1246 struct rq *rq = cpu_rq(cpu);
1247 u64 diff = rq->clock - rq->nohz_stamp;
1248
1249 rq->nohz_stamp = rq->clock;
1250
1251 return diff < (NSEC_PER_SEC / HZ) >> 1;
1252}
1253
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02001254#endif /* CONFIG_NO_HZ */
Thomas Gleixner06d83082008-03-22 09:20:24 +01001255
Peter Zijlstrae9e92502009-09-01 10:34:37 +02001256static u64 sched_avg_period(void)
1257{
1258 return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2;
1259}
1260
1261static void sched_avg_update(struct rq *rq)
1262{
1263 s64 period = sched_avg_period();
1264
1265 while ((s64)(rq->clock - rq->age_stamp) > period) {
1266 rq->age_stamp += period;
1267 rq->rt_avg /= 2;
1268 }
1269}
1270
1271static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
1272{
1273 rq->rt_avg += rt_delta;
1274 sched_avg_update(rq);
1275}
1276
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02001277#else /* !CONFIG_SMP */
Peter Zijlstra31656512008-07-18 18:01:23 +02001278static void resched_task(struct task_struct *p)
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001279{
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001280 assert_raw_spin_locked(&task_rq(p)->lock);
Peter Zijlstra31656512008-07-18 18:01:23 +02001281 set_tsk_need_resched(p);
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001282}
Peter Zijlstrae9e92502009-09-01 10:34:37 +02001283
1284static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
1285{
1286}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02001287#endif /* CONFIG_SMP */
Ingo Molnarc24d20d2007-07-09 18:51:59 +02001288
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001289#if BITS_PER_LONG == 32
1290# define WMULT_CONST (~0UL)
1291#else
1292# define WMULT_CONST (1UL << 32)
1293#endif
1294
1295#define WMULT_SHIFT 32
1296
Ingo Molnar194081e2007-08-09 11:16:51 +02001297/*
1298 * Shift right and round:
1299 */
Ingo Molnarcf2ab462007-09-05 14:32:49 +02001300#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
Ingo Molnar194081e2007-08-09 11:16:51 +02001301
Peter Zijlstraa7be37a2008-06-27 13:41:11 +02001302/*
1303 * delta *= weight / lw
1304 */
Ingo Molnarcb1c4fc2007-08-02 17:41:40 +02001305static unsigned long
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001306calc_delta_mine(unsigned long delta_exec, unsigned long weight,
1307 struct load_weight *lw)
1308{
1309 u64 tmp;
1310
Lai Jiangshan7a232e02008-06-12 16:43:07 +08001311 if (!lw->inv_weight) {
1312 if (BITS_PER_LONG > 32 && unlikely(lw->weight >= WMULT_CONST))
1313 lw->inv_weight = 1;
1314 else
1315 lw->inv_weight = 1 + (WMULT_CONST-lw->weight/2)
1316 / (lw->weight+1);
1317 }
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001318
1319 tmp = (u64)delta_exec * weight;
1320 /*
1321 * Check whether we'd overflow the 64-bit multiplication:
1322 */
Ingo Molnar194081e2007-08-09 11:16:51 +02001323 if (unlikely(tmp > WMULT_CONST))
Ingo Molnarcf2ab462007-09-05 14:32:49 +02001324 tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight,
Ingo Molnar194081e2007-08-09 11:16:51 +02001325 WMULT_SHIFT/2);
1326 else
Ingo Molnarcf2ab462007-09-05 14:32:49 +02001327 tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT);
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001328
Ingo Molnarecf691d2007-08-02 17:41:40 +02001329 return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001330}
1331
Ingo Molnar10919852007-10-15 17:00:04 +02001332static inline void update_load_add(struct load_weight *lw, unsigned long inc)
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001333{
1334 lw->weight += inc;
Ingo Molnare89996a2008-03-14 23:48:28 +01001335 lw->inv_weight = 0;
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001336}
1337
Ingo Molnar10919852007-10-15 17:00:04 +02001338static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001339{
1340 lw->weight -= dec;
Ingo Molnare89996a2008-03-14 23:48:28 +01001341 lw->inv_weight = 0;
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001342}
1343
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344/*
Peter Williams2dd73a42006-06-27 02:54:34 -07001345 * To aid in avoiding the subversion of "niceness" due to uneven distribution
1346 * of tasks with abnormal "nice" values across CPUs the contribution that
1347 * each task makes to its run queue's load is weighted according to its
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01001348 * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a
Peter Williams2dd73a42006-06-27 02:54:34 -07001349 * scaled version of the new time slice allocation that they receive on time
1350 * slice expiry etc.
1351 */
1352
Peter Zijlstracce7ade2009-01-15 14:53:37 +01001353#define WEIGHT_IDLEPRIO 3
1354#define WMULT_IDLEPRIO 1431655765
Ingo Molnardd41f592007-07-09 18:51:59 +02001355
1356/*
1357 * Nice levels are multiplicative, with a gentle 10% change for every
1358 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to
1359 * nice 1, it will get ~10% less CPU time than another CPU-bound task
1360 * that remained on nice 0.
1361 *
1362 * The "10% effect" is relative and cumulative: from _any_ nice level,
1363 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level
Ingo Molnarf9153ee2007-07-16 09:46:30 +02001364 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
1365 * If a task goes up by ~10% and another task goes down by ~10% then
1366 * the relative distance between them is ~25%.)
Ingo Molnardd41f592007-07-09 18:51:59 +02001367 */
1368static const int prio_to_weight[40] = {
Ingo Molnar254753d2007-08-09 11:16:51 +02001369 /* -20 */ 88761, 71755, 56483, 46273, 36291,
1370 /* -15 */ 29154, 23254, 18705, 14949, 11916,
1371 /* -10 */ 9548, 7620, 6100, 4904, 3906,
1372 /* -5 */ 3121, 2501, 1991, 1586, 1277,
1373 /* 0 */ 1024, 820, 655, 526, 423,
1374 /* 5 */ 335, 272, 215, 172, 137,
1375 /* 10 */ 110, 87, 70, 56, 45,
1376 /* 15 */ 36, 29, 23, 18, 15,
Ingo Molnardd41f592007-07-09 18:51:59 +02001377};
1378
Ingo Molnar5714d2d2007-07-16 09:46:31 +02001379/*
1380 * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated.
1381 *
1382 * In cases where the weight does not change often, we can use the
1383 * precalculated inverse to speed up arithmetics by turning divisions
1384 * into multiplications:
1385 */
Ingo Molnardd41f592007-07-09 18:51:59 +02001386static const u32 prio_to_wmult[40] = {
Ingo Molnar254753d2007-08-09 11:16:51 +02001387 /* -20 */ 48388, 59856, 76040, 92818, 118348,
1388 /* -15 */ 147320, 184698, 229616, 287308, 360437,
1389 /* -10 */ 449829, 563644, 704093, 875809, 1099582,
1390 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
1391 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
1392 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
1393 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
1394 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
Ingo Molnardd41f592007-07-09 18:51:59 +02001395};
Peter Williams2dd73a42006-06-27 02:54:34 -07001396
Bharata B Raoef12fef2009-03-31 10:02:22 +05301397/* Time spent by the tasks of the cpu accounting group executing in ... */
1398enum cpuacct_stat_index {
1399 CPUACCT_STAT_USER, /* ... user mode */
1400 CPUACCT_STAT_SYSTEM, /* ... kernel mode */
1401
1402 CPUACCT_STAT_NSTATS,
1403};
1404
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01001405#ifdef CONFIG_CGROUP_CPUACCT
1406static void cpuacct_charge(struct task_struct *tsk, u64 cputime);
Bharata B Raoef12fef2009-03-31 10:02:22 +05301407static void cpuacct_update_stats(struct task_struct *tsk,
1408 enum cpuacct_stat_index idx, cputime_t val);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01001409#else
1410static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {}
Bharata B Raoef12fef2009-03-31 10:02:22 +05301411static inline void cpuacct_update_stats(struct task_struct *tsk,
1412 enum cpuacct_stat_index idx, cputime_t val) {}
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01001413#endif
1414
Peter Zijlstra18d95a22008-04-19 19:45:00 +02001415static inline void inc_cpu_load(struct rq *rq, unsigned long load)
1416{
1417 update_load_add(&rq->load, load);
1418}
1419
1420static inline void dec_cpu_load(struct rq *rq, unsigned long load)
1421{
1422 update_load_sub(&rq->load, load);
1423}
1424
Ingo Molnar7940ca32008-08-19 13:40:47 +02001425#if (defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)) || defined(CONFIG_RT_GROUP_SCHED)
Peter Zijlstraeb755802008-08-19 12:33:05 +02001426typedef int (*tg_visitor)(struct task_group *, void *);
1427
1428/*
1429 * Iterate the full tree, calling @down when first entering a node and @up when
1430 * leaving it for the final time.
1431 */
1432static int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
1433{
1434 struct task_group *parent, *child;
1435 int ret;
1436
1437 rcu_read_lock();
1438 parent = &root_task_group;
1439down:
1440 ret = (*down)(parent, data);
1441 if (ret)
1442 goto out_unlock;
1443 list_for_each_entry_rcu(child, &parent->children, siblings) {
1444 parent = child;
1445 goto down;
1446
1447up:
1448 continue;
1449 }
1450 ret = (*up)(parent, data);
1451 if (ret)
1452 goto out_unlock;
1453
1454 child = parent;
1455 parent = parent->parent;
1456 if (parent)
1457 goto up;
1458out_unlock:
1459 rcu_read_unlock();
1460
1461 return ret;
1462}
1463
1464static int tg_nop(struct task_group *tg, void *data)
1465{
1466 return 0;
1467}
1468#endif
1469
Gregory Haskinse7693a32008-01-25 21:08:09 +01001470#ifdef CONFIG_SMP
Peter Zijlstraf5f08f32009-09-10 13:35:28 +02001471/* Used instead of source_load when we know the type == 0 */
1472static unsigned long weighted_cpuload(const int cpu)
1473{
1474 return cpu_rq(cpu)->load.weight;
1475}
1476
1477/*
1478 * Return a low guess at the load of a migration-source cpu weighted
1479 * according to the scheduling class and "nice" value.
1480 *
1481 * We want to under-estimate the load of migration sources, to
1482 * balance conservatively.
1483 */
1484static unsigned long source_load(int cpu, int type)
1485{
1486 struct rq *rq = cpu_rq(cpu);
1487 unsigned long total = weighted_cpuload(cpu);
1488
1489 if (type == 0 || !sched_feat(LB_BIAS))
1490 return total;
1491
1492 return min(rq->cpu_load[type-1], total);
1493}
1494
1495/*
1496 * Return a high guess at the load of a migration-target cpu weighted
1497 * according to the scheduling class and "nice" value.
1498 */
1499static unsigned long target_load(int cpu, int type)
1500{
1501 struct rq *rq = cpu_rq(cpu);
1502 unsigned long total = weighted_cpuload(cpu);
1503
1504 if (type == 0 || !sched_feat(LB_BIAS))
1505 return total;
1506
1507 return max(rq->cpu_load[type-1], total);
1508}
1509
Peter Zijlstraae154be2009-09-10 14:40:57 +02001510static struct sched_group *group_of(int cpu)
1511{
Paul E. McKenneyd11c5632010-02-22 17:04:50 -08001512 struct sched_domain *sd = rcu_dereference_sched(cpu_rq(cpu)->sd);
Peter Zijlstraae154be2009-09-10 14:40:57 +02001513
1514 if (!sd)
1515 return NULL;
1516
1517 return sd->groups;
1518}
1519
1520static unsigned long power_of(int cpu)
1521{
1522 struct sched_group *group = group_of(cpu);
1523
1524 if (!group)
1525 return SCHED_LOAD_SCALE;
1526
1527 return group->cpu_power;
1528}
1529
Gregory Haskinse7693a32008-01-25 21:08:09 +01001530static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001531
Peter Zijlstraa8a51d52008-06-27 13:41:26 +02001532static unsigned long cpu_avg_load_per_task(int cpu)
1533{
1534 struct rq *rq = cpu_rq(cpu);
Ingo Molnaraf6d5962008-11-29 20:45:15 +01001535 unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
Peter Zijlstraa8a51d52008-06-27 13:41:26 +02001536
Steven Rostedt4cd42622008-11-26 21:04:24 -05001537 if (nr_running)
1538 rq->avg_load_per_task = rq->load.weight / nr_running;
Balbir Singha2d47772008-11-12 16:19:00 +05301539 else
1540 rq->avg_load_per_task = 0;
Peter Zijlstraa8a51d52008-06-27 13:41:26 +02001541
1542 return rq->avg_load_per_task;
1543}
1544
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001545#ifdef CONFIG_FAIR_GROUP_SCHED
1546
Tejun Heo43cf38e2010-02-02 14:38:57 +09001547static __read_mostly unsigned long __percpu *update_shares_data;
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001548
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001549static void __set_se_shares(struct sched_entity *se, unsigned long shares);
1550
1551/*
1552 * Calculate and set the cpu's group shares.
1553 */
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001554static void update_group_shares_cpu(struct task_group *tg, int cpu,
1555 unsigned long sd_shares,
1556 unsigned long sd_rq_weight,
Jiri Kosina4a6cc4b2009-10-29 00:26:00 +09001557 unsigned long *usd_rq_weight)
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001558{
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001559 unsigned long shares, rq_weight;
Peter Zijlstraa5004272009-07-27 14:04:49 +02001560 int boost = 0;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001561
Jiri Kosina4a6cc4b2009-10-29 00:26:00 +09001562 rq_weight = usd_rq_weight[cpu];
Peter Zijlstraa5004272009-07-27 14:04:49 +02001563 if (!rq_weight) {
1564 boost = 1;
1565 rq_weight = NICE_0_LOAD;
1566 }
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001567
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001568 /*
Peter Zijlstraa8af7242009-08-21 13:58:54 +02001569 * \Sum_j shares_j * rq_weight_i
1570 * shares_i = -----------------------------
1571 * \Sum_j rq_weight_j
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001572 */
Ken Chenec4e0e22008-11-18 22:41:57 -08001573 shares = (sd_shares * rq_weight) / sd_rq_weight;
Peter Zijlstraffda12a2008-10-17 19:27:02 +02001574 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001575
Peter Zijlstraffda12a2008-10-17 19:27:02 +02001576 if (abs(shares - tg->se[cpu]->load.weight) >
1577 sysctl_sched_shares_thresh) {
1578 struct rq *rq = cpu_rq(cpu);
1579 unsigned long flags;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001580
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001581 raw_spin_lock_irqsave(&rq->lock, flags);
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001582 tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight;
Peter Zijlstraa5004272009-07-27 14:04:49 +02001583 tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
Peter Zijlstraffda12a2008-10-17 19:27:02 +02001584 __set_se_shares(tg->se[cpu], shares);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001585 raw_spin_unlock_irqrestore(&rq->lock, flags);
Peter Zijlstraffda12a2008-10-17 19:27:02 +02001586 }
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001587}
1588
1589/*
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001590 * Re-compute the task group their per cpu shares over the given domain.
1591 * This needs to be done in a bottom-up fashion because the rq weight of a
1592 * parent group depends on the shares of its child groups.
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001593 */
Peter Zijlstraeb755802008-08-19 12:33:05 +02001594static int tg_shares_up(struct task_group *tg, void *data)
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001595{
Peter Zijlstracd8ad402009-12-03 18:00:07 +01001596 unsigned long weight, rq_weight = 0, sum_weight = 0, shares = 0;
Jiri Kosina4a6cc4b2009-10-29 00:26:00 +09001597 unsigned long *usd_rq_weight;
Peter Zijlstraeb755802008-08-19 12:33:05 +02001598 struct sched_domain *sd = data;
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001599 unsigned long flags;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001600 int i;
1601
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001602 if (!tg->se[0])
1603 return 0;
1604
1605 local_irq_save(flags);
Jiri Kosina4a6cc4b2009-10-29 00:26:00 +09001606 usd_rq_weight = per_cpu_ptr(update_shares_data, smp_processor_id());
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001607
Rusty Russell758b2cd2008-11-25 02:35:04 +10301608 for_each_cpu(i, sched_domain_span(sd)) {
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001609 weight = tg->cfs_rq[i]->load.weight;
Jiri Kosina4a6cc4b2009-10-29 00:26:00 +09001610 usd_rq_weight[i] = weight;
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001611
Peter Zijlstracd8ad402009-12-03 18:00:07 +01001612 rq_weight += weight;
Ken Chenec4e0e22008-11-18 22:41:57 -08001613 /*
1614 * If there are currently no tasks on the cpu pretend there
1615 * is one of average load so that when a new task gets to
1616 * run here it will not get delayed by group starvation.
1617 */
Ken Chenec4e0e22008-11-18 22:41:57 -08001618 if (!weight)
1619 weight = NICE_0_LOAD;
1620
Peter Zijlstracd8ad402009-12-03 18:00:07 +01001621 sum_weight += weight;
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001622 shares += tg->cfs_rq[i]->shares;
1623 }
1624
Peter Zijlstracd8ad402009-12-03 18:00:07 +01001625 if (!rq_weight)
1626 rq_weight = sum_weight;
1627
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001628 if ((!shares && rq_weight) || shares > tg->shares)
1629 shares = tg->shares;
1630
1631 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1632 shares = tg->shares;
1633
Rusty Russell758b2cd2008-11-25 02:35:04 +10301634 for_each_cpu(i, sched_domain_span(sd))
Jiri Kosina4a6cc4b2009-10-29 00:26:00 +09001635 update_group_shares_cpu(tg, i, shares, rq_weight, usd_rq_weight);
Peter Zijlstra34d76c42009-08-27 13:08:56 +02001636
1637 local_irq_restore(flags);
Peter Zijlstraeb755802008-08-19 12:33:05 +02001638
1639 return 0;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001640}
1641
1642/*
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001643 * Compute the cpu's hierarchical load factor for each task group.
1644 * This needs to be done in a top-down fashion because the load of a child
1645 * group is a fraction of its parents load.
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001646 */
Peter Zijlstraeb755802008-08-19 12:33:05 +02001647static int tg_load_down(struct task_group *tg, void *data)
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001648{
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001649 unsigned long load;
Peter Zijlstraeb755802008-08-19 12:33:05 +02001650 long cpu = (long)data;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001651
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001652 if (!tg->parent) {
1653 load = cpu_rq(cpu)->load.weight;
1654 } else {
1655 load = tg->parent->cfs_rq[cpu]->h_load;
1656 load *= tg->cfs_rq[cpu]->shares;
1657 load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
1658 }
1659
1660 tg->cfs_rq[cpu]->h_load = load;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001661
Peter Zijlstraeb755802008-08-19 12:33:05 +02001662 return 0;
Peter Zijlstra4d8d5952008-06-27 13:41:19 +02001663}
1664
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001665static void update_shares(struct sched_domain *sd)
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001666{
Peter Zijlstrae7097152009-06-03 15:41:20 +02001667 s64 elapsed;
1668 u64 now;
1669
1670 if (root_task_group_empty())
1671 return;
1672
1673 now = cpu_clock(raw_smp_processor_id());
1674 elapsed = now - sd->last_update;
Peter Zijlstra2398f2c2008-06-27 13:41:35 +02001675
1676 if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
1677 sd->last_update = now;
Peter Zijlstraeb755802008-08-19 12:33:05 +02001678 walk_tg_tree(tg_nop, tg_shares_up, sd);
Peter Zijlstra2398f2c2008-06-27 13:41:35 +02001679 }
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001680}
1681
Peter Zijlstraeb755802008-08-19 12:33:05 +02001682static void update_h_load(long cpu)
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001683{
Peter Zijlstrae7097152009-06-03 15:41:20 +02001684 if (root_task_group_empty())
1685 return;
1686
Peter Zijlstraeb755802008-08-19 12:33:05 +02001687 walk_tg_tree(tg_load_down, tg_nop, (void *)cpu);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001688}
1689
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001690#else
1691
Peter Zijlstrac8cba852008-06-27 13:41:23 +02001692static inline void update_shares(struct sched_domain *sd)
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001693{
1694}
1695
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001696#endif
1697
Gregory Haskins8f45e2b2008-12-29 09:39:51 -05001698#ifdef CONFIG_PREEMPT
1699
Peter Zijlstrab78bb862009-09-15 14:23:18 +02001700static void double_rq_lock(struct rq *rq1, struct rq *rq2);
1701
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001702/*
Gregory Haskins8f45e2b2008-12-29 09:39:51 -05001703 * fair double_lock_balance: Safely acquires both rq->locks in a fair
1704 * way at the expense of forcing extra atomic operations in all
1705 * invocations. This assures that the double_lock is acquired using the
1706 * same underlying policy as the spinlock_t on this architecture, which
1707 * reduces latency compared to the unfair variant below. However, it
1708 * also adds more overhead and therefore may reduce throughput.
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001709 */
Gregory Haskins8f45e2b2008-12-29 09:39:51 -05001710static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
1711 __releases(this_rq->lock)
1712 __acquires(busiest->lock)
1713 __acquires(this_rq->lock)
1714{
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001715 raw_spin_unlock(&this_rq->lock);
Gregory Haskins8f45e2b2008-12-29 09:39:51 -05001716 double_rq_lock(this_rq, busiest);
1717
1718 return 1;
1719}
1720
1721#else
1722/*
1723 * Unfair double_lock_balance: Optimizes throughput at the expense of
1724 * latency by eliminating extra atomic operations when the locks are
1725 * already in proper order on entry. This favors lower cpu-ids and will
1726 * grant the double lock to lower cpus over higher ids under contention,
1727 * regardless of entry order into the function.
1728 */
1729static int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001730 __releases(this_rq->lock)
1731 __acquires(busiest->lock)
1732 __acquires(this_rq->lock)
1733{
1734 int ret = 0;
1735
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001736 if (unlikely(!raw_spin_trylock(&busiest->lock))) {
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001737 if (busiest < this_rq) {
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001738 raw_spin_unlock(&this_rq->lock);
1739 raw_spin_lock(&busiest->lock);
1740 raw_spin_lock_nested(&this_rq->lock,
1741 SINGLE_DEPTH_NESTING);
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001742 ret = 1;
1743 } else
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001744 raw_spin_lock_nested(&busiest->lock,
1745 SINGLE_DEPTH_NESTING);
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001746 }
1747 return ret;
1748}
1749
Gregory Haskins8f45e2b2008-12-29 09:39:51 -05001750#endif /* CONFIG_PREEMPT */
1751
1752/*
1753 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
1754 */
1755static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
1756{
1757 if (unlikely(!irqs_disabled())) {
1758 /* printk() doesn't work good under rq->lock */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001759 raw_spin_unlock(&this_rq->lock);
Gregory Haskins8f45e2b2008-12-29 09:39:51 -05001760 BUG_ON(1);
1761 }
1762
1763 return _double_lock_balance(this_rq, busiest);
1764}
1765
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001766static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
1767 __releases(busiest->lock)
1768{
Thomas Gleixner05fa7852009-11-17 14:28:38 +01001769 raw_spin_unlock(&busiest->lock);
Alexey Dobriyan70574a92008-11-28 22:08:00 +03001770 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
1771}
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001772
1773/*
1774 * double_rq_lock - safely lock two runqueues
1775 *
1776 * Note this does not disable interrupts like task_rq_lock,
1777 * you need to do so manually before calling.
1778 */
1779static void double_rq_lock(struct rq *rq1, struct rq *rq2)
1780 __acquires(rq1->lock)
1781 __acquires(rq2->lock)
1782{
1783 BUG_ON(!irqs_disabled());
1784 if (rq1 == rq2) {
1785 raw_spin_lock(&rq1->lock);
1786 __acquire(rq2->lock); /* Fake it out ;) */
1787 } else {
1788 if (rq1 < rq2) {
1789 raw_spin_lock(&rq1->lock);
1790 raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
1791 } else {
1792 raw_spin_lock(&rq2->lock);
1793 raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
1794 }
1795 }
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001796}
1797
1798/*
1799 * double_rq_unlock - safely unlock two runqueues
1800 *
1801 * Note this does not restore interrupts like task_rq_unlock,
1802 * you need to do so manually after calling.
1803 */
1804static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
1805 __releases(rq1->lock)
1806 __releases(rq2->lock)
1807{
1808 raw_spin_unlock(&rq1->lock);
1809 if (rq1 != rq2)
1810 raw_spin_unlock(&rq2->lock);
1811 else
1812 __release(rq2->lock);
1813}
1814
Peter Zijlstra18d95a22008-04-19 19:45:00 +02001815#endif
Peter Zijlstra18d95a22008-04-19 19:45:00 +02001816
1817#ifdef CONFIG_FAIR_GROUP_SCHED
1818static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1819{
Vegard Nossum30432092008-06-27 21:35:50 +02001820#ifdef CONFIG_SMP
Ingo Molnar34e83e82008-06-27 15:42:36 +02001821 cfs_rq->shares = shares;
1822#endif
Peter Zijlstra18d95a22008-04-19 19:45:00 +02001823}
1824#endif
1825
Peter Zijlstra74f51872010-04-22 21:50:19 +02001826static void calc_load_account_idle(struct rq *this_rq);
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01001827static void update_sysctl(void);
Christian Ehrhardtacb4a842009-11-30 12:16:48 +01001828static int get_update_sysctl_factor(void);
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02001829
Peter Zijlstracd29fe62009-11-27 17:32:46 +01001830static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
1831{
1832 set_task_rq(p, cpu);
1833#ifdef CONFIG_SMP
1834 /*
1835 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
1836 * successfuly executed on another CPU. We must ensure that updates of
1837 * per-task data have been completed by this moment.
1838 */
1839 smp_wmb();
1840 task_thread_info(p)->cpu = cpu;
1841#endif
1842}
Gregory Haskinse7693a32008-01-25 21:08:09 +01001843
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001844static const struct sched_class rt_sched_class;
Ingo Molnardd41f592007-07-09 18:51:59 +02001845
1846#define sched_class_highest (&rt_sched_class)
Gregory Haskins1f11eb62008-06-04 15:04:05 -04001847#define for_each_class(class) \
1848 for (class = sched_class_highest; class; class = class->next)
Ingo Molnardd41f592007-07-09 18:51:59 +02001849
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001850#include "sched_stats.h"
1851
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001852static void inc_nr_running(struct rq *rq)
Ingo Molnar6363ca52008-05-29 11:28:57 +02001853{
1854 rq->nr_running++;
Ingo Molnar6363ca52008-05-29 11:28:57 +02001855}
1856
Peter Zijlstrac09595f2008-06-27 13:41:14 +02001857static void dec_nr_running(struct rq *rq)
Ingo Molnar9c217242007-08-02 17:41:40 +02001858{
1859 rq->nr_running--;
Ingo Molnar9c217242007-08-02 17:41:40 +02001860}
1861
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001862static void set_load_weight(struct task_struct *p)
1863{
1864 if (task_has_rt_policy(p)) {
Ingo Molnardd41f592007-07-09 18:51:59 +02001865 p->se.load.weight = prio_to_weight[0] * 2;
1866 p->se.load.inv_weight = prio_to_wmult[0] >> 1;
1867 return;
1868 }
1869
1870 /*
1871 * SCHED_IDLE tasks get minimal weight:
1872 */
1873 if (p->policy == SCHED_IDLE) {
1874 p->se.load.weight = WEIGHT_IDLEPRIO;
1875 p->se.load.inv_weight = WMULT_IDLEPRIO;
1876 return;
1877 }
1878
1879 p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO];
1880 p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
Ingo Molnar45bf76d2007-07-09 18:51:59 +02001881}
1882
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001883static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
Gregory Haskins2087a1a2008-06-27 14:30:00 -06001884{
Mike Galbraitha64692a2010-03-11 17:16:20 +01001885 update_rq_clock(rq);
Ingo Molnar71f8bd42007-07-09 18:51:59 +02001886 sched_info_queued(p);
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001887 p->sched_class->enqueue_task(rq, p, flags);
Ingo Molnardd41f592007-07-09 18:51:59 +02001888 p->se.on_rq = 1;
1889}
1890
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001891static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
Ingo Molnardd41f592007-07-09 18:51:59 +02001892{
Mike Galbraitha64692a2010-03-11 17:16:20 +01001893 update_rq_clock(rq);
Ankita Garg46ac22b2008-07-01 14:30:06 +05301894 sched_info_dequeued(p);
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001895 p->sched_class->dequeue_task(rq, p, flags);
Ingo Molnardd41f592007-07-09 18:51:59 +02001896 p->se.on_rq = 0;
Ingo Molnar71f8bd42007-07-09 18:51:59 +02001897}
1898
1899/*
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001900 * activate_task - move a task to the runqueue.
1901 */
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001902static void activate_task(struct rq *rq, struct task_struct *p, int flags)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001903{
1904 if (task_contributes_to_load(p))
1905 rq->nr_uninterruptible--;
1906
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001907 enqueue_task(rq, p, flags);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001908 inc_nr_running(rq);
1909}
1910
1911/*
1912 * deactivate_task - remove a task from the runqueue.
1913 */
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001914static void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001915{
1916 if (task_contributes_to_load(p))
1917 rq->nr_uninterruptible++;
1918
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01001919 dequeue_task(rq, p, flags);
Peter Zijlstra1e3c88b2009-12-17 17:00:43 +01001920 dec_nr_running(rq);
1921}
1922
1923#include "sched_idletask.c"
1924#include "sched_fair.c"
1925#include "sched_rt.c"
1926#ifdef CONFIG_SCHED_DEBUG
1927# include "sched_debug.c"
1928#endif
1929
1930/*
Ingo Molnardd41f592007-07-09 18:51:59 +02001931 * __normal_prio - return the priority that is based on the static prio
Ingo Molnar71f8bd42007-07-09 18:51:59 +02001932 */
Ingo Molnar14531182007-07-09 18:51:59 +02001933static inline int __normal_prio(struct task_struct *p)
1934{
Ingo Molnardd41f592007-07-09 18:51:59 +02001935 return p->static_prio;
Ingo Molnar14531182007-07-09 18:51:59 +02001936}
1937
1938/*
Ingo Molnarb29739f2006-06-27 02:54:51 -07001939 * Calculate the expected normal priority: i.e. priority
1940 * without taking RT-inheritance into account. Might be
1941 * boosted by interactivity modifiers. Changes upon fork,
1942 * setprio syscalls, and whenever the interactivity
1943 * estimator recalculates.
1944 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07001945static inline int normal_prio(struct task_struct *p)
Ingo Molnarb29739f2006-06-27 02:54:51 -07001946{
1947 int prio;
1948
Ingo Molnare05606d2007-07-09 18:51:59 +02001949 if (task_has_rt_policy(p))
Ingo Molnarb29739f2006-06-27 02:54:51 -07001950 prio = MAX_RT_PRIO-1 - p->rt_priority;
1951 else
1952 prio = __normal_prio(p);
1953 return prio;
1954}
1955
1956/*
1957 * Calculate the current priority, i.e. the priority
1958 * taken into account by the scheduler. This value might
1959 * be boosted by RT tasks, or might be boosted by
1960 * interactivity modifiers. Will be RT if the task got
1961 * RT-boosted. If not then it returns p->normal_prio.
1962 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07001963static int effective_prio(struct task_struct *p)
Ingo Molnarb29739f2006-06-27 02:54:51 -07001964{
1965 p->normal_prio = normal_prio(p);
1966 /*
1967 * If we are RT tasks or we were boosted to RT priority,
1968 * keep the priority unchanged. Otherwise, update priority
1969 * to the normal priority:
1970 */
1971 if (!rt_prio(p->prio))
1972 return p->normal_prio;
1973 return p->prio;
1974}
1975
Linus Torvalds1da177e2005-04-16 15:20:36 -07001976/**
1977 * task_curr - is this task currently executing on a CPU?
1978 * @p: the task in question.
1979 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07001980inline int task_curr(const struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981{
1982 return cpu_curr(task_cpu(p)) == p;
1983}
1984
Steven Rostedtcb469842008-01-25 21:08:22 +01001985static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1986 const struct sched_class *prev_class,
1987 int oldprio, int running)
1988{
1989 if (prev_class != p->sched_class) {
1990 if (prev_class->switched_from)
1991 prev_class->switched_from(rq, p, running);
1992 p->sched_class->switched_to(rq, p, running);
1993 } else
1994 p->sched_class->prio_changed(rq, p, oldprio, running);
1995}
1996
Linus Torvalds1da177e2005-04-16 15:20:36 -07001997#ifdef CONFIG_SMP
Ingo Molnarcc367732007-10-15 17:00:18 +02001998/*
1999 * Is this task likely cache-hot:
2000 */
Gregory Haskinse7693a32008-01-25 21:08:09 +01002001static int
Ingo Molnarcc367732007-10-15 17:00:18 +02002002task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
2003{
2004 s64 delta;
2005
Peter Zijlstrae6c8fba2009-12-16 18:04:33 +01002006 if (p->sched_class != &fair_sched_class)
2007 return 0;
2008
Ingo Molnarf540a602008-03-15 17:10:34 +01002009 /*
2010 * Buddy candidates are cache hot:
2011 */
Mike Galbraithf685cea2009-10-23 23:09:22 +02002012 if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running &&
Peter Zijlstra47932412008-11-04 21:25:09 +01002013 (&p->se == cfs_rq_of(&p->se)->next ||
2014 &p->se == cfs_rq_of(&p->se)->last))
Ingo Molnarf540a602008-03-15 17:10:34 +01002015 return 1;
2016
Ingo Molnar6bc16652007-10-15 17:00:18 +02002017 if (sysctl_sched_migration_cost == -1)
2018 return 1;
2019 if (sysctl_sched_migration_cost == 0)
2020 return 0;
2021
Ingo Molnarcc367732007-10-15 17:00:18 +02002022 delta = now - p->se.exec_start;
2023
2024 return delta < (s64)sysctl_sched_migration_cost;
2025}
2026
Ingo Molnardd41f592007-07-09 18:51:59 +02002027void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
Ingo Molnarc65cc872007-07-09 18:51:58 +02002028{
Peter Zijlstrae2912002009-12-16 18:04:36 +01002029#ifdef CONFIG_SCHED_DEBUG
2030 /*
2031 * We should never call set_task_cpu() on a blocked task,
2032 * ttwu() will sort out the placement.
2033 */
Peter Zijlstra077614e2009-12-17 13:16:31 +01002034 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
2035 !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
Peter Zijlstrae2912002009-12-16 18:04:36 +01002036#endif
2037
Mathieu Desnoyersde1d7282009-05-05 16:49:59 +08002038 trace_sched_migrate_task(p, new_cpu);
Peter Zijlstracbc34ed2008-12-10 08:08:22 +01002039
Peter Zijlstra0c697742009-12-22 15:43:19 +01002040 if (task_cpu(p) != new_cpu) {
2041 p->se.nr_migrations++;
2042 perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
2043 }
Ingo Molnardd41f592007-07-09 18:51:59 +02002044
2045 __set_task_cpu(p, new_cpu);
Ingo Molnarc65cc872007-07-09 18:51:58 +02002046}
2047
Tejun Heo969c7922010-05-06 18:49:21 +02002048struct migration_arg {
Ingo Molnar36c8b582006-07-03 00:25:41 -07002049 struct task_struct *task;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002050 int dest_cpu;
Ingo Molnar70b97a72006-07-03 00:25:42 -07002051};
Linus Torvalds1da177e2005-04-16 15:20:36 -07002052
Tejun Heo969c7922010-05-06 18:49:21 +02002053static int migration_cpu_stop(void *data);
2054
Linus Torvalds1da177e2005-04-16 15:20:36 -07002055/*
2056 * The task's runqueue lock must be held.
2057 * Returns true if you have to wait for migration thread.
2058 */
Tejun Heo969c7922010-05-06 18:49:21 +02002059static bool migrate_task(struct task_struct *p, int dest_cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002060{
Ingo Molnar70b97a72006-07-03 00:25:42 -07002061 struct rq *rq = task_rq(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002062
2063 /*
2064 * If the task is not on a runqueue (and not running), then
Peter Zijlstrae2912002009-12-16 18:04:36 +01002065 * the next wake-up will properly place the task.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002066 */
Tejun Heo969c7922010-05-06 18:49:21 +02002067 return p->se.on_rq || task_running(rq, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002068}
2069
2070/*
2071 * wait_task_inactive - wait for a thread to unschedule.
2072 *
Roland McGrath85ba2d82008-07-25 19:45:58 -07002073 * If @match_state is nonzero, it's the @p->state value just checked and
2074 * not expected to change. If it changes, i.e. @p might have woken up,
2075 * then return zero. When we succeed in waiting for @p to be off its CPU,
2076 * we return a positive number (its total switch count). If a second call
2077 * a short while later returns the same number, the caller can be sure that
2078 * @p has remained unscheduled the whole time.
2079 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080 * The caller must ensure that the task *will* unschedule sometime soon,
2081 * else this function might spin for a *long* time. This function can't
2082 * be called with interrupts off, or it may introduce deadlock with
2083 * smp_call_function() if an IPI is sent by the same process we are
2084 * waiting to become inactive.
2085 */
Roland McGrath85ba2d82008-07-25 19:45:58 -07002086unsigned long wait_task_inactive(struct task_struct *p, long match_state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002087{
2088 unsigned long flags;
Ingo Molnardd41f592007-07-09 18:51:59 +02002089 int running, on_rq;
Roland McGrath85ba2d82008-07-25 19:45:58 -07002090 unsigned long ncsw;
Ingo Molnar70b97a72006-07-03 00:25:42 -07002091 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002092
Andi Kleen3a5c3592007-10-15 17:00:14 +02002093 for (;;) {
2094 /*
2095 * We do the initial early heuristics without holding
2096 * any task-queue locks at all. We'll only try to get
2097 * the runqueue lock when things look like they will
2098 * work out!
2099 */
2100 rq = task_rq(p);
Linus Torvaldsfa490cf2007-06-18 09:34:40 -07002101
Andi Kleen3a5c3592007-10-15 17:00:14 +02002102 /*
2103 * If the task is actively running on another CPU
2104 * still, just relax and busy-wait without holding
2105 * any locks.
2106 *
2107 * NOTE! Since we don't hold any locks, it's not
2108 * even sure that "rq" stays as the right runqueue!
2109 * But we don't care, since "task_running()" will
2110 * return false if the runqueue has changed and p
2111 * is actually now running somewhere else!
2112 */
Roland McGrath85ba2d82008-07-25 19:45:58 -07002113 while (task_running(rq, p)) {
2114 if (match_state && unlikely(p->state != match_state))
2115 return 0;
Andi Kleen3a5c3592007-10-15 17:00:14 +02002116 cpu_relax();
Roland McGrath85ba2d82008-07-25 19:45:58 -07002117 }
Linus Torvaldsfa490cf2007-06-18 09:34:40 -07002118
Andi Kleen3a5c3592007-10-15 17:00:14 +02002119 /*
2120 * Ok, time to look more closely! We need the rq
2121 * lock now, to be *sure*. If we're wrong, we'll
2122 * just go back and repeat.
2123 */
2124 rq = task_rq_lock(p, &flags);
Peter Zijlstra27a9da62010-05-04 20:36:56 +02002125 trace_sched_wait_task(p);
Andi Kleen3a5c3592007-10-15 17:00:14 +02002126 running = task_running(rq, p);
2127 on_rq = p->se.on_rq;
Roland McGrath85ba2d82008-07-25 19:45:58 -07002128 ncsw = 0;
Oleg Nesterovf31e11d2008-08-20 16:54:44 -07002129 if (!match_state || p->state == match_state)
Oleg Nesterov93dcf552008-08-20 16:54:44 -07002130 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
Andi Kleen3a5c3592007-10-15 17:00:14 +02002131 task_rq_unlock(rq, &flags);
Linus Torvaldsfa490cf2007-06-18 09:34:40 -07002132
Andi Kleen3a5c3592007-10-15 17:00:14 +02002133 /*
Roland McGrath85ba2d82008-07-25 19:45:58 -07002134 * If it changed from the expected state, bail out now.
2135 */
2136 if (unlikely(!ncsw))
2137 break;
2138
2139 /*
Andi Kleen3a5c3592007-10-15 17:00:14 +02002140 * Was it really running after all now that we
2141 * checked with the proper locks actually held?
2142 *
2143 * Oops. Go back and try again..
2144 */
2145 if (unlikely(running)) {
2146 cpu_relax();
2147 continue;
2148 }
2149
2150 /*
2151 * It's not enough that it's not actively running,
2152 * it must be off the runqueue _entirely_, and not
2153 * preempted!
2154 *
Luis Henriques80dd99b2009-03-16 19:58:09 +00002155 * So if it was still runnable (but just not actively
Andi Kleen3a5c3592007-10-15 17:00:14 +02002156 * running right now), it's preempted, and we should
2157 * yield - it could be a while.
2158 */
2159 if (unlikely(on_rq)) {
2160 schedule_timeout_uninterruptible(1);
2161 continue;
2162 }
2163
2164 /*
2165 * Ahh, all good. It wasn't running, and it wasn't
2166 * runnable, which means that it will never become
2167 * running in the future either. We're all done!
2168 */
2169 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002170 }
Roland McGrath85ba2d82008-07-25 19:45:58 -07002171
2172 return ncsw;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002173}
2174
2175/***
2176 * kick_process - kick a running thread to enter/exit the kernel
2177 * @p: the to-be-kicked thread
2178 *
2179 * Cause a process which is running on another CPU to enter
2180 * kernel-mode, without any delay. (to get signals handled.)
2181 *
2182 * NOTE: this function doesnt have to take the runqueue lock,
2183 * because all it wants to ensure is that the remote task enters
2184 * the kernel. If the IPI races and the task has been migrated
2185 * to another CPU then no harm is done and the purpose has been
2186 * achieved as well.
2187 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07002188void kick_process(struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002189{
2190 int cpu;
2191
2192 preempt_disable();
2193 cpu = task_cpu(p);
2194 if ((cpu != smp_processor_id()) && task_curr(p))
2195 smp_send_reschedule(cpu);
2196 preempt_enable();
2197}
Rusty Russellb43e3522009-06-12 22:27:00 -06002198EXPORT_SYMBOL_GPL(kick_process);
Nick Piggin476d1392005-06-25 14:57:29 -07002199#endif /* CONFIG_SMP */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002200
Thomas Gleixner0793a612008-12-04 20:12:29 +01002201/**
2202 * task_oncpu_function_call - call a function on the cpu on which a task runs
2203 * @p: the task to evaluate
2204 * @func: the function to be called
2205 * @info: the function call argument
2206 *
2207 * Calls the function @func when the task is currently running. This might
2208 * be on the current CPU, which just calls the function directly
2209 */
2210void task_oncpu_function_call(struct task_struct *p,
2211 void (*func) (void *info), void *info)
2212{
2213 int cpu;
2214
2215 preempt_disable();
2216 cpu = task_cpu(p);
2217 if (task_curr(p))
2218 smp_call_function_single(cpu, func, info, 1);
2219 preempt_enable();
2220}
2221
Peter Zijlstra970b13b2009-11-25 13:31:39 +01002222#ifdef CONFIG_SMP
Oleg Nesterov30da6882010-03-15 10:10:19 +01002223/*
2224 * ->cpus_allowed is protected by either TASK_WAKING or rq->lock held.
2225 */
Peter Zijlstra5da9a0f2009-12-16 18:04:38 +01002226static int select_fallback_rq(int cpu, struct task_struct *p)
2227{
2228 int dest_cpu;
2229 const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));
2230
2231 /* Look for allowed, online CPU in same node. */
2232 for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
2233 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
2234 return dest_cpu;
2235
2236 /* Any allowed, online CPU? */
2237 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
2238 if (dest_cpu < nr_cpu_ids)
2239 return dest_cpu;
2240
2241 /* No more Mr. Nice Guy. */
Oleg Nesterov897f0b32010-03-15 10:10:03 +01002242 if (unlikely(dest_cpu >= nr_cpu_ids)) {
Oleg Nesterov9084bb82010-03-15 10:10:27 +01002243 dest_cpu = cpuset_cpus_allowed_fallback(p);
Peter Zijlstra5da9a0f2009-12-16 18:04:38 +01002244 /*
2245 * Don't tell them about moving exiting tasks or
2246 * kernel threads (both mm NULL), since they never
2247 * leave kernel.
2248 */
2249 if (p->mm && printk_ratelimit()) {
2250 printk(KERN_INFO "process %d (%s) no "
2251 "longer affine to cpu%d\n",
2252 task_pid_nr(p), p->comm, cpu);
2253 }
2254 }
2255
2256 return dest_cpu;
2257}
2258
Peter Zijlstrae2912002009-12-16 18:04:36 +01002259/*
Oleg Nesterov30da6882010-03-15 10:10:19 +01002260 * The caller (fork, wakeup) owns TASK_WAKING, ->cpus_allowed is stable.
Peter Zijlstrae2912002009-12-16 18:04:36 +01002261 */
Peter Zijlstra970b13b2009-11-25 13:31:39 +01002262static inline
Peter Zijlstra0017d732010-03-24 18:34:10 +01002263int select_task_rq(struct rq *rq, struct task_struct *p, int sd_flags, int wake_flags)
Peter Zijlstra970b13b2009-11-25 13:31:39 +01002264{
Peter Zijlstra0017d732010-03-24 18:34:10 +01002265 int cpu = p->sched_class->select_task_rq(rq, p, sd_flags, wake_flags);
Peter Zijlstrae2912002009-12-16 18:04:36 +01002266
2267 /*
2268 * In order not to call set_task_cpu() on a blocking task we need
2269 * to rely on ttwu() to place the task on a valid ->cpus_allowed
2270 * cpu.
2271 *
2272 * Since this is common to all placement strategies, this lives here.
2273 *
2274 * [ this allows ->select_task() to simply return task_cpu(p) and
2275 * not worry about this generic constraint ]
2276 */
2277 if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
Peter Zijlstra70f11202009-12-20 17:36:27 +01002278 !cpu_online(cpu)))
Peter Zijlstra5da9a0f2009-12-16 18:04:38 +01002279 cpu = select_fallback_rq(task_cpu(p), p);
Peter Zijlstrae2912002009-12-16 18:04:36 +01002280
2281 return cpu;
Peter Zijlstra970b13b2009-11-25 13:31:39 +01002282}
Mike Galbraith09a40af2010-04-15 07:29:59 +02002283
2284static void update_avg(u64 *avg, u64 sample)
2285{
2286 s64 diff = sample - *avg;
2287 *avg += diff >> 3;
2288}
Peter Zijlstra970b13b2009-11-25 13:31:39 +01002289#endif
2290
Linus Torvalds1da177e2005-04-16 15:20:36 -07002291/***
2292 * try_to_wake_up - wake up a thread
2293 * @p: the to-be-woken-up thread
2294 * @state: the mask of task states that can be woken
2295 * @sync: do a synchronous wakeup?
2296 *
2297 * Put it on the run-queue if it's not already there. The "current"
2298 * thread is always on the run-queue (except when the actual
2299 * re-schedule is in progress), and as such you're allowed to do
2300 * the simpler "current->state = TASK_RUNNING" to mark yourself
2301 * runnable without the overhead of this.
2302 *
2303 * returns failure only if the task is already active.
2304 */
Peter Zijlstra7d478722009-09-14 19:55:44 +02002305static int try_to_wake_up(struct task_struct *p, unsigned int state,
2306 int wake_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002307{
Ingo Molnarcc367732007-10-15 17:00:18 +02002308 int cpu, orig_cpu, this_cpu, success = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002309 unsigned long flags;
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002310 unsigned long en_flags = ENQUEUE_WAKEUP;
Dan Carpenterab3b3aa2010-03-06 14:17:52 +03002311 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002312
Peter Zijlstrae9c84312009-09-15 14:43:03 +02002313 this_cpu = get_cpu();
Peter Zijlstra2398f2c2008-06-27 13:41:35 +02002314
Linus Torvalds04e2f172008-02-23 18:05:03 -08002315 smp_wmb();
Dan Carpenterab3b3aa2010-03-06 14:17:52 +03002316 rq = task_rq_lock(p, &flags);
Peter Zijlstrae9c84312009-09-15 14:43:03 +02002317 if (!(p->state & state))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002318 goto out;
2319
Ingo Molnardd41f592007-07-09 18:51:59 +02002320 if (p->se.on_rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002321 goto out_running;
2322
2323 cpu = task_cpu(p);
Ingo Molnarcc367732007-10-15 17:00:18 +02002324 orig_cpu = cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002325
2326#ifdef CONFIG_SMP
2327 if (unlikely(task_running(rq, p)))
2328 goto out_activate;
2329
Peter Zijlstrae9c84312009-09-15 14:43:03 +02002330 /*
2331 * In order to handle concurrent wakeups and release the rq->lock
2332 * we put the task in TASK_WAKING state.
Ingo Molnareb240732009-09-16 21:09:13 +02002333 *
2334 * First fix up the nr_uninterruptible count:
Peter Zijlstrae9c84312009-09-15 14:43:03 +02002335 */
Peter Zijlstracc87f762010-03-26 12:22:14 +01002336 if (task_contributes_to_load(p)) {
2337 if (likely(cpu_online(orig_cpu)))
2338 rq->nr_uninterruptible--;
2339 else
2340 this_rq()->nr_uninterruptible--;
2341 }
Peter Zijlstrae9c84312009-09-15 14:43:03 +02002342 p->state = TASK_WAKING;
Peter Zijlstraefbbd052009-12-16 18:04:40 +01002343
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002344 if (p->sched_class->task_waking) {
Peter Zijlstraefbbd052009-12-16 18:04:40 +01002345 p->sched_class->task_waking(rq, p);
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002346 en_flags |= ENQUEUE_WAKING;
Peter Zijlstra0970d292010-02-15 14:45:54 +01002347 }
Peter Zijlstraab19cb22009-11-27 15:44:43 +01002348
Peter Zijlstra0017d732010-03-24 18:34:10 +01002349 cpu = select_task_rq(rq, p, SD_BALANCE_WAKE, wake_flags);
2350 if (cpu != orig_cpu)
Mike Galbraithf5dc3752009-10-09 08:35:03 +02002351 set_task_cpu(p, cpu);
Peter Zijlstra0017d732010-03-24 18:34:10 +01002352 __task_rq_unlock(rq);
Peter Zijlstraab19cb22009-11-27 15:44:43 +01002353
Peter Zijlstra0970d292010-02-15 14:45:54 +01002354 rq = cpu_rq(cpu);
2355 raw_spin_lock(&rq->lock);
Mike Galbraithf5dc3752009-10-09 08:35:03 +02002356
Peter Zijlstra0970d292010-02-15 14:45:54 +01002357 /*
2358 * We migrated the task without holding either rq->lock, however
2359 * since the task is not on the task list itself, nobody else
2360 * will try and migrate the task, hence the rq should match the
2361 * cpu we just moved it to.
2362 */
2363 WARN_ON(task_cpu(p) != cpu);
Peter Zijlstrae9c84312009-09-15 14:43:03 +02002364 WARN_ON(p->state != TASK_WAKING);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002365
Gregory Haskinse7693a32008-01-25 21:08:09 +01002366#ifdef CONFIG_SCHEDSTATS
2367 schedstat_inc(rq, ttwu_count);
2368 if (cpu == this_cpu)
2369 schedstat_inc(rq, ttwu_local);
2370 else {
2371 struct sched_domain *sd;
2372 for_each_domain(this_cpu, sd) {
Rusty Russell758b2cd2008-11-25 02:35:04 +10302373 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
Gregory Haskinse7693a32008-01-25 21:08:09 +01002374 schedstat_inc(sd, ttwu_wake_remote);
2375 break;
2376 }
2377 }
2378 }
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02002379#endif /* CONFIG_SCHEDSTATS */
Gregory Haskinse7693a32008-01-25 21:08:09 +01002380
Linus Torvalds1da177e2005-04-16 15:20:36 -07002381out_activate:
2382#endif /* CONFIG_SMP */
Lucas De Marchi41acab82010-03-10 23:37:45 -03002383 schedstat_inc(p, se.statistics.nr_wakeups);
Peter Zijlstra7d478722009-09-14 19:55:44 +02002384 if (wake_flags & WF_SYNC)
Lucas De Marchi41acab82010-03-10 23:37:45 -03002385 schedstat_inc(p, se.statistics.nr_wakeups_sync);
Ingo Molnarcc367732007-10-15 17:00:18 +02002386 if (orig_cpu != cpu)
Lucas De Marchi41acab82010-03-10 23:37:45 -03002387 schedstat_inc(p, se.statistics.nr_wakeups_migrate);
Ingo Molnarcc367732007-10-15 17:00:18 +02002388 if (cpu == this_cpu)
Lucas De Marchi41acab82010-03-10 23:37:45 -03002389 schedstat_inc(p, se.statistics.nr_wakeups_local);
Ingo Molnarcc367732007-10-15 17:00:18 +02002390 else
Lucas De Marchi41acab82010-03-10 23:37:45 -03002391 schedstat_inc(p, se.statistics.nr_wakeups_remote);
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01002392 activate_task(rq, p, en_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393 success = 1;
2394
2395out_running:
Peter Zijlstra27a9da62010-05-04 20:36:56 +02002396 trace_sched_wakeup(p, success);
Peter Zijlstra7d478722009-09-14 19:55:44 +02002397 check_preempt_curr(rq, p, wake_flags);
Ingo Molnar4ae7d5c2008-03-19 01:42:00 +01002398
Linus Torvalds1da177e2005-04-16 15:20:36 -07002399 p->state = TASK_RUNNING;
Steven Rostedt9a897c52008-01-25 21:08:22 +01002400#ifdef CONFIG_SMP
Peter Zijlstraefbbd052009-12-16 18:04:40 +01002401 if (p->sched_class->task_woken)
2402 p->sched_class->task_woken(rq, p);
Mike Galbraitheae0c9d2009-11-10 03:50:02 +01002403
2404 if (unlikely(rq->idle_stamp)) {
2405 u64 delta = rq->clock - rq->idle_stamp;
2406 u64 max = 2*sysctl_sched_migration_cost;
2407
2408 if (delta > max)
2409 rq->avg_idle = max;
2410 else
2411 update_avg(&rq->avg_idle, delta);
2412 rq->idle_stamp = 0;
2413 }
Steven Rostedt9a897c52008-01-25 21:08:22 +01002414#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002415out:
2416 task_rq_unlock(rq, &flags);
Peter Zijlstrae9c84312009-09-15 14:43:03 +02002417 put_cpu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418
2419 return success;
2420}
2421
David Howells50fa6102009-04-28 15:01:38 +01002422/**
2423 * wake_up_process - Wake up a specific process
2424 * @p: The process to be woken up.
2425 *
2426 * Attempt to wake up the nominated process and move it to the set of runnable
2427 * processes. Returns 1 if the process was woken up, 0 if it was already
2428 * running.
2429 *
2430 * It may be assumed that this function implies a write memory barrier before
2431 * changing the task state if and only if any tasks are woken up.
2432 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08002433int wake_up_process(struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002434{
Matthew Wilcoxd9514f62007-12-06 11:07:07 -05002435 return try_to_wake_up(p, TASK_ALL, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002436}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002437EXPORT_SYMBOL(wake_up_process);
2438
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08002439int wake_up_state(struct task_struct *p, unsigned int state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440{
2441 return try_to_wake_up(p, state, 0);
2442}
2443
Linus Torvalds1da177e2005-04-16 15:20:36 -07002444/*
2445 * Perform scheduler related setup for a newly forked process p.
2446 * p is forked by current.
Ingo Molnardd41f592007-07-09 18:51:59 +02002447 *
2448 * __sched_fork() is basic setup used by init_idle() too:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449 */
Ingo Molnardd41f592007-07-09 18:51:59 +02002450static void __sched_fork(struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451{
Ingo Molnardd41f592007-07-09 18:51:59 +02002452 p->se.exec_start = 0;
2453 p->se.sum_exec_runtime = 0;
Ingo Molnarf6cf8912007-08-28 12:53:24 +02002454 p->se.prev_sum_exec_runtime = 0;
Ingo Molnar6c594c22008-12-14 12:34:15 +01002455 p->se.nr_migrations = 0;
Ingo Molnar6cfb0d52007-08-02 17:41:40 +02002456
2457#ifdef CONFIG_SCHEDSTATS
Lucas De Marchi41acab82010-03-10 23:37:45 -03002458 memset(&p->se.statistics, 0, sizeof(p->se.statistics));
Ingo Molnar6cfb0d52007-08-02 17:41:40 +02002459#endif
Nick Piggin476d1392005-06-25 14:57:29 -07002460
Peter Zijlstrafa717062008-01-25 21:08:27 +01002461 INIT_LIST_HEAD(&p->rt.run_list);
Ingo Molnardd41f592007-07-09 18:51:59 +02002462 p->se.on_rq = 0;
Peter Zijlstra4a55bd52008-04-19 19:45:00 +02002463 INIT_LIST_HEAD(&p->se.group_node);
Nick Piggin476d1392005-06-25 14:57:29 -07002464
Avi Kivitye107be32007-07-26 13:40:43 +02002465#ifdef CONFIG_PREEMPT_NOTIFIERS
2466 INIT_HLIST_HEAD(&p->preempt_notifiers);
2467#endif
Ingo Molnardd41f592007-07-09 18:51:59 +02002468}
2469
2470/*
2471 * fork()/clone()-time setup:
2472 */
2473void sched_fork(struct task_struct *p, int clone_flags)
2474{
2475 int cpu = get_cpu();
2476
2477 __sched_fork(p);
Peter Zijlstra06b83b52009-12-16 18:04:35 +01002478 /*
Peter Zijlstra0017d732010-03-24 18:34:10 +01002479 * We mark the process as running here. This guarantees that
Peter Zijlstra06b83b52009-12-16 18:04:35 +01002480 * nobody will actually run it, and a signal or other external
2481 * event cannot wake it up and insert it on the runqueue either.
2482 */
Peter Zijlstra0017d732010-03-24 18:34:10 +01002483 p->state = TASK_RUNNING;
Ingo Molnardd41f592007-07-09 18:51:59 +02002484
Ingo Molnarb29739f2006-06-27 02:54:51 -07002485 /*
Mike Galbraithb9dc29e2009-06-17 10:46:01 +02002486 * Revert to default priority/policy on fork if requested.
2487 */
2488 if (unlikely(p->sched_reset_on_fork)) {
Peter Williamsf83f9ac2009-09-24 06:47:10 +00002489 if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) {
Mike Galbraithb9dc29e2009-06-17 10:46:01 +02002490 p->policy = SCHED_NORMAL;
Peter Williamsf83f9ac2009-09-24 06:47:10 +00002491 p->normal_prio = p->static_prio;
2492 }
Mike Galbraithb9dc29e2009-06-17 10:46:01 +02002493
Mike Galbraith6c697bd2009-06-17 10:48:02 +02002494 if (PRIO_TO_NICE(p->static_prio) < 0) {
2495 p->static_prio = NICE_TO_PRIO(0);
Peter Williamsf83f9ac2009-09-24 06:47:10 +00002496 p->normal_prio = p->static_prio;
Mike Galbraith6c697bd2009-06-17 10:48:02 +02002497 set_load_weight(p);
2498 }
2499
Mike Galbraithb9dc29e2009-06-17 10:46:01 +02002500 /*
2501 * We don't need the reset flag anymore after the fork. It has
2502 * fulfilled its duty:
2503 */
2504 p->sched_reset_on_fork = 0;
2505 }
Lennart Poetteringca94c442009-06-15 17:17:47 +02002506
Peter Williamsf83f9ac2009-09-24 06:47:10 +00002507 /*
2508 * Make sure we do not leak PI boosting priority to the child.
2509 */
2510 p->prio = current->normal_prio;
2511
Hiroshi Shimamoto2ddbf952007-10-15 17:00:11 +02002512 if (!rt_prio(p->prio))
2513 p->sched_class = &fair_sched_class;
Ingo Molnarb29739f2006-06-27 02:54:51 -07002514
Peter Zijlstracd29fe62009-11-27 17:32:46 +01002515 if (p->sched_class->task_fork)
2516 p->sched_class->task_fork(p);
2517
Peter Zijlstra5f3edc12009-09-10 13:42:00 +02002518 set_task_cpu(p, cpu);
2519
Chandra Seetharaman52f17b62006-07-14 00:24:38 -07002520#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
Ingo Molnardd41f592007-07-09 18:51:59 +02002521 if (likely(sched_info_on()))
Chandra Seetharaman52f17b62006-07-14 00:24:38 -07002522 memset(&p->sched_info, 0, sizeof(p->sched_info));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002523#endif
Chen, Kenneth Wd6077cb2006-02-14 13:53:10 -08002524#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
Nick Piggin4866cde2005-06-25 14:57:23 -07002525 p->oncpu = 0;
2526#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002527#ifdef CONFIG_PREEMPT
Nick Piggin4866cde2005-06-25 14:57:23 -07002528 /* Want to start with kernel preemption disabled. */
Al Viroa1261f52005-11-13 16:06:55 -08002529 task_thread_info(p)->preempt_count = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002530#endif
Gregory Haskins917b6272008-12-29 09:39:53 -05002531 plist_node_init(&p->pushable_tasks, MAX_PRIO);
2532
Nick Piggin476d1392005-06-25 14:57:29 -07002533 put_cpu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002534}
2535
2536/*
2537 * wake_up_new_task - wake up a newly created task for the first time.
2538 *
2539 * This function will do some initial scheduler statistics housekeeping
2540 * that must be done for every newly created context, then puts the task
2541 * on the runqueue and wakes it.
2542 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08002543void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002544{
2545 unsigned long flags;
Ingo Molnardd41f592007-07-09 18:51:59 +02002546 struct rq *rq;
Andrew Mortonc8906922010-03-11 14:08:43 -08002547 int cpu __maybe_unused = get_cpu();
Peter Zijlstrafabf3182010-01-21 21:04:57 +01002548
2549#ifdef CONFIG_SMP
Peter Zijlstra0017d732010-03-24 18:34:10 +01002550 rq = task_rq_lock(p, &flags);
2551 p->state = TASK_WAKING;
2552
Peter Zijlstrafabf3182010-01-21 21:04:57 +01002553 /*
2554 * Fork balancing, do it here and not earlier because:
2555 * - cpus_allowed can change in the fork path
2556 * - any previously selected cpu might disappear through hotplug
2557 *
Peter Zijlstra0017d732010-03-24 18:34:10 +01002558 * We set TASK_WAKING so that select_task_rq() can drop rq->lock
2559 * without people poking at ->cpus_allowed.
Peter Zijlstrafabf3182010-01-21 21:04:57 +01002560 */
Peter Zijlstra0017d732010-03-24 18:34:10 +01002561 cpu = select_task_rq(rq, p, SD_BALANCE_FORK, 0);
Peter Zijlstrafabf3182010-01-21 21:04:57 +01002562 set_task_cpu(p, cpu);
Peter Zijlstra0017d732010-03-24 18:34:10 +01002563
2564 p->state = TASK_RUNNING;
2565 task_rq_unlock(rq, &flags);
Peter Zijlstrafabf3182010-01-21 21:04:57 +01002566#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002567
Peter Zijlstra0017d732010-03-24 18:34:10 +01002568 rq = task_rq_lock(p, &flags);
Peter Zijlstracd29fe62009-11-27 17:32:46 +01002569 activate_task(rq, p, 0);
Peter Zijlstra27a9da62010-05-04 20:36:56 +02002570 trace_sched_wakeup_new(p, 1);
Peter Zijlstraa7558e02009-09-14 20:02:34 +02002571 check_preempt_curr(rq, p, WF_FORK);
Steven Rostedt9a897c52008-01-25 21:08:22 +01002572#ifdef CONFIG_SMP
Peter Zijlstraefbbd052009-12-16 18:04:40 +01002573 if (p->sched_class->task_woken)
2574 p->sched_class->task_woken(rq, p);
Steven Rostedt9a897c52008-01-25 21:08:22 +01002575#endif
Ingo Molnardd41f592007-07-09 18:51:59 +02002576 task_rq_unlock(rq, &flags);
Peter Zijlstrafabf3182010-01-21 21:04:57 +01002577 put_cpu();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002578}
2579
Avi Kivitye107be32007-07-26 13:40:43 +02002580#ifdef CONFIG_PREEMPT_NOTIFIERS
2581
2582/**
Luis Henriques80dd99b2009-03-16 19:58:09 +00002583 * preempt_notifier_register - tell me when current is being preempted & rescheduled
Randy Dunlap421cee22007-07-31 00:37:50 -07002584 * @notifier: notifier struct to register
Avi Kivitye107be32007-07-26 13:40:43 +02002585 */
2586void preempt_notifier_register(struct preempt_notifier *notifier)
2587{
2588 hlist_add_head(&notifier->link, &current->preempt_notifiers);
2589}
2590EXPORT_SYMBOL_GPL(preempt_notifier_register);
2591
2592/**
2593 * preempt_notifier_unregister - no longer interested in preemption notifications
Randy Dunlap421cee22007-07-31 00:37:50 -07002594 * @notifier: notifier struct to unregister
Avi Kivitye107be32007-07-26 13:40:43 +02002595 *
2596 * This is safe to call from within a preemption notifier.
2597 */
2598void preempt_notifier_unregister(struct preempt_notifier *notifier)
2599{
2600 hlist_del(&notifier->link);
2601}
2602EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2603
2604static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2605{
2606 struct preempt_notifier *notifier;
2607 struct hlist_node *node;
2608
2609 hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
2610 notifier->ops->sched_in(notifier, raw_smp_processor_id());
2611}
2612
2613static void
2614fire_sched_out_preempt_notifiers(struct task_struct *curr,
2615 struct task_struct *next)
2616{
2617 struct preempt_notifier *notifier;
2618 struct hlist_node *node;
2619
2620 hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link)
2621 notifier->ops->sched_out(notifier, next);
2622}
2623
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02002624#else /* !CONFIG_PREEMPT_NOTIFIERS */
Avi Kivitye107be32007-07-26 13:40:43 +02002625
2626static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2627{
2628}
2629
2630static void
2631fire_sched_out_preempt_notifiers(struct task_struct *curr,
2632 struct task_struct *next)
2633{
2634}
2635
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02002636#endif /* CONFIG_PREEMPT_NOTIFIERS */
Avi Kivitye107be32007-07-26 13:40:43 +02002637
Linus Torvalds1da177e2005-04-16 15:20:36 -07002638/**
Nick Piggin4866cde2005-06-25 14:57:23 -07002639 * prepare_task_switch - prepare to switch tasks
2640 * @rq: the runqueue preparing to switch
Randy Dunlap421cee22007-07-31 00:37:50 -07002641 * @prev: the current task that is being switched out
Nick Piggin4866cde2005-06-25 14:57:23 -07002642 * @next: the task we are going to switch to.
2643 *
2644 * This is called with the rq lock held and interrupts off. It must
2645 * be paired with a subsequent finish_task_switch after the context
2646 * switch.
2647 *
2648 * prepare_task_switch sets up locking and calls architecture specific
2649 * hooks.
2650 */
Avi Kivitye107be32007-07-26 13:40:43 +02002651static inline void
2652prepare_task_switch(struct rq *rq, struct task_struct *prev,
2653 struct task_struct *next)
Nick Piggin4866cde2005-06-25 14:57:23 -07002654{
Avi Kivitye107be32007-07-26 13:40:43 +02002655 fire_sched_out_preempt_notifiers(prev, next);
Nick Piggin4866cde2005-06-25 14:57:23 -07002656 prepare_lock_switch(rq, next);
2657 prepare_arch_switch(next);
2658}
2659
2660/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002661 * finish_task_switch - clean up after a task-switch
Jeff Garzik344baba2005-09-07 01:15:17 -04002662 * @rq: runqueue associated with task-switch
Linus Torvalds1da177e2005-04-16 15:20:36 -07002663 * @prev: the thread we just switched away from.
2664 *
Nick Piggin4866cde2005-06-25 14:57:23 -07002665 * finish_task_switch must be called after the context switch, paired
2666 * with a prepare_task_switch call before the context switch.
2667 * finish_task_switch will reconcile locking set up by prepare_task_switch,
2668 * and do any other architecture-specific cleanup actions.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002669 *
2670 * Note that we may have delayed dropping an mm in context_switch(). If
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01002671 * so, we finish that here outside of the runqueue lock. (Doing it
Linus Torvalds1da177e2005-04-16 15:20:36 -07002672 * with the lock held can cause deadlocks; see schedule() for
2673 * details.)
2674 */
Alexey Dobriyana9957442007-10-15 17:00:13 +02002675static void finish_task_switch(struct rq *rq, struct task_struct *prev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002676 __releases(rq->lock)
2677{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002678 struct mm_struct *mm = rq->prev_mm;
Oleg Nesterov55a101f2006-09-29 02:01:10 -07002679 long prev_state;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002680
2681 rq->prev_mm = NULL;
2682
2683 /*
2684 * A task struct has one reference for the use as "current".
Oleg Nesterovc394cc92006-09-29 02:01:11 -07002685 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
Oleg Nesterov55a101f2006-09-29 02:01:10 -07002686 * schedule one last time. The schedule call will never return, and
2687 * the scheduled task must drop that reference.
Oleg Nesterovc394cc92006-09-29 02:01:11 -07002688 * The test for TASK_DEAD must occur while the runqueue locks are
Linus Torvalds1da177e2005-04-16 15:20:36 -07002689 * still held, otherwise prev could be scheduled on another cpu, die
2690 * there before we look at prev->state, and then the reference would
2691 * be dropped twice.
2692 * Manfred Spraul <manfred@colorfullife.com>
2693 */
Oleg Nesterov55a101f2006-09-29 02:01:10 -07002694 prev_state = prev->state;
Nick Piggin4866cde2005-06-25 14:57:23 -07002695 finish_arch_switch(prev);
Jamie Iles8381f652010-01-08 15:27:33 +00002696#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
2697 local_irq_disable();
2698#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
Peter Zijlstra49f47432009-12-27 11:51:52 +01002699 perf_event_task_sched_in(current);
Jamie Iles8381f652010-01-08 15:27:33 +00002700#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
2701 local_irq_enable();
2702#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
Nick Piggin4866cde2005-06-25 14:57:23 -07002703 finish_lock_switch(rq, prev);
Steven Rostedte8fa1362008-01-25 21:08:05 +01002704
Avi Kivitye107be32007-07-26 13:40:43 +02002705 fire_sched_in_preempt_notifiers(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002706 if (mm)
2707 mmdrop(mm);
Oleg Nesterovc394cc92006-09-29 02:01:11 -07002708 if (unlikely(prev_state == TASK_DEAD)) {
bibo maoc6fd91f2006-03-26 01:38:20 -08002709 /*
2710 * Remove function-return probe instances associated with this
2711 * task and put them back on the free list.
Ingo Molnar9761eea2007-07-09 18:52:00 +02002712 */
bibo maoc6fd91f2006-03-26 01:38:20 -08002713 kprobe_flush_task(prev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002714 put_task_struct(prev);
bibo maoc6fd91f2006-03-26 01:38:20 -08002715 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002716}
2717
Gregory Haskins3f029d32009-07-29 11:08:47 -04002718#ifdef CONFIG_SMP
2719
2720/* assumes rq->lock is held */
2721static inline void pre_schedule(struct rq *rq, struct task_struct *prev)
2722{
2723 if (prev->sched_class->pre_schedule)
2724 prev->sched_class->pre_schedule(rq, prev);
2725}
2726
2727/* rq->lock is NOT held, but preemption is disabled */
2728static inline void post_schedule(struct rq *rq)
2729{
2730 if (rq->post_schedule) {
2731 unsigned long flags;
2732
Thomas Gleixner05fa7852009-11-17 14:28:38 +01002733 raw_spin_lock_irqsave(&rq->lock, flags);
Gregory Haskins3f029d32009-07-29 11:08:47 -04002734 if (rq->curr->sched_class->post_schedule)
2735 rq->curr->sched_class->post_schedule(rq);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01002736 raw_spin_unlock_irqrestore(&rq->lock, flags);
Gregory Haskins3f029d32009-07-29 11:08:47 -04002737
2738 rq->post_schedule = 0;
2739 }
2740}
2741
2742#else
2743
2744static inline void pre_schedule(struct rq *rq, struct task_struct *p)
2745{
2746}
2747
2748static inline void post_schedule(struct rq *rq)
2749{
2750}
2751
2752#endif
2753
Linus Torvalds1da177e2005-04-16 15:20:36 -07002754/**
2755 * schedule_tail - first thing a freshly forked thread must call.
2756 * @prev: the thread we just switched away from.
2757 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07002758asmlinkage void schedule_tail(struct task_struct *prev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002759 __releases(rq->lock)
2760{
Ingo Molnar70b97a72006-07-03 00:25:42 -07002761 struct rq *rq = this_rq();
2762
Nick Piggin4866cde2005-06-25 14:57:23 -07002763 finish_task_switch(rq, prev);
Steven Rostedtda19ab52009-07-29 00:21:22 -04002764
Gregory Haskins3f029d32009-07-29 11:08:47 -04002765 /*
2766 * FIXME: do we need to worry about rq being invalidated by the
2767 * task_switch?
2768 */
2769 post_schedule(rq);
Steven Rostedtda19ab52009-07-29 00:21:22 -04002770
Nick Piggin4866cde2005-06-25 14:57:23 -07002771#ifdef __ARCH_WANT_UNLOCKED_CTXSW
2772 /* In this case, finish_task_switch does not reenable preemption */
2773 preempt_enable();
2774#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002775 if (current->set_child_tid)
Pavel Emelyanovb4888932007-10-18 23:40:14 -07002776 put_user(task_pid_vnr(current), current->set_child_tid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002777}
2778
2779/*
2780 * context_switch - switch to the new MM and the new
2781 * thread's register state.
2782 */
Ingo Molnardd41f592007-07-09 18:51:59 +02002783static inline void
Ingo Molnar70b97a72006-07-03 00:25:42 -07002784context_switch(struct rq *rq, struct task_struct *prev,
Ingo Molnar36c8b582006-07-03 00:25:41 -07002785 struct task_struct *next)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002786{
Ingo Molnardd41f592007-07-09 18:51:59 +02002787 struct mm_struct *mm, *oldmm;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002788
Avi Kivitye107be32007-07-26 13:40:43 +02002789 prepare_task_switch(rq, prev, next);
Peter Zijlstra27a9da62010-05-04 20:36:56 +02002790 trace_sched_switch(prev, next);
Ingo Molnardd41f592007-07-09 18:51:59 +02002791 mm = next->mm;
2792 oldmm = prev->active_mm;
Zachary Amsden9226d122007-02-13 13:26:21 +01002793 /*
2794 * For paravirt, this is coupled with an exit in switch_to to
2795 * combine the page table reload and the switch backend into
2796 * one hypercall.
2797 */
Jeremy Fitzhardinge224101e2009-02-18 11:18:57 -08002798 arch_start_context_switch(prev);
Zachary Amsden9226d122007-02-13 13:26:21 +01002799
Tim Blechmann710390d2009-11-24 11:55:27 +01002800 if (likely(!mm)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002801 next->active_mm = oldmm;
2802 atomic_inc(&oldmm->mm_count);
2803 enter_lazy_tlb(oldmm, next);
2804 } else
2805 switch_mm(oldmm, mm, next);
2806
Tim Blechmann710390d2009-11-24 11:55:27 +01002807 if (likely(!prev->mm)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002808 prev->active_mm = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002809 rq->prev_mm = oldmm;
2810 }
Ingo Molnar3a5f5e42006-07-14 00:24:27 -07002811 /*
2812 * Since the runqueue lock will be released by the next
2813 * task (which is an invalid locking op but in the case
2814 * of the scheduler it's an obvious special-case), so we
2815 * do an early lockdep release here:
2816 */
2817#ifndef __ARCH_WANT_UNLOCKED_CTXSW
Ingo Molnar8a25d5d2006-07-03 00:24:54 -07002818 spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
Ingo Molnar3a5f5e42006-07-14 00:24:27 -07002819#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002820
2821 /* Here we just switch the register state and the stack. */
2822 switch_to(prev, next, prev);
2823
Ingo Molnardd41f592007-07-09 18:51:59 +02002824 barrier();
2825 /*
2826 * this_rq must be evaluated again because prev may have moved
2827 * CPUs since it called schedule(), thus the 'rq' on its stack
2828 * frame will be invalid.
2829 */
2830 finish_task_switch(this_rq(), prev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002831}
2832
2833/*
2834 * nr_running, nr_uninterruptible and nr_context_switches:
2835 *
2836 * externally visible scheduler statistics: current number of runnable
2837 * threads, current number of uninterruptible-sleeping threads, total
2838 * number of context switches performed since bootup.
2839 */
2840unsigned long nr_running(void)
2841{
2842 unsigned long i, sum = 0;
2843
2844 for_each_online_cpu(i)
2845 sum += cpu_rq(i)->nr_running;
2846
2847 return sum;
2848}
2849
2850unsigned long nr_uninterruptible(void)
2851{
2852 unsigned long i, sum = 0;
2853
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08002854 for_each_possible_cpu(i)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002855 sum += cpu_rq(i)->nr_uninterruptible;
2856
2857 /*
2858 * Since we read the counters lockless, it might be slightly
2859 * inaccurate. Do not allow it to go below zero though:
2860 */
2861 if (unlikely((long)sum < 0))
2862 sum = 0;
2863
2864 return sum;
2865}
2866
2867unsigned long long nr_context_switches(void)
2868{
Steven Rostedtcc94abf2006-06-27 02:54:31 -07002869 int i;
2870 unsigned long long sum = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002871
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08002872 for_each_possible_cpu(i)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002873 sum += cpu_rq(i)->nr_switches;
2874
2875 return sum;
2876}
2877
2878unsigned long nr_iowait(void)
2879{
2880 unsigned long i, sum = 0;
2881
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08002882 for_each_possible_cpu(i)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002883 sum += atomic_read(&cpu_rq(i)->nr_iowait);
2884
2885 return sum;
2886}
2887
Arjan van de Ven69d25872009-09-21 17:04:08 -07002888unsigned long nr_iowait_cpu(void)
2889{
2890 struct rq *this = this_rq();
2891 return atomic_read(&this->nr_iowait);
2892}
2893
2894unsigned long this_cpu_load(void)
2895{
2896 struct rq *this = this_rq();
2897 return this->cpu_load[0];
2898}
2899
2900
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02002901/* Variables and functions for calc_load */
2902static atomic_long_t calc_load_tasks;
2903static unsigned long calc_load_update;
2904unsigned long avenrun[3];
2905EXPORT_SYMBOL(avenrun);
2906
Peter Zijlstra74f51872010-04-22 21:50:19 +02002907static long calc_load_fold_active(struct rq *this_rq)
2908{
2909 long nr_active, delta = 0;
2910
2911 nr_active = this_rq->nr_running;
2912 nr_active += (long) this_rq->nr_uninterruptible;
2913
2914 if (nr_active != this_rq->calc_load_active) {
2915 delta = nr_active - this_rq->calc_load_active;
2916 this_rq->calc_load_active = nr_active;
2917 }
2918
2919 return delta;
2920}
2921
2922#ifdef CONFIG_NO_HZ
2923/*
2924 * For NO_HZ we delay the active fold to the next LOAD_FREQ update.
2925 *
2926 * When making the ILB scale, we should try to pull this in as well.
2927 */
2928static atomic_long_t calc_load_tasks_idle;
2929
2930static void calc_load_account_idle(struct rq *this_rq)
2931{
2932 long delta;
2933
2934 delta = calc_load_fold_active(this_rq);
2935 if (delta)
2936 atomic_long_add(delta, &calc_load_tasks_idle);
2937}
2938
2939static long calc_load_fold_idle(void)
2940{
2941 long delta = 0;
2942
2943 /*
2944 * Its got a race, we don't care...
2945 */
2946 if (atomic_long_read(&calc_load_tasks_idle))
2947 delta = atomic_long_xchg(&calc_load_tasks_idle, 0);
2948
2949 return delta;
2950}
2951#else
2952static void calc_load_account_idle(struct rq *this_rq)
2953{
2954}
2955
2956static inline long calc_load_fold_idle(void)
2957{
2958 return 0;
2959}
2960#endif
2961
Thomas Gleixner2d024942009-05-02 20:08:52 +02002962/**
2963 * get_avenrun - get the load average array
2964 * @loads: pointer to dest load array
2965 * @offset: offset to add
2966 * @shift: shift count to shift the result left
2967 *
2968 * These values are estimates at best, so no need for locking.
2969 */
2970void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
2971{
2972 loads[0] = (avenrun[0] + offset) << shift;
2973 loads[1] = (avenrun[1] + offset) << shift;
2974 loads[2] = (avenrun[2] + offset) << shift;
2975}
2976
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02002977static unsigned long
2978calc_load(unsigned long load, unsigned long exp, unsigned long active)
Jack Steinerdb1b1fe2006-03-31 02:31:21 -08002979{
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02002980 load *= exp;
2981 load += active * (FIXED_1 - exp);
2982 return load >> FSHIFT;
2983}
Jack Steinerdb1b1fe2006-03-31 02:31:21 -08002984
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02002985/*
2986 * calc_load - update the avenrun load estimates 10 ticks after the
2987 * CPUs have updated calc_load_tasks.
2988 */
2989void calc_global_load(void)
2990{
2991 unsigned long upd = calc_load_update + 10;
2992 long active;
2993
2994 if (time_before(jiffies, upd))
2995 return;
2996
2997 active = atomic_long_read(&calc_load_tasks);
2998 active = active > 0 ? active * FIXED_1 : 0;
2999
3000 avenrun[0] = calc_load(avenrun[0], EXP_1, active);
3001 avenrun[1] = calc_load(avenrun[1], EXP_5, active);
3002 avenrun[2] = calc_load(avenrun[2], EXP_15, active);
3003
3004 calc_load_update += LOAD_FREQ;
3005}
3006
3007/*
Peter Zijlstra74f51872010-04-22 21:50:19 +02003008 * Called from update_cpu_load() to periodically update this CPU's
3009 * active count.
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02003010 */
3011static void calc_load_account_active(struct rq *this_rq)
3012{
Peter Zijlstra74f51872010-04-22 21:50:19 +02003013 long delta;
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02003014
Peter Zijlstra74f51872010-04-22 21:50:19 +02003015 if (time_before(jiffies, this_rq->calc_load_update))
3016 return;
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02003017
Peter Zijlstra74f51872010-04-22 21:50:19 +02003018 delta = calc_load_fold_active(this_rq);
3019 delta += calc_load_fold_idle();
3020 if (delta)
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02003021 atomic_long_add(delta, &calc_load_tasks);
Peter Zijlstra74f51872010-04-22 21:50:19 +02003022
3023 this_rq->calc_load_update += LOAD_FREQ;
Jack Steinerdb1b1fe2006-03-31 02:31:21 -08003024}
3025
Linus Torvalds1da177e2005-04-16 15:20:36 -07003026/*
Ingo Molnardd41f592007-07-09 18:51:59 +02003027 * Update rq->cpu_load[] statistics. This function is usually called every
3028 * scheduler tick (TICK_NSEC).
Ingo Molnar48f24c42006-07-03 00:25:40 -07003029 */
Ingo Molnardd41f592007-07-09 18:51:59 +02003030static void update_cpu_load(struct rq *this_rq)
Ingo Molnar48f24c42006-07-03 00:25:40 -07003031{
Dmitry Adamushko495eca42007-10-15 17:00:06 +02003032 unsigned long this_load = this_rq->load.weight;
Ingo Molnardd41f592007-07-09 18:51:59 +02003033 int i, scale;
3034
3035 this_rq->nr_load_updates++;
Ingo Molnardd41f592007-07-09 18:51:59 +02003036
3037 /* Update our load: */
3038 for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) {
3039 unsigned long old_load, new_load;
3040
3041 /* scale is effectively 1 << i now, and >> i divides by scale */
3042
3043 old_load = this_rq->cpu_load[i];
3044 new_load = this_load;
Ingo Molnara25707f2007-10-15 17:00:03 +02003045 /*
3046 * Round up the averaging division if load is increasing. This
3047 * prevents us from getting stuck on 9 if the load is 10, for
3048 * example.
3049 */
3050 if (new_load > old_load)
3051 new_load += scale-1;
Ingo Molnardd41f592007-07-09 18:51:59 +02003052 this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i;
3053 }
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02003054
Peter Zijlstra74f51872010-04-22 21:50:19 +02003055 calc_load_account_active(this_rq);
Ingo Molnar48f24c42006-07-03 00:25:40 -07003056}
3057
Ingo Molnardd41f592007-07-09 18:51:59 +02003058#ifdef CONFIG_SMP
3059
Ingo Molnar48f24c42006-07-03 00:25:40 -07003060/*
Peter Zijlstra38022902009-12-16 18:04:37 +01003061 * sched_exec - execve() is a valuable balancing opportunity, because at
3062 * this point the task has the smallest effective memory and cache footprint.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003063 */
Peter Zijlstra38022902009-12-16 18:04:37 +01003064void sched_exec(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003065{
Peter Zijlstra38022902009-12-16 18:04:37 +01003066 struct task_struct *p = current;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003067 unsigned long flags;
Ingo Molnar70b97a72006-07-03 00:25:42 -07003068 struct rq *rq;
Peter Zijlstra0017d732010-03-24 18:34:10 +01003069 int dest_cpu;
Peter Zijlstra38022902009-12-16 18:04:37 +01003070
Linus Torvalds1da177e2005-04-16 15:20:36 -07003071 rq = task_rq_lock(p, &flags);
Peter Zijlstra0017d732010-03-24 18:34:10 +01003072 dest_cpu = p->sched_class->select_task_rq(rq, p, SD_BALANCE_EXEC, 0);
3073 if (dest_cpu == smp_processor_id())
3074 goto unlock;
Peter Zijlstra38022902009-12-16 18:04:37 +01003075
3076 /*
3077 * select_task_rq() can race against ->cpus_allowed
3078 */
Oleg Nesterov30da6882010-03-15 10:10:19 +01003079 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed) &&
Tejun Heo969c7922010-05-06 18:49:21 +02003080 likely(cpu_active(dest_cpu)) && migrate_task(p, dest_cpu)) {
3081 struct migration_arg arg = { p, dest_cpu };
Ingo Molnar36c8b582006-07-03 00:25:41 -07003082
Linus Torvalds1da177e2005-04-16 15:20:36 -07003083 task_rq_unlock(rq, &flags);
Tejun Heo969c7922010-05-06 18:49:21 +02003084 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003085 return;
3086 }
Peter Zijlstra0017d732010-03-24 18:34:10 +01003087unlock:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003088 task_rq_unlock(rq, &flags);
3089}
3090
Linus Torvalds1da177e2005-04-16 15:20:36 -07003091#endif
3092
Linus Torvalds1da177e2005-04-16 15:20:36 -07003093DEFINE_PER_CPU(struct kernel_stat, kstat);
3094
3095EXPORT_PER_CPU_SYMBOL(kstat);
3096
3097/*
Hidetoshi Setoc5f8d992009-03-31 16:56:03 +09003098 * Return any ns on the sched_clock that have not yet been accounted in
Frank Mayharf06febc2008-09-12 09:54:39 -07003099 * @p in case that task is currently running.
Hidetoshi Setoc5f8d992009-03-31 16:56:03 +09003100 *
3101 * Called with task_rq_lock() held on @rq.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003102 */
Hidetoshi Setoc5f8d992009-03-31 16:56:03 +09003103static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
3104{
3105 u64 ns = 0;
3106
3107 if (task_current(rq, p)) {
3108 update_rq_clock(rq);
3109 ns = rq->clock - p->se.exec_start;
3110 if ((s64)ns < 0)
3111 ns = 0;
3112 }
3113
3114 return ns;
3115}
3116
Frank Mayharbb34d922008-09-12 09:54:39 -07003117unsigned long long task_delta_exec(struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003118{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003119 unsigned long flags;
Ingo Molnar41b86e92007-07-09 18:51:58 +02003120 struct rq *rq;
Frank Mayharbb34d922008-09-12 09:54:39 -07003121 u64 ns = 0;
Ingo Molnar48f24c42006-07-03 00:25:40 -07003122
Ingo Molnar41b86e92007-07-09 18:51:58 +02003123 rq = task_rq_lock(p, &flags);
Hidetoshi Setoc5f8d992009-03-31 16:56:03 +09003124 ns = do_task_delta_exec(p, rq);
3125 task_rq_unlock(rq, &flags);
Ingo Molnar15084872008-09-30 08:28:17 +02003126
Hidetoshi Setoc5f8d992009-03-31 16:56:03 +09003127 return ns;
3128}
Frank Mayharf06febc2008-09-12 09:54:39 -07003129
Hidetoshi Setoc5f8d992009-03-31 16:56:03 +09003130/*
3131 * Return accounted runtime for the task.
3132 * In case the task is currently running, return the runtime plus current's
3133 * pending runtime that have not been accounted yet.
3134 */
3135unsigned long long task_sched_runtime(struct task_struct *p)
3136{
3137 unsigned long flags;
3138 struct rq *rq;
3139 u64 ns = 0;
Ingo Molnar48f24c42006-07-03 00:25:40 -07003140
Hidetoshi Setoc5f8d992009-03-31 16:56:03 +09003141 rq = task_rq_lock(p, &flags);
3142 ns = p->se.sum_exec_runtime + do_task_delta_exec(p, rq);
3143 task_rq_unlock(rq, &flags);
3144
3145 return ns;
3146}
3147
3148/*
3149 * Return sum_exec_runtime for the thread group.
3150 * In case the task is currently running, return the sum plus current's
3151 * pending runtime that have not been accounted yet.
3152 *
3153 * Note that the thread group might have other running tasks as well,
3154 * so the return value not includes other pending runtime that other
3155 * running tasks might have.
3156 */
3157unsigned long long thread_group_sched_runtime(struct task_struct *p)
3158{
3159 struct task_cputime totals;
3160 unsigned long flags;
3161 struct rq *rq;
3162 u64 ns;
3163
3164 rq = task_rq_lock(p, &flags);
3165 thread_group_cputime(p, &totals);
3166 ns = totals.sum_exec_runtime + do_task_delta_exec(p, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003167 task_rq_unlock(rq, &flags);
3168
3169 return ns;
3170}
3171
3172/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003173 * Account user cpu time to a process.
3174 * @p: the process that the cpu time gets accounted to
Linus Torvalds1da177e2005-04-16 15:20:36 -07003175 * @cputime: the cpu time spent in user space since the last update
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003176 * @cputime_scaled: cputime scaled by cpu frequency
Linus Torvalds1da177e2005-04-16 15:20:36 -07003177 */
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003178void account_user_time(struct task_struct *p, cputime_t cputime,
3179 cputime_t cputime_scaled)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003180{
3181 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3182 cputime64_t tmp;
3183
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003184 /* Add user time to process. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003185 p->utime = cputime_add(p->utime, cputime);
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003186 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
Frank Mayharf06febc2008-09-12 09:54:39 -07003187 account_group_user_time(p, cputime);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003188
3189 /* Add user time to cpustat. */
3190 tmp = cputime_to_cputime64(cputime);
3191 if (TASK_NICE(p) > 0)
3192 cpustat->nice = cputime64_add(cpustat->nice, tmp);
3193 else
3194 cpustat->user = cputime64_add(cpustat->user, tmp);
Bharata B Raoef12fef2009-03-31 10:02:22 +05303195
3196 cpuacct_update_stats(p, CPUACCT_STAT_USER, cputime);
Jonathan Lim49b5cf32008-07-25 01:48:40 -07003197 /* Account for user time used */
3198 acct_update_integrals(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003199}
3200
3201/*
Laurent Vivier94886b82007-10-15 17:00:19 +02003202 * Account guest cpu time to a process.
3203 * @p: the process that the cpu time gets accounted to
3204 * @cputime: the cpu time spent in virtual machine since the last update
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003205 * @cputime_scaled: cputime scaled by cpu frequency
Laurent Vivier94886b82007-10-15 17:00:19 +02003206 */
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003207static void account_guest_time(struct task_struct *p, cputime_t cputime,
3208 cputime_t cputime_scaled)
Laurent Vivier94886b82007-10-15 17:00:19 +02003209{
3210 cputime64_t tmp;
3211 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
3212
3213 tmp = cputime_to_cputime64(cputime);
3214
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003215 /* Add guest time to process. */
Laurent Vivier94886b82007-10-15 17:00:19 +02003216 p->utime = cputime_add(p->utime, cputime);
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003217 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
Frank Mayharf06febc2008-09-12 09:54:39 -07003218 account_group_user_time(p, cputime);
Laurent Vivier94886b82007-10-15 17:00:19 +02003219 p->gtime = cputime_add(p->gtime, cputime);
3220
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003221 /* Add guest time to cpustat. */
Ryota Ozakice0e7b22009-10-24 01:20:10 +09003222 if (TASK_NICE(p) > 0) {
3223 cpustat->nice = cputime64_add(cpustat->nice, tmp);
3224 cpustat->guest_nice = cputime64_add(cpustat->guest_nice, tmp);
3225 } else {
3226 cpustat->user = cputime64_add(cpustat->user, tmp);
3227 cpustat->guest = cputime64_add(cpustat->guest, tmp);
3228 }
Laurent Vivier94886b82007-10-15 17:00:19 +02003229}
3230
3231/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07003232 * Account system cpu time to a process.
3233 * @p: the process that the cpu time gets accounted to
3234 * @hardirq_offset: the offset to subtract from hardirq_count()
3235 * @cputime: the cpu time spent in kernel space since the last update
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003236 * @cputime_scaled: cputime scaled by cpu frequency
Linus Torvalds1da177e2005-04-16 15:20:36 -07003237 */
3238void account_system_time(struct task_struct *p, int hardirq_offset,
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003239 cputime_t cputime, cputime_t cputime_scaled)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003240{
3241 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003242 cputime64_t tmp;
3243
Harvey Harrison983ed7a2008-04-24 18:17:55 -07003244 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003245 account_guest_time(p, cputime, cputime_scaled);
Harvey Harrison983ed7a2008-04-24 18:17:55 -07003246 return;
3247 }
Laurent Vivier94886b82007-10-15 17:00:19 +02003248
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003249 /* Add system time to process. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003250 p->stime = cputime_add(p->stime, cputime);
Martin Schwidefsky457533a2008-12-31 15:11:37 +01003251 p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
Frank Mayharf06febc2008-09-12 09:54:39 -07003252 account_group_system_time(p, cputime);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003253
3254 /* Add system time to cpustat. */
3255 tmp = cputime_to_cputime64(cputime);
3256 if (hardirq_count() - hardirq_offset)
3257 cpustat->irq = cputime64_add(cpustat->irq, tmp);
3258 else if (softirq_count())
3259 cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003260 else
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003261 cpustat->system = cputime64_add(cpustat->system, tmp);
3262
Bharata B Raoef12fef2009-03-31 10:02:22 +05303263 cpuacct_update_stats(p, CPUACCT_STAT_SYSTEM, cputime);
3264
Linus Torvalds1da177e2005-04-16 15:20:36 -07003265 /* Account for system time used */
3266 acct_update_integrals(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003267}
3268
3269/*
3270 * Account for involuntary wait time.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003271 * @steal: the cpu time spent in involuntary wait
Linus Torvalds1da177e2005-04-16 15:20:36 -07003272 */
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003273void account_steal_time(cputime_t cputime)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003274{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003275 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003276 cputime64_t cputime64 = cputime_to_cputime64(cputime);
3277
3278 cpustat->steal = cputime64_add(cpustat->steal, cputime64);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003279}
3280
Christoph Lameter7835b982006-12-10 02:20:22 -08003281/*
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003282 * Account for idle time.
3283 * @cputime: the cpu time spent in idle wait
Linus Torvalds1da177e2005-04-16 15:20:36 -07003284 */
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003285void account_idle_time(cputime_t cputime)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003286{
3287 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003288 cputime64_t cputime64 = cputime_to_cputime64(cputime);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003289 struct rq *rq = this_rq();
3290
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003291 if (atomic_read(&rq->nr_iowait) > 0)
3292 cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
3293 else
3294 cpustat->idle = cputime64_add(cpustat->idle, cputime64);
Christoph Lameter7835b982006-12-10 02:20:22 -08003295}
3296
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003297#ifndef CONFIG_VIRT_CPU_ACCOUNTING
3298
3299/*
3300 * Account a single tick of cpu time.
3301 * @p: the process that the cpu time gets accounted to
3302 * @user_tick: indicates if the tick is a user or a system tick
3303 */
3304void account_process_tick(struct task_struct *p, int user_tick)
3305{
Stanislaw Gruszkaa42548a2009-07-29 12:15:29 +02003306 cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy);
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003307 struct rq *rq = this_rq();
3308
3309 if (user_tick)
Stanislaw Gruszkaa42548a2009-07-29 12:15:29 +02003310 account_user_time(p, cputime_one_jiffy, one_jiffy_scaled);
Eric Dumazetf5f293a2009-04-29 14:44:49 +02003311 else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
Stanislaw Gruszkaa42548a2009-07-29 12:15:29 +02003312 account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy,
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003313 one_jiffy_scaled);
3314 else
Stanislaw Gruszkaa42548a2009-07-29 12:15:29 +02003315 account_idle_time(cputime_one_jiffy);
Martin Schwidefsky79741dd2008-12-31 15:11:38 +01003316}
3317
3318/*
3319 * Account multiple ticks of steal time.
3320 * @p: the process from which the cpu time has been stolen
3321 * @ticks: number of stolen ticks
3322 */
3323void account_steal_ticks(unsigned long ticks)
3324{
3325 account_steal_time(jiffies_to_cputime(ticks));
3326}
3327
3328/*
3329 * Account multiple ticks of idle time.
3330 * @ticks: number of stolen ticks
3331 */
3332void account_idle_ticks(unsigned long ticks)
3333{
3334 account_idle_time(jiffies_to_cputime(ticks));
3335}
3336
3337#endif
3338
Christoph Lameter7835b982006-12-10 02:20:22 -08003339/*
Balbir Singh49048622008-09-05 18:12:23 +02003340 * Use precise platform statistics if available:
3341 */
3342#ifdef CONFIG_VIRT_CPU_ACCOUNTING
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003343void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
Balbir Singh49048622008-09-05 18:12:23 +02003344{
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09003345 *ut = p->utime;
3346 *st = p->stime;
Balbir Singh49048622008-09-05 18:12:23 +02003347}
3348
Hidetoshi Seto0cf55e12009-12-02 17:28:07 +09003349void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
Balbir Singh49048622008-09-05 18:12:23 +02003350{
Hidetoshi Seto0cf55e12009-12-02 17:28:07 +09003351 struct task_cputime cputime;
3352
3353 thread_group_cputime(p, &cputime);
3354
3355 *ut = cputime.utime;
3356 *st = cputime.stime;
Balbir Singh49048622008-09-05 18:12:23 +02003357}
3358#else
Hidetoshi Seto761b1d22009-11-12 13:33:45 +09003359
3360#ifndef nsecs_to_cputime
Hidetoshi Setob7b20df2009-11-26 14:49:27 +09003361# define nsecs_to_cputime(__nsecs) nsecs_to_jiffies(__nsecs)
Hidetoshi Seto761b1d22009-11-12 13:33:45 +09003362#endif
3363
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003364void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
Balbir Singh49048622008-09-05 18:12:23 +02003365{
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09003366 cputime_t rtime, utime = p->utime, total = cputime_add(utime, p->stime);
Balbir Singh49048622008-09-05 18:12:23 +02003367
3368 /*
3369 * Use CFS's precise accounting:
3370 */
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003371 rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
Balbir Singh49048622008-09-05 18:12:23 +02003372
3373 if (total) {
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003374 u64 temp;
Balbir Singh49048622008-09-05 18:12:23 +02003375
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003376 temp = (u64)(rtime * utime);
Balbir Singh49048622008-09-05 18:12:23 +02003377 do_div(temp, total);
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003378 utime = (cputime_t)temp;
3379 } else
3380 utime = rtime;
Balbir Singh49048622008-09-05 18:12:23 +02003381
3382 /*
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003383 * Compare with previous values, to keep monotonicity:
Balbir Singh49048622008-09-05 18:12:23 +02003384 */
Hidetoshi Seto761b1d22009-11-12 13:33:45 +09003385 p->prev_utime = max(p->prev_utime, utime);
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09003386 p->prev_stime = max(p->prev_stime, cputime_sub(rtime, p->prev_utime));
Balbir Singh49048622008-09-05 18:12:23 +02003387
Hidetoshi Setod99ca3b2009-12-02 17:26:47 +09003388 *ut = p->prev_utime;
3389 *st = p->prev_stime;
Hidetoshi Setod180c5b2009-11-26 14:48:30 +09003390}
Balbir Singh49048622008-09-05 18:12:23 +02003391
Hidetoshi Seto0cf55e12009-12-02 17:28:07 +09003392/*
3393 * Must be called with siglock held.
3394 */
3395void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
3396{
3397 struct signal_struct *sig = p->signal;
3398 struct task_cputime cputime;
3399 cputime_t rtime, utime, total;
3400
3401 thread_group_cputime(p, &cputime);
3402
3403 total = cputime_add(cputime.utime, cputime.stime);
3404 rtime = nsecs_to_cputime(cputime.sum_exec_runtime);
3405
3406 if (total) {
3407 u64 temp;
3408
3409 temp = (u64)(rtime * cputime.utime);
3410 do_div(temp, total);
3411 utime = (cputime_t)temp;
3412 } else
3413 utime = rtime;
3414
3415 sig->prev_utime = max(sig->prev_utime, utime);
3416 sig->prev_stime = max(sig->prev_stime,
3417 cputime_sub(rtime, sig->prev_utime));
3418
3419 *ut = sig->prev_utime;
3420 *st = sig->prev_stime;
Balbir Singh49048622008-09-05 18:12:23 +02003421}
3422#endif
3423
Balbir Singh49048622008-09-05 18:12:23 +02003424/*
Christoph Lameter7835b982006-12-10 02:20:22 -08003425 * This function gets called by the timer code, with HZ frequency.
3426 * We call it with interrupts disabled.
3427 *
3428 * It also gets called by the fork code, when changing the parent's
3429 * timeslices.
3430 */
3431void scheduler_tick(void)
3432{
Christoph Lameter7835b982006-12-10 02:20:22 -08003433 int cpu = smp_processor_id();
3434 struct rq *rq = cpu_rq(cpu);
Ingo Molnardd41f592007-07-09 18:51:59 +02003435 struct task_struct *curr = rq->curr;
Peter Zijlstra3e51f332008-05-03 18:29:28 +02003436
3437 sched_clock_tick();
Christoph Lameter7835b982006-12-10 02:20:22 -08003438
Thomas Gleixner05fa7852009-11-17 14:28:38 +01003439 raw_spin_lock(&rq->lock);
Peter Zijlstra3e51f332008-05-03 18:29:28 +02003440 update_rq_clock(rq);
Ingo Molnarf1a438d2007-08-09 11:16:45 +02003441 update_cpu_load(rq);
Peter Zijlstrafa85ae22008-01-25 21:08:29 +01003442 curr->sched_class->task_tick(rq, curr, 0);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01003443 raw_spin_unlock(&rq->lock);
Ingo Molnardd41f592007-07-09 18:51:59 +02003444
Peter Zijlstra49f47432009-12-27 11:51:52 +01003445 perf_event_task_tick(curr);
Peter Zijlstrae220d2d2009-05-23 18:28:55 +02003446
Christoph Lametere418e1c2006-12-10 02:20:23 -08003447#ifdef CONFIG_SMP
Ingo Molnardd41f592007-07-09 18:51:59 +02003448 rq->idle_at_tick = idle_cpu(cpu);
3449 trigger_load_balance(rq, cpu);
Christoph Lametere418e1c2006-12-10 02:20:23 -08003450#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003451}
3452
Lai Jiangshan132380a2009-04-02 14:18:25 +08003453notrace unsigned long get_parent_ip(unsigned long addr)
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003454{
3455 if (in_lock_functions(addr)) {
3456 addr = CALLER_ADDR2;
3457 if (in_lock_functions(addr))
3458 addr = CALLER_ADDR3;
3459 }
3460 return addr;
3461}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003462
Steven Rostedt7e49fcc2009-01-22 19:01:40 -05003463#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
3464 defined(CONFIG_PREEMPT_TRACER))
3465
Srinivasa Ds43627582008-02-23 15:24:04 -08003466void __kprobes add_preempt_count(int val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003467{
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003468#ifdef CONFIG_DEBUG_PREEMPT
Linus Torvalds1da177e2005-04-16 15:20:36 -07003469 /*
3470 * Underflow?
3471 */
Ingo Molnar9a11b49a2006-07-03 00:24:33 -07003472 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
3473 return;
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003474#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003475 preempt_count() += val;
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003476#ifdef CONFIG_DEBUG_PREEMPT
Linus Torvalds1da177e2005-04-16 15:20:36 -07003477 /*
3478 * Spinlock count overflowing soon?
3479 */
Miguel Ojeda Sandonis33859f72006-12-10 02:20:38 -08003480 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
3481 PREEMPT_MASK - 10);
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003482#endif
3483 if (preempt_count() == val)
3484 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003485}
3486EXPORT_SYMBOL(add_preempt_count);
3487
Srinivasa Ds43627582008-02-23 15:24:04 -08003488void __kprobes sub_preempt_count(int val)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003489{
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003490#ifdef CONFIG_DEBUG_PREEMPT
Linus Torvalds1da177e2005-04-16 15:20:36 -07003491 /*
3492 * Underflow?
3493 */
Ingo Molnar01e3eb82009-01-12 13:00:50 +01003494 if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
Ingo Molnar9a11b49a2006-07-03 00:24:33 -07003495 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003496 /*
3497 * Is the spinlock portion underflowing?
3498 */
Ingo Molnar9a11b49a2006-07-03 00:24:33 -07003499 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
3500 !(preempt_count() & PREEMPT_MASK)))
3501 return;
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003502#endif
Ingo Molnar9a11b49a2006-07-03 00:24:33 -07003503
Steven Rostedt6cd8a4b2008-05-12 21:20:42 +02003504 if (preempt_count() == val)
3505 trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003506 preempt_count() -= val;
3507}
3508EXPORT_SYMBOL(sub_preempt_count);
3509
3510#endif
3511
3512/*
Ingo Molnardd41f592007-07-09 18:51:59 +02003513 * Print scheduling while atomic bug:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003514 */
Ingo Molnardd41f592007-07-09 18:51:59 +02003515static noinline void __schedule_bug(struct task_struct *prev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003516{
Satyam Sharma838225b2007-10-24 18:23:50 +02003517 struct pt_regs *regs = get_irq_regs();
3518
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01003519 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
3520 prev->comm, prev->pid, preempt_count());
Satyam Sharma838225b2007-10-24 18:23:50 +02003521
Ingo Molnardd41f592007-07-09 18:51:59 +02003522 debug_show_held_locks(prev);
Arjan van de Vene21f5b12008-05-23 09:05:58 -07003523 print_modules();
Ingo Molnardd41f592007-07-09 18:51:59 +02003524 if (irqs_disabled())
3525 print_irqtrace_events(prev);
Satyam Sharma838225b2007-10-24 18:23:50 +02003526
3527 if (regs)
3528 show_regs(regs);
3529 else
3530 dump_stack();
Ingo Molnardd41f592007-07-09 18:51:59 +02003531}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003532
Ingo Molnardd41f592007-07-09 18:51:59 +02003533/*
3534 * Various schedule()-time debugging checks and statistics:
3535 */
3536static inline void schedule_debug(struct task_struct *prev)
3537{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003538 /*
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01003539 * Test if we are atomic. Since do_exit() needs to call into
Linus Torvalds1da177e2005-04-16 15:20:36 -07003540 * schedule() atomically, we ignore that path for now.
3541 * Otherwise, whine if we are scheduling when we should not be.
3542 */
Roel Kluin3f33a7c2008-05-13 23:44:11 +02003543 if (unlikely(in_atomic_preempt_off() && !prev->exit_state))
Ingo Molnardd41f592007-07-09 18:51:59 +02003544 __schedule_bug(prev);
3545
Linus Torvalds1da177e2005-04-16 15:20:36 -07003546 profile_hit(SCHED_PROFILING, __builtin_return_address(0));
3547
Ingo Molnar2d723762007-10-15 17:00:12 +02003548 schedstat_inc(this_rq(), sched_count);
Ingo Molnarb8efb562007-10-15 17:00:10 +02003549#ifdef CONFIG_SCHEDSTATS
3550 if (unlikely(prev->lock_depth >= 0)) {
Ingo Molnar2d723762007-10-15 17:00:12 +02003551 schedstat_inc(this_rq(), bkl_count);
3552 schedstat_inc(prev, sched_info.bkl_count);
Ingo Molnarb8efb562007-10-15 17:00:10 +02003553 }
3554#endif
Ingo Molnardd41f592007-07-09 18:51:59 +02003555}
3556
Peter Zijlstra6cecd082009-11-30 13:00:37 +01003557static void put_prev_task(struct rq *rq, struct task_struct *prev)
Mike Galbraithdf1c99d2009-03-10 19:08:11 +01003558{
Mike Galbraitha64692a2010-03-11 17:16:20 +01003559 if (prev->se.on_rq)
3560 update_rq_clock(rq);
3561 rq->skip_clock_update = 0;
Peter Zijlstra6cecd082009-11-30 13:00:37 +01003562 prev->sched_class->put_prev_task(rq, prev);
Mike Galbraithdf1c99d2009-03-10 19:08:11 +01003563}
3564
Ingo Molnardd41f592007-07-09 18:51:59 +02003565/*
3566 * Pick up the highest-prio task:
3567 */
3568static inline struct task_struct *
Wang Chenb67802e2009-03-02 13:55:26 +08003569pick_next_task(struct rq *rq)
Ingo Molnardd41f592007-07-09 18:51:59 +02003570{
Ingo Molnar5522d5d2007-10-15 17:00:12 +02003571 const struct sched_class *class;
Ingo Molnardd41f592007-07-09 18:51:59 +02003572 struct task_struct *p;
3573
3574 /*
3575 * Optimization: we know that if all tasks are in
3576 * the fair class we can call that function directly:
3577 */
3578 if (likely(rq->nr_running == rq->cfs.nr_running)) {
Ingo Molnarfb8d4722007-08-09 11:16:48 +02003579 p = fair_sched_class.pick_next_task(rq);
Ingo Molnardd41f592007-07-09 18:51:59 +02003580 if (likely(p))
3581 return p;
3582 }
3583
3584 class = sched_class_highest;
3585 for ( ; ; ) {
Ingo Molnarfb8d4722007-08-09 11:16:48 +02003586 p = class->pick_next_task(rq);
Ingo Molnardd41f592007-07-09 18:51:59 +02003587 if (p)
3588 return p;
3589 /*
3590 * Will never be NULL as the idle class always
3591 * returns a non-NULL p:
3592 */
3593 class = class->next;
3594 }
3595}
3596
3597/*
3598 * schedule() is the main scheduler function.
3599 */
Peter Zijlstraff743342009-03-13 12:21:26 +01003600asmlinkage void __sched schedule(void)
Ingo Molnardd41f592007-07-09 18:51:59 +02003601{
3602 struct task_struct *prev, *next;
Harvey Harrison67ca7bd2008-02-15 09:56:36 -08003603 unsigned long *switch_count;
Ingo Molnardd41f592007-07-09 18:51:59 +02003604 struct rq *rq;
Peter Zijlstra31656512008-07-18 18:01:23 +02003605 int cpu;
Ingo Molnardd41f592007-07-09 18:51:59 +02003606
Peter Zijlstraff743342009-03-13 12:21:26 +01003607need_resched:
3608 preempt_disable();
Ingo Molnardd41f592007-07-09 18:51:59 +02003609 cpu = smp_processor_id();
3610 rq = cpu_rq(cpu);
Paul E. McKenney25502a62010-04-01 17:37:01 -07003611 rcu_note_context_switch(cpu);
Ingo Molnardd41f592007-07-09 18:51:59 +02003612 prev = rq->curr;
3613 switch_count = &prev->nivcsw;
3614
Linus Torvalds1da177e2005-04-16 15:20:36 -07003615 release_kernel_lock(prev);
3616need_resched_nonpreemptible:
Linus Torvalds1da177e2005-04-16 15:20:36 -07003617
Ingo Molnardd41f592007-07-09 18:51:59 +02003618 schedule_debug(prev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003619
Peter Zijlstra31656512008-07-18 18:01:23 +02003620 if (sched_feat(HRTICK))
Mike Galbraithf333fdc2008-05-12 21:20:55 +02003621 hrtick_clear(rq);
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003622
Thomas Gleixner05fa7852009-11-17 14:28:38 +01003623 raw_spin_lock_irq(&rq->lock);
Ingo Molnar1e819952007-10-15 17:00:13 +02003624 clear_tsk_need_resched(prev);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003625
Ingo Molnardd41f592007-07-09 18:51:59 +02003626 if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
Oleg Nesterov16882c12008-06-08 21:20:41 +04003627 if (unlikely(signal_pending_state(prev->state, prev)))
Ingo Molnardd41f592007-07-09 18:51:59 +02003628 prev->state = TASK_RUNNING;
Oleg Nesterov16882c12008-06-08 21:20:41 +04003629 else
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01003630 deactivate_task(rq, prev, DEQUEUE_SLEEP);
Ingo Molnardd41f592007-07-09 18:51:59 +02003631 switch_count = &prev->nvcsw;
3632 }
3633
Gregory Haskins3f029d32009-07-29 11:08:47 -04003634 pre_schedule(rq, prev);
Steven Rostedtf65eda42008-01-25 21:08:07 +01003635
Ingo Molnardd41f592007-07-09 18:51:59 +02003636 if (unlikely(!rq->nr_running))
3637 idle_balance(cpu, rq);
3638
Mike Galbraithdf1c99d2009-03-10 19:08:11 +01003639 put_prev_task(rq, prev);
Wang Chenb67802e2009-03-02 13:55:26 +08003640 next = pick_next_task(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003641
Linus Torvalds1da177e2005-04-16 15:20:36 -07003642 if (likely(prev != next)) {
David Simner673a90a2008-04-29 10:08:59 +01003643 sched_info_switch(prev, next);
Peter Zijlstra49f47432009-12-27 11:51:52 +01003644 perf_event_task_sched_out(prev, next);
David Simner673a90a2008-04-29 10:08:59 +01003645
Linus Torvalds1da177e2005-04-16 15:20:36 -07003646 rq->nr_switches++;
3647 rq->curr = next;
3648 ++*switch_count;
3649
Ingo Molnardd41f592007-07-09 18:51:59 +02003650 context_switch(rq, prev, next); /* unlocks the rq */
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003651 /*
3652 * the context switch might have flipped the stack from under
3653 * us, hence refresh the local variables.
3654 */
3655 cpu = smp_processor_id();
3656 rq = cpu_rq(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003657 } else
Thomas Gleixner05fa7852009-11-17 14:28:38 +01003658 raw_spin_unlock_irq(&rq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003659
Gregory Haskins3f029d32009-07-29 11:08:47 -04003660 post_schedule(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003661
Yong Zhang6d558c32010-01-11 14:21:25 +08003662 if (unlikely(reacquire_kernel_lock(current) < 0)) {
3663 prev = rq->curr;
3664 switch_count = &prev->nivcsw;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003665 goto need_resched_nonpreemptible;
Yong Zhang6d558c32010-01-11 14:21:25 +08003666 }
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01003667
Linus Torvalds1da177e2005-04-16 15:20:36 -07003668 preempt_enable_no_resched();
Peter Zijlstraff743342009-03-13 12:21:26 +01003669 if (need_resched())
Linus Torvalds1da177e2005-04-16 15:20:36 -07003670 goto need_resched;
3671}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003672EXPORT_SYMBOL(schedule);
3673
Frederic Weisbeckerc08f7822009-12-02 20:49:17 +01003674#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
Peter Zijlstra0d66bf62009-01-12 14:01:47 +01003675/*
3676 * Look out! "owner" is an entirely speculative pointer
3677 * access and not reliable.
3678 */
3679int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner)
3680{
3681 unsigned int cpu;
3682 struct rq *rq;
3683
3684 if (!sched_feat(OWNER_SPIN))
3685 return 0;
3686
3687#ifdef CONFIG_DEBUG_PAGEALLOC
3688 /*
3689 * Need to access the cpu field knowing that
3690 * DEBUG_PAGEALLOC could have unmapped it if
3691 * the mutex owner just released it and exited.
3692 */
3693 if (probe_kernel_address(&owner->cpu, cpu))
Benjamin Herrenschmidt4b402212010-04-16 23:20:00 +02003694 return 0;
Peter Zijlstra0d66bf62009-01-12 14:01:47 +01003695#else
3696 cpu = owner->cpu;
3697#endif
3698
3699 /*
3700 * Even if the access succeeded (likely case),
3701 * the cpu field may no longer be valid.
3702 */
3703 if (cpu >= nr_cpumask_bits)
Benjamin Herrenschmidt4b402212010-04-16 23:20:00 +02003704 return 0;
Peter Zijlstra0d66bf62009-01-12 14:01:47 +01003705
3706 /*
3707 * We need to validate that we can do a
3708 * get_cpu() and that we have the percpu area.
3709 */
3710 if (!cpu_online(cpu))
Benjamin Herrenschmidt4b402212010-04-16 23:20:00 +02003711 return 0;
Peter Zijlstra0d66bf62009-01-12 14:01:47 +01003712
3713 rq = cpu_rq(cpu);
3714
3715 for (;;) {
3716 /*
3717 * Owner changed, break to re-assess state.
3718 */
3719 if (lock->owner != owner)
3720 break;
3721
3722 /*
3723 * Is that owner really running on that cpu?
3724 */
3725 if (task_thread_info(rq->curr) != owner || need_resched())
3726 return 0;
3727
3728 cpu_relax();
3729 }
Benjamin Herrenschmidt4b402212010-04-16 23:20:00 +02003730
Peter Zijlstra0d66bf62009-01-12 14:01:47 +01003731 return 1;
3732}
3733#endif
3734
Linus Torvalds1da177e2005-04-16 15:20:36 -07003735#ifdef CONFIG_PREEMPT
3736/*
Andreas Mohr2ed6e342006-07-10 04:43:52 -07003737 * this is the entry point to schedule() from in-kernel preemption
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01003738 * off of preempt_enable. Kernel preemptions off return from interrupt
Linus Torvalds1da177e2005-04-16 15:20:36 -07003739 * occur there and call schedule directly.
3740 */
3741asmlinkage void __sched preempt_schedule(void)
3742{
3743 struct thread_info *ti = current_thread_info();
Ingo Molnar6478d882008-01-25 21:08:33 +01003744
Linus Torvalds1da177e2005-04-16 15:20:36 -07003745 /*
3746 * If there is a non-zero preempt_count or interrupts are disabled,
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01003747 * we do not want to preempt the current task. Just return..
Linus Torvalds1da177e2005-04-16 15:20:36 -07003748 */
Nick Pigginbeed33a2006-10-11 01:21:52 -07003749 if (likely(ti->preempt_count || irqs_disabled()))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003750 return;
3751
Andi Kleen3a5c3592007-10-15 17:00:14 +02003752 do {
3753 add_preempt_count(PREEMPT_ACTIVE);
Andi Kleen3a5c3592007-10-15 17:00:14 +02003754 schedule();
Andi Kleen3a5c3592007-10-15 17:00:14 +02003755 sub_preempt_count(PREEMPT_ACTIVE);
3756
3757 /*
3758 * Check again in case we missed a preemption opportunity
3759 * between schedule and now.
3760 */
3761 barrier();
Lai Jiangshan5ed0cec2009-03-06 19:40:20 +08003762 } while (need_resched());
Linus Torvalds1da177e2005-04-16 15:20:36 -07003763}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003764EXPORT_SYMBOL(preempt_schedule);
3765
3766/*
Andreas Mohr2ed6e342006-07-10 04:43:52 -07003767 * this is the entry point to schedule() from kernel preemption
Linus Torvalds1da177e2005-04-16 15:20:36 -07003768 * off of irq context.
3769 * Note, that this is called and return with irqs disabled. This will
3770 * protect us against recursive calling from irq.
3771 */
3772asmlinkage void __sched preempt_schedule_irq(void)
3773{
3774 struct thread_info *ti = current_thread_info();
Ingo Molnar6478d882008-01-25 21:08:33 +01003775
Andreas Mohr2ed6e342006-07-10 04:43:52 -07003776 /* Catch callers which need to be fixed */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003777 BUG_ON(ti->preempt_count || !irqs_disabled());
3778
Andi Kleen3a5c3592007-10-15 17:00:14 +02003779 do {
3780 add_preempt_count(PREEMPT_ACTIVE);
Andi Kleen3a5c3592007-10-15 17:00:14 +02003781 local_irq_enable();
3782 schedule();
3783 local_irq_disable();
Andi Kleen3a5c3592007-10-15 17:00:14 +02003784 sub_preempt_count(PREEMPT_ACTIVE);
3785
3786 /*
3787 * Check again in case we missed a preemption opportunity
3788 * between schedule and now.
3789 */
3790 barrier();
Lai Jiangshan5ed0cec2009-03-06 19:40:20 +08003791 } while (need_resched());
Linus Torvalds1da177e2005-04-16 15:20:36 -07003792}
3793
3794#endif /* CONFIG_PREEMPT */
3795
Peter Zijlstra63859d42009-09-15 19:14:42 +02003796int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
Ingo Molnar95cdf3b2005-09-10 00:26:11 -07003797 void *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003798{
Peter Zijlstra63859d42009-09-15 19:14:42 +02003799 return try_to_wake_up(curr->private, mode, wake_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003800}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003801EXPORT_SYMBOL(default_wake_function);
3802
3803/*
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01003804 * The core wakeup function. Non-exclusive wakeups (nr_exclusive == 0) just
3805 * wake everything up. If it's an exclusive wakeup (nr_exclusive == small +ve
Linus Torvalds1da177e2005-04-16 15:20:36 -07003806 * number) then we wake all the non-exclusive tasks and one exclusive task.
3807 *
3808 * There are circumstances in which we can try to wake a task which has already
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01003809 * started to run but is not in state TASK_RUNNING. try_to_wake_up() returns
Linus Torvalds1da177e2005-04-16 15:20:36 -07003810 * zero in this (rare) case, and we handle it by continuing to scan the queue.
3811 */
Johannes Weiner78ddb082009-04-14 16:53:05 +02003812static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
Peter Zijlstra63859d42009-09-15 19:14:42 +02003813 int nr_exclusive, int wake_flags, void *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003814{
Matthias Kaehlcke2e458742007-10-15 17:00:02 +02003815 wait_queue_t *curr, *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003816
Matthias Kaehlcke2e458742007-10-15 17:00:02 +02003817 list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
Ingo Molnar48f24c42006-07-03 00:25:40 -07003818 unsigned flags = curr->flags;
3819
Peter Zijlstra63859d42009-09-15 19:14:42 +02003820 if (curr->func(curr, mode, wake_flags, key) &&
Ingo Molnar48f24c42006-07-03 00:25:40 -07003821 (flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003822 break;
3823 }
3824}
3825
3826/**
3827 * __wake_up - wake up threads blocked on a waitqueue.
3828 * @q: the waitqueue
3829 * @mode: which threads
3830 * @nr_exclusive: how many wake-one or wake-many threads to wake up
Martin Waitz67be2dd2005-05-01 08:59:26 -07003831 * @key: is directly passed to the wakeup function
David Howells50fa6102009-04-28 15:01:38 +01003832 *
3833 * It may be assumed that this function implies a write memory barrier before
3834 * changing the task state if and only if any tasks are woken up.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003835 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08003836void __wake_up(wait_queue_head_t *q, unsigned int mode,
Ingo Molnar95cdf3b2005-09-10 00:26:11 -07003837 int nr_exclusive, void *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003838{
3839 unsigned long flags;
3840
3841 spin_lock_irqsave(&q->lock, flags);
3842 __wake_up_common(q, mode, nr_exclusive, 0, key);
3843 spin_unlock_irqrestore(&q->lock, flags);
3844}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003845EXPORT_SYMBOL(__wake_up);
3846
3847/*
3848 * Same as __wake_up but called with the spinlock in wait_queue_head_t held.
3849 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -08003850void __wake_up_locked(wait_queue_head_t *q, unsigned int mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003851{
3852 __wake_up_common(q, mode, 1, 0, NULL);
3853}
Michal Nazarewicz22c43c82010-05-05 12:53:11 +02003854EXPORT_SYMBOL_GPL(__wake_up_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003855
Davide Libenzi4ede8162009-03-31 15:24:20 -07003856void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key)
3857{
3858 __wake_up_common(q, mode, 1, 0, key);
3859}
3860
Linus Torvalds1da177e2005-04-16 15:20:36 -07003861/**
Davide Libenzi4ede8162009-03-31 15:24:20 -07003862 * __wake_up_sync_key - wake up threads blocked on a waitqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003863 * @q: the waitqueue
3864 * @mode: which threads
3865 * @nr_exclusive: how many wake-one or wake-many threads to wake up
Davide Libenzi4ede8162009-03-31 15:24:20 -07003866 * @key: opaque value to be passed to wakeup targets
Linus Torvalds1da177e2005-04-16 15:20:36 -07003867 *
3868 * The sync wakeup differs that the waker knows that it will schedule
3869 * away soon, so while the target thread will be woken up, it will not
3870 * be migrated to another CPU - ie. the two threads are 'synchronized'
3871 * with each other. This can prevent needless bouncing between CPUs.
3872 *
3873 * On UP it can prevent extra preemption.
David Howells50fa6102009-04-28 15:01:38 +01003874 *
3875 * It may be assumed that this function implies a write memory barrier before
3876 * changing the task state if and only if any tasks are woken up.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003877 */
Davide Libenzi4ede8162009-03-31 15:24:20 -07003878void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode,
3879 int nr_exclusive, void *key)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003880{
3881 unsigned long flags;
Peter Zijlstra7d478722009-09-14 19:55:44 +02003882 int wake_flags = WF_SYNC;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003883
3884 if (unlikely(!q))
3885 return;
3886
3887 if (unlikely(!nr_exclusive))
Peter Zijlstra7d478722009-09-14 19:55:44 +02003888 wake_flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003889
3890 spin_lock_irqsave(&q->lock, flags);
Peter Zijlstra7d478722009-09-14 19:55:44 +02003891 __wake_up_common(q, mode, nr_exclusive, wake_flags, key);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003892 spin_unlock_irqrestore(&q->lock, flags);
3893}
Davide Libenzi4ede8162009-03-31 15:24:20 -07003894EXPORT_SYMBOL_GPL(__wake_up_sync_key);
3895
3896/*
3897 * __wake_up_sync - see __wake_up_sync_key()
3898 */
3899void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr_exclusive)
3900{
3901 __wake_up_sync_key(q, mode, nr_exclusive, NULL);
3902}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003903EXPORT_SYMBOL_GPL(__wake_up_sync); /* For internal use only */
3904
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02003905/**
3906 * complete: - signals a single thread waiting on this completion
3907 * @x: holds the state of this particular completion
3908 *
3909 * This will wake up a single thread waiting on this completion. Threads will be
3910 * awakened in the same order in which they were queued.
3911 *
3912 * See also complete_all(), wait_for_completion() and related routines.
David Howells50fa6102009-04-28 15:01:38 +01003913 *
3914 * It may be assumed that this function implies a write memory barrier before
3915 * changing the task state if and only if any tasks are woken up.
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02003916 */
Ingo Molnarb15136e2007-10-24 18:23:48 +02003917void complete(struct completion *x)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003918{
3919 unsigned long flags;
3920
3921 spin_lock_irqsave(&x->wait.lock, flags);
3922 x->done++;
Matthew Wilcoxd9514f62007-12-06 11:07:07 -05003923 __wake_up_common(&x->wait, TASK_NORMAL, 1, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003924 spin_unlock_irqrestore(&x->wait.lock, flags);
3925}
3926EXPORT_SYMBOL(complete);
3927
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02003928/**
3929 * complete_all: - signals all threads waiting on this completion
3930 * @x: holds the state of this particular completion
3931 *
3932 * This will wake up all threads waiting on this particular completion event.
David Howells50fa6102009-04-28 15:01:38 +01003933 *
3934 * It may be assumed that this function implies a write memory barrier before
3935 * changing the task state if and only if any tasks are woken up.
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02003936 */
Ingo Molnarb15136e2007-10-24 18:23:48 +02003937void complete_all(struct completion *x)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003938{
3939 unsigned long flags;
3940
3941 spin_lock_irqsave(&x->wait.lock, flags);
3942 x->done += UINT_MAX/2;
Matthew Wilcoxd9514f62007-12-06 11:07:07 -05003943 __wake_up_common(&x->wait, TASK_NORMAL, 0, 0, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003944 spin_unlock_irqrestore(&x->wait.lock, flags);
3945}
3946EXPORT_SYMBOL(complete_all);
3947
Andi Kleen8cbbe862007-10-15 17:00:14 +02003948static inline long __sched
3949do_wait_for_common(struct completion *x, long timeout, int state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003950{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003951 if (!x->done) {
3952 DECLARE_WAITQUEUE(wait, current);
3953
Changli Gaoa93d2f12010-05-07 14:33:26 +08003954 __add_wait_queue_tail_exclusive(&x->wait, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003955 do {
Oleg Nesterov94d3d822008-08-20 16:54:41 -07003956 if (signal_pending_state(state, current)) {
Oleg Nesterovea71a542008-06-20 18:32:20 +04003957 timeout = -ERESTARTSYS;
3958 break;
Andi Kleen8cbbe862007-10-15 17:00:14 +02003959 }
3960 __set_current_state(state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003961 spin_unlock_irq(&x->wait.lock);
Andi Kleen8cbbe862007-10-15 17:00:14 +02003962 timeout = schedule_timeout(timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003963 spin_lock_irq(&x->wait.lock);
Oleg Nesterovea71a542008-06-20 18:32:20 +04003964 } while (!x->done && timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003965 __remove_wait_queue(&x->wait, &wait);
Oleg Nesterovea71a542008-06-20 18:32:20 +04003966 if (!x->done)
3967 return timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003968 }
3969 x->done--;
Oleg Nesterovea71a542008-06-20 18:32:20 +04003970 return timeout ?: 1;
Andi Kleen8cbbe862007-10-15 17:00:14 +02003971}
3972
3973static long __sched
3974wait_for_common(struct completion *x, long timeout, int state)
3975{
3976 might_sleep();
3977
3978 spin_lock_irq(&x->wait.lock);
3979 timeout = do_wait_for_common(x, timeout, state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003980 spin_unlock_irq(&x->wait.lock);
Andi Kleen8cbbe862007-10-15 17:00:14 +02003981 return timeout;
3982}
3983
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02003984/**
3985 * wait_for_completion: - waits for completion of a task
3986 * @x: holds the state of this particular completion
3987 *
3988 * This waits to be signaled for completion of a specific task. It is NOT
3989 * interruptible and there is no timeout.
3990 *
3991 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
3992 * and interrupt capability. Also see complete().
3993 */
Ingo Molnarb15136e2007-10-24 18:23:48 +02003994void __sched wait_for_completion(struct completion *x)
Andi Kleen8cbbe862007-10-15 17:00:14 +02003995{
3996 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003997}
3998EXPORT_SYMBOL(wait_for_completion);
3999
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02004000/**
4001 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
4002 * @x: holds the state of this particular completion
4003 * @timeout: timeout value in jiffies
4004 *
4005 * This waits for either a completion of a specific task to be signaled or for a
4006 * specified timeout to expire. The timeout is in jiffies. It is not
4007 * interruptible.
4008 */
Ingo Molnarb15136e2007-10-24 18:23:48 +02004009unsigned long __sched
Linus Torvalds1da177e2005-04-16 15:20:36 -07004010wait_for_completion_timeout(struct completion *x, unsigned long timeout)
4011{
Andi Kleen8cbbe862007-10-15 17:00:14 +02004012 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004013}
4014EXPORT_SYMBOL(wait_for_completion_timeout);
4015
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02004016/**
4017 * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
4018 * @x: holds the state of this particular completion
4019 *
4020 * This waits for completion of a specific task to be signaled. It is
4021 * interruptible.
4022 */
Andi Kleen8cbbe862007-10-15 17:00:14 +02004023int __sched wait_for_completion_interruptible(struct completion *x)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004024{
Andi Kleen51e97992007-10-18 21:32:55 +02004025 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
4026 if (t == -ERESTARTSYS)
4027 return t;
4028 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004029}
4030EXPORT_SYMBOL(wait_for_completion_interruptible);
4031
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02004032/**
4033 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
4034 * @x: holds the state of this particular completion
4035 * @timeout: timeout value in jiffies
4036 *
4037 * This waits for either a completion of a specific task to be signaled or for a
4038 * specified timeout to expire. It is interruptible. The timeout is in jiffies.
4039 */
Ingo Molnarb15136e2007-10-24 18:23:48 +02004040unsigned long __sched
Linus Torvalds1da177e2005-04-16 15:20:36 -07004041wait_for_completion_interruptible_timeout(struct completion *x,
4042 unsigned long timeout)
4043{
Andi Kleen8cbbe862007-10-15 17:00:14 +02004044 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004045}
4046EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
4047
Kevin Diggs65eb3dc2008-08-26 10:26:54 +02004048/**
4049 * wait_for_completion_killable: - waits for completion of a task (killable)
4050 * @x: holds the state of this particular completion
4051 *
4052 * This waits to be signaled for completion of a specific task. It can be
4053 * interrupted by a kill signal.
4054 */
Matthew Wilcox009e5772007-12-06 12:29:54 -05004055int __sched wait_for_completion_killable(struct completion *x)
4056{
4057 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
4058 if (t == -ERESTARTSYS)
4059 return t;
4060 return 0;
4061}
4062EXPORT_SYMBOL(wait_for_completion_killable);
4063
Dave Chinnerbe4de352008-08-15 00:40:44 -07004064/**
4065 * try_wait_for_completion - try to decrement a completion without blocking
4066 * @x: completion structure
4067 *
4068 * Returns: 0 if a decrement cannot be done without blocking
4069 * 1 if a decrement succeeded.
4070 *
4071 * If a completion is being used as a counting completion,
4072 * attempt to decrement the counter without blocking. This
4073 * enables us to avoid waiting if the resource the completion
4074 * is protecting is not available.
4075 */
4076bool try_wait_for_completion(struct completion *x)
4077{
Rafael J. Wysocki7539a3b2009-12-13 00:07:30 +01004078 unsigned long flags;
Dave Chinnerbe4de352008-08-15 00:40:44 -07004079 int ret = 1;
4080
Rafael J. Wysocki7539a3b2009-12-13 00:07:30 +01004081 spin_lock_irqsave(&x->wait.lock, flags);
Dave Chinnerbe4de352008-08-15 00:40:44 -07004082 if (!x->done)
4083 ret = 0;
4084 else
4085 x->done--;
Rafael J. Wysocki7539a3b2009-12-13 00:07:30 +01004086 spin_unlock_irqrestore(&x->wait.lock, flags);
Dave Chinnerbe4de352008-08-15 00:40:44 -07004087 return ret;
4088}
4089EXPORT_SYMBOL(try_wait_for_completion);
4090
4091/**
4092 * completion_done - Test to see if a completion has any waiters
4093 * @x: completion structure
4094 *
4095 * Returns: 0 if there are waiters (wait_for_completion() in progress)
4096 * 1 if there are no waiters.
4097 *
4098 */
4099bool completion_done(struct completion *x)
4100{
Rafael J. Wysocki7539a3b2009-12-13 00:07:30 +01004101 unsigned long flags;
Dave Chinnerbe4de352008-08-15 00:40:44 -07004102 int ret = 1;
4103
Rafael J. Wysocki7539a3b2009-12-13 00:07:30 +01004104 spin_lock_irqsave(&x->wait.lock, flags);
Dave Chinnerbe4de352008-08-15 00:40:44 -07004105 if (!x->done)
4106 ret = 0;
Rafael J. Wysocki7539a3b2009-12-13 00:07:30 +01004107 spin_unlock_irqrestore(&x->wait.lock, flags);
Dave Chinnerbe4de352008-08-15 00:40:44 -07004108 return ret;
4109}
4110EXPORT_SYMBOL(completion_done);
4111
Andi Kleen8cbbe862007-10-15 17:00:14 +02004112static long __sched
4113sleep_on_common(wait_queue_head_t *q, int state, long timeout)
Ingo Molnar0fec1712007-07-09 18:52:01 +02004114{
4115 unsigned long flags;
4116 wait_queue_t wait;
4117
4118 init_waitqueue_entry(&wait, current);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004119
Andi Kleen8cbbe862007-10-15 17:00:14 +02004120 __set_current_state(state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004121
Andi Kleen8cbbe862007-10-15 17:00:14 +02004122 spin_lock_irqsave(&q->lock, flags);
4123 __add_wait_queue(q, &wait);
4124 spin_unlock(&q->lock);
4125 timeout = schedule_timeout(timeout);
4126 spin_lock_irq(&q->lock);
4127 __remove_wait_queue(q, &wait);
4128 spin_unlock_irqrestore(&q->lock, flags);
4129
4130 return timeout;
4131}
4132
4133void __sched interruptible_sleep_on(wait_queue_head_t *q)
4134{
4135 sleep_on_common(q, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004136}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004137EXPORT_SYMBOL(interruptible_sleep_on);
4138
Ingo Molnar0fec1712007-07-09 18:52:01 +02004139long __sched
Ingo Molnar95cdf3b2005-09-10 00:26:11 -07004140interruptible_sleep_on_timeout(wait_queue_head_t *q, long timeout)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004141{
Andi Kleen8cbbe862007-10-15 17:00:14 +02004142 return sleep_on_common(q, TASK_INTERRUPTIBLE, timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004143}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004144EXPORT_SYMBOL(interruptible_sleep_on_timeout);
4145
Ingo Molnar0fec1712007-07-09 18:52:01 +02004146void __sched sleep_on(wait_queue_head_t *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004147{
Andi Kleen8cbbe862007-10-15 17:00:14 +02004148 sleep_on_common(q, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004149}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004150EXPORT_SYMBOL(sleep_on);
4151
Ingo Molnar0fec1712007-07-09 18:52:01 +02004152long __sched sleep_on_timeout(wait_queue_head_t *q, long timeout)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004153{
Andi Kleen8cbbe862007-10-15 17:00:14 +02004154 return sleep_on_common(q, TASK_UNINTERRUPTIBLE, timeout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004155}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004156EXPORT_SYMBOL(sleep_on_timeout);
4157
Ingo Molnarb29739f2006-06-27 02:54:51 -07004158#ifdef CONFIG_RT_MUTEXES
4159
4160/*
4161 * rt_mutex_setprio - set the current priority of a task
4162 * @p: task
4163 * @prio: prio value (kernel-internal form)
4164 *
4165 * This function changes the 'effective' priority of a task. It does
4166 * not touch ->normal_prio like __setscheduler().
4167 *
4168 * Used by the rt_mutex code to implement priority inheritance logic.
4169 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07004170void rt_mutex_setprio(struct task_struct *p, int prio)
Ingo Molnarb29739f2006-06-27 02:54:51 -07004171{
4172 unsigned long flags;
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02004173 int oldprio, on_rq, running;
Ingo Molnar70b97a72006-07-03 00:25:42 -07004174 struct rq *rq;
Thomas Gleixner83ab0aa2010-02-17 09:05:48 +01004175 const struct sched_class *prev_class;
Ingo Molnarb29739f2006-06-27 02:54:51 -07004176
4177 BUG_ON(prio < 0 || prio > MAX_PRIO);
4178
4179 rq = task_rq_lock(p, &flags);
4180
Andrew Mortond5f9f942007-05-08 20:27:06 -07004181 oldprio = p->prio;
Thomas Gleixner83ab0aa2010-02-17 09:05:48 +01004182 prev_class = p->sched_class;
Ingo Molnardd41f592007-07-09 18:51:59 +02004183 on_rq = p->se.on_rq;
Dmitry Adamushko051a1d12007-12-18 15:21:13 +01004184 running = task_current(rq, p);
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07004185 if (on_rq)
Ingo Molnar69be72c2007-08-09 11:16:49 +02004186 dequeue_task(rq, p, 0);
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07004187 if (running)
4188 p->sched_class->put_prev_task(rq, p);
Ingo Molnardd41f592007-07-09 18:51:59 +02004189
4190 if (rt_prio(prio))
4191 p->sched_class = &rt_sched_class;
4192 else
4193 p->sched_class = &fair_sched_class;
4194
Ingo Molnarb29739f2006-06-27 02:54:51 -07004195 p->prio = prio;
4196
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07004197 if (running)
4198 p->sched_class->set_curr_task(rq);
Ingo Molnardd41f592007-07-09 18:51:59 +02004199 if (on_rq) {
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01004200 enqueue_task(rq, p, oldprio < prio ? ENQUEUE_HEAD : 0);
Steven Rostedtcb469842008-01-25 21:08:22 +01004201
4202 check_class_changed(rq, p, prev_class, oldprio, running);
Ingo Molnarb29739f2006-06-27 02:54:51 -07004203 }
4204 task_rq_unlock(rq, &flags);
4205}
4206
4207#endif
4208
Ingo Molnar36c8b582006-07-03 00:25:41 -07004209void set_user_nice(struct task_struct *p, long nice)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004210{
Ingo Molnardd41f592007-07-09 18:51:59 +02004211 int old_prio, delta, on_rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004212 unsigned long flags;
Ingo Molnar70b97a72006-07-03 00:25:42 -07004213 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004214
4215 if (TASK_NICE(p) == nice || nice < -20 || nice > 19)
4216 return;
4217 /*
4218 * We have to be careful, if called from sys_setpriority(),
4219 * the task might be in the middle of scheduling on another CPU.
4220 */
4221 rq = task_rq_lock(p, &flags);
4222 /*
4223 * The RT priorities are set via sched_setscheduler(), but we still
4224 * allow the 'normal' nice value to be set - but as expected
4225 * it wont have any effect on scheduling until the task is
Ingo Molnardd41f592007-07-09 18:51:59 +02004226 * SCHED_FIFO/SCHED_RR:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004227 */
Ingo Molnare05606d2007-07-09 18:51:59 +02004228 if (task_has_rt_policy(p)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004229 p->static_prio = NICE_TO_PRIO(nice);
4230 goto out_unlock;
4231 }
Ingo Molnardd41f592007-07-09 18:51:59 +02004232 on_rq = p->se.on_rq;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02004233 if (on_rq)
Ingo Molnar69be72c2007-08-09 11:16:49 +02004234 dequeue_task(rq, p, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004235
Linus Torvalds1da177e2005-04-16 15:20:36 -07004236 p->static_prio = NICE_TO_PRIO(nice);
Peter Williams2dd73a42006-06-27 02:54:34 -07004237 set_load_weight(p);
Ingo Molnarb29739f2006-06-27 02:54:51 -07004238 old_prio = p->prio;
4239 p->prio = effective_prio(p);
4240 delta = p->prio - old_prio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004241
Ingo Molnardd41f592007-07-09 18:51:59 +02004242 if (on_rq) {
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01004243 enqueue_task(rq, p, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004244 /*
Andrew Mortond5f9f942007-05-08 20:27:06 -07004245 * If the task increased its priority or is running and
4246 * lowered its priority, then reschedule its CPU:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004247 */
Andrew Mortond5f9f942007-05-08 20:27:06 -07004248 if (delta < 0 || (delta > 0 && task_running(rq, p)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004249 resched_task(rq->curr);
4250 }
4251out_unlock:
4252 task_rq_unlock(rq, &flags);
4253}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004254EXPORT_SYMBOL(set_user_nice);
4255
Matt Mackalle43379f2005-05-01 08:59:00 -07004256/*
4257 * can_nice - check if a task can reduce its nice value
4258 * @p: task
4259 * @nice: nice value
4260 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07004261int can_nice(const struct task_struct *p, const int nice)
Matt Mackalle43379f2005-05-01 08:59:00 -07004262{
Matt Mackall024f4742005-08-18 11:24:19 -07004263 /* convert nice value [19,-20] to rlimit style value [1,40] */
4264 int nice_rlim = 20 - nice;
Ingo Molnar48f24c42006-07-03 00:25:40 -07004265
Jiri Slaby78d7d402010-03-05 13:42:54 -08004266 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
Matt Mackalle43379f2005-05-01 08:59:00 -07004267 capable(CAP_SYS_NICE));
4268}
4269
Linus Torvalds1da177e2005-04-16 15:20:36 -07004270#ifdef __ARCH_WANT_SYS_NICE
4271
4272/*
4273 * sys_nice - change the priority of the current process.
4274 * @increment: priority increment
4275 *
4276 * sys_setpriority is a more generic, but much slower function that
4277 * does similar things.
4278 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004279SYSCALL_DEFINE1(nice, int, increment)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004280{
Ingo Molnar48f24c42006-07-03 00:25:40 -07004281 long nice, retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004282
4283 /*
4284 * Setpriority might change our priority at the same moment.
4285 * We don't have to worry. Conceptually one call occurs first
4286 * and we have a single winner.
4287 */
Matt Mackalle43379f2005-05-01 08:59:00 -07004288 if (increment < -40)
4289 increment = -40;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004290 if (increment > 40)
4291 increment = 40;
4292
Américo Wang2b8f8362009-02-16 18:54:21 +08004293 nice = TASK_NICE(current) + increment;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004294 if (nice < -20)
4295 nice = -20;
4296 if (nice > 19)
4297 nice = 19;
4298
Matt Mackalle43379f2005-05-01 08:59:00 -07004299 if (increment < 0 && !can_nice(current, nice))
4300 return -EPERM;
4301
Linus Torvalds1da177e2005-04-16 15:20:36 -07004302 retval = security_task_setnice(current, nice);
4303 if (retval)
4304 return retval;
4305
4306 set_user_nice(current, nice);
4307 return 0;
4308}
4309
4310#endif
4311
4312/**
4313 * task_prio - return the priority value of a given task.
4314 * @p: the task in question.
4315 *
4316 * This is the priority value as seen by users in /proc.
4317 * RT tasks are offset by -200. Normal tasks are centered
4318 * around 0, value goes from -16 to +15.
4319 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07004320int task_prio(const struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004321{
4322 return p->prio - MAX_RT_PRIO;
4323}
4324
4325/**
4326 * task_nice - return the nice value of a given task.
4327 * @p: the task in question.
4328 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07004329int task_nice(const struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004330{
4331 return TASK_NICE(p);
4332}
Pavel Roskin150d8be2008-03-05 16:56:37 -05004333EXPORT_SYMBOL(task_nice);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004334
4335/**
4336 * idle_cpu - is a given cpu idle currently?
4337 * @cpu: the processor in question.
4338 */
4339int idle_cpu(int cpu)
4340{
4341 return cpu_curr(cpu) == cpu_rq(cpu)->idle;
4342}
4343
Linus Torvalds1da177e2005-04-16 15:20:36 -07004344/**
4345 * idle_task - return the idle task for a given cpu.
4346 * @cpu: the processor in question.
4347 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07004348struct task_struct *idle_task(int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004349{
4350 return cpu_rq(cpu)->idle;
4351}
4352
4353/**
4354 * find_process_by_pid - find a process with a matching PID value.
4355 * @pid: the pid in question.
4356 */
Alexey Dobriyana9957442007-10-15 17:00:13 +02004357static struct task_struct *find_process_by_pid(pid_t pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004358{
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07004359 return pid ? find_task_by_vpid(pid) : current;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004360}
4361
4362/* Actually do priority change: must hold rq lock. */
Ingo Molnardd41f592007-07-09 18:51:59 +02004363static void
4364__setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004365{
Ingo Molnardd41f592007-07-09 18:51:59 +02004366 BUG_ON(p->se.on_rq);
Ingo Molnar48f24c42006-07-03 00:25:40 -07004367
Linus Torvalds1da177e2005-04-16 15:20:36 -07004368 p->policy = policy;
4369 p->rt_priority = prio;
Ingo Molnarb29739f2006-06-27 02:54:51 -07004370 p->normal_prio = normal_prio(p);
4371 /* we are holding p->pi_lock already */
4372 p->prio = rt_mutex_getprio(p);
Peter Zijlstraffd44db2009-11-10 20:12:01 +01004373 if (rt_prio(p->prio))
4374 p->sched_class = &rt_sched_class;
4375 else
4376 p->sched_class = &fair_sched_class;
Peter Williams2dd73a42006-06-27 02:54:34 -07004377 set_load_weight(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004378}
4379
David Howellsc69e8d92008-11-14 10:39:19 +11004380/*
4381 * check the target process has a UID that matches the current process's
4382 */
4383static bool check_same_owner(struct task_struct *p)
4384{
4385 const struct cred *cred = current_cred(), *pcred;
4386 bool match;
4387
4388 rcu_read_lock();
4389 pcred = __task_cred(p);
4390 match = (cred->euid == pcred->euid ||
4391 cred->euid == pcred->uid);
4392 rcu_read_unlock();
4393 return match;
4394}
4395
Rusty Russell961ccdd2008-06-23 13:55:38 +10004396static int __sched_setscheduler(struct task_struct *p, int policy,
4397 struct sched_param *param, bool user)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004398{
Srivatsa Vaddagiri83b699e2007-10-15 17:00:08 +02004399 int retval, oldprio, oldpolicy = -1, on_rq, running;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004400 unsigned long flags;
Thomas Gleixner83ab0aa2010-02-17 09:05:48 +01004401 const struct sched_class *prev_class;
Ingo Molnar70b97a72006-07-03 00:25:42 -07004402 struct rq *rq;
Lennart Poetteringca94c442009-06-15 17:17:47 +02004403 int reset_on_fork;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004404
Steven Rostedt66e53932006-06-27 02:54:44 -07004405 /* may grab non-irq protected spin_locks */
4406 BUG_ON(in_interrupt());
Linus Torvalds1da177e2005-04-16 15:20:36 -07004407recheck:
4408 /* double check policy once rq lock held */
Lennart Poetteringca94c442009-06-15 17:17:47 +02004409 if (policy < 0) {
4410 reset_on_fork = p->sched_reset_on_fork;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004411 policy = oldpolicy = p->policy;
Lennart Poetteringca94c442009-06-15 17:17:47 +02004412 } else {
4413 reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
4414 policy &= ~SCHED_RESET_ON_FORK;
4415
4416 if (policy != SCHED_FIFO && policy != SCHED_RR &&
4417 policy != SCHED_NORMAL && policy != SCHED_BATCH &&
4418 policy != SCHED_IDLE)
4419 return -EINVAL;
4420 }
4421
Linus Torvalds1da177e2005-04-16 15:20:36 -07004422 /*
4423 * Valid priorities for SCHED_FIFO and SCHED_RR are
Ingo Molnardd41f592007-07-09 18:51:59 +02004424 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
4425 * SCHED_BATCH and SCHED_IDLE is 0.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004426 */
4427 if (param->sched_priority < 0 ||
Ingo Molnar95cdf3b2005-09-10 00:26:11 -07004428 (p->mm && param->sched_priority > MAX_USER_RT_PRIO-1) ||
Steven Rostedtd46523e2005-07-25 16:28:39 -04004429 (!p->mm && param->sched_priority > MAX_RT_PRIO-1))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004430 return -EINVAL;
Ingo Molnare05606d2007-07-09 18:51:59 +02004431 if (rt_policy(policy) != (param->sched_priority != 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004432 return -EINVAL;
4433
Olivier Croquette37e4ab32005-06-25 14:57:32 -07004434 /*
4435 * Allow unprivileged RT tasks to decrease priority:
4436 */
Rusty Russell961ccdd2008-06-23 13:55:38 +10004437 if (user && !capable(CAP_SYS_NICE)) {
Ingo Molnare05606d2007-07-09 18:51:59 +02004438 if (rt_policy(policy)) {
Oleg Nesterov8dc3e902006-09-29 02:00:50 -07004439 unsigned long rlim_rtprio;
Oleg Nesterov5fe1d752006-09-29 02:00:48 -07004440
Oleg Nesterov8dc3e902006-09-29 02:00:50 -07004441 if (!lock_task_sighand(p, &flags))
4442 return -ESRCH;
Jiri Slaby78d7d402010-03-05 13:42:54 -08004443 rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO);
Oleg Nesterov8dc3e902006-09-29 02:00:50 -07004444 unlock_task_sighand(p, &flags);
Oleg Nesterov5fe1d752006-09-29 02:00:48 -07004445
Oleg Nesterov8dc3e902006-09-29 02:00:50 -07004446 /* can't set/change the rt policy */
4447 if (policy != p->policy && !rlim_rtprio)
4448 return -EPERM;
4449
4450 /* can't increase priority */
4451 if (param->sched_priority > p->rt_priority &&
4452 param->sched_priority > rlim_rtprio)
4453 return -EPERM;
4454 }
Ingo Molnardd41f592007-07-09 18:51:59 +02004455 /*
4456 * Like positive nice levels, dont allow tasks to
4457 * move out of SCHED_IDLE either:
4458 */
4459 if (p->policy == SCHED_IDLE && policy != SCHED_IDLE)
4460 return -EPERM;
Oleg Nesterov8dc3e902006-09-29 02:00:50 -07004461
Olivier Croquette37e4ab32005-06-25 14:57:32 -07004462 /* can't change other user's priorities */
David Howellsc69e8d92008-11-14 10:39:19 +11004463 if (!check_same_owner(p))
Olivier Croquette37e4ab32005-06-25 14:57:32 -07004464 return -EPERM;
Lennart Poetteringca94c442009-06-15 17:17:47 +02004465
4466 /* Normal users shall not reset the sched_reset_on_fork flag */
4467 if (p->sched_reset_on_fork && !reset_on_fork)
4468 return -EPERM;
Olivier Croquette37e4ab32005-06-25 14:57:32 -07004469 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004470
Jeremy Fitzhardinge725aad22008-08-03 09:33:03 -07004471 if (user) {
Peter Zijlstrab68aa232008-02-13 15:45:40 +01004472#ifdef CONFIG_RT_GROUP_SCHED
Jeremy Fitzhardinge725aad22008-08-03 09:33:03 -07004473 /*
4474 * Do not allow realtime tasks into groups that have no runtime
4475 * assigned.
4476 */
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02004477 if (rt_bandwidth_enabled() && rt_policy(policy) &&
4478 task_group(p)->rt_bandwidth.rt_runtime == 0)
Jeremy Fitzhardinge725aad22008-08-03 09:33:03 -07004479 return -EPERM;
Peter Zijlstrab68aa232008-02-13 15:45:40 +01004480#endif
4481
Jeremy Fitzhardinge725aad22008-08-03 09:33:03 -07004482 retval = security_task_setscheduler(p, policy, param);
4483 if (retval)
4484 return retval;
4485 }
4486
Linus Torvalds1da177e2005-04-16 15:20:36 -07004487 /*
Ingo Molnarb29739f2006-06-27 02:54:51 -07004488 * make sure no PI-waiters arrive (or leave) while we are
4489 * changing the priority of the task:
4490 */
Thomas Gleixner1d615482009-11-17 14:54:03 +01004491 raw_spin_lock_irqsave(&p->pi_lock, flags);
Ingo Molnarb29739f2006-06-27 02:54:51 -07004492 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07004493 * To be able to change p->policy safely, the apropriate
4494 * runqueue lock must be held.
4495 */
Ingo Molnarb29739f2006-06-27 02:54:51 -07004496 rq = __task_rq_lock(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004497 /* recheck policy now with rq lock held */
4498 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
4499 policy = oldpolicy = -1;
Ingo Molnarb29739f2006-06-27 02:54:51 -07004500 __task_rq_unlock(rq);
Thomas Gleixner1d615482009-11-17 14:54:03 +01004501 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004502 goto recheck;
4503 }
Ingo Molnardd41f592007-07-09 18:51:59 +02004504 on_rq = p->se.on_rq;
Dmitry Adamushko051a1d12007-12-18 15:21:13 +01004505 running = task_current(rq, p);
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07004506 if (on_rq)
Ingo Molnar2e1cb742007-08-09 11:16:49 +02004507 deactivate_task(rq, p, 0);
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07004508 if (running)
4509 p->sched_class->put_prev_task(rq, p);
Dmitry Adamushkof6b53202007-10-15 17:00:08 +02004510
Lennart Poetteringca94c442009-06-15 17:17:47 +02004511 p->sched_reset_on_fork = reset_on_fork;
4512
Linus Torvalds1da177e2005-04-16 15:20:36 -07004513 oldprio = p->prio;
Thomas Gleixner83ab0aa2010-02-17 09:05:48 +01004514 prev_class = p->sched_class;
Ingo Molnardd41f592007-07-09 18:51:59 +02004515 __setscheduler(rq, p, policy, param->sched_priority);
Dmitry Adamushkof6b53202007-10-15 17:00:08 +02004516
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07004517 if (running)
4518 p->sched_class->set_curr_task(rq);
Ingo Molnardd41f592007-07-09 18:51:59 +02004519 if (on_rq) {
4520 activate_task(rq, p, 0);
Steven Rostedtcb469842008-01-25 21:08:22 +01004521
4522 check_class_changed(rq, p, prev_class, oldprio, running);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004523 }
Ingo Molnarb29739f2006-06-27 02:54:51 -07004524 __task_rq_unlock(rq);
Thomas Gleixner1d615482009-11-17 14:54:03 +01004525 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
Ingo Molnarb29739f2006-06-27 02:54:51 -07004526
Thomas Gleixner95e02ca2006-06-27 02:55:02 -07004527 rt_mutex_adjust_pi(p);
4528
Linus Torvalds1da177e2005-04-16 15:20:36 -07004529 return 0;
4530}
Rusty Russell961ccdd2008-06-23 13:55:38 +10004531
4532/**
4533 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
4534 * @p: the task in question.
4535 * @policy: new policy.
4536 * @param: structure containing the new RT priority.
4537 *
4538 * NOTE that the task may be already dead.
4539 */
4540int sched_setscheduler(struct task_struct *p, int policy,
4541 struct sched_param *param)
4542{
4543 return __sched_setscheduler(p, policy, param, true);
4544}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004545EXPORT_SYMBOL_GPL(sched_setscheduler);
4546
Rusty Russell961ccdd2008-06-23 13:55:38 +10004547/**
4548 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
4549 * @p: the task in question.
4550 * @policy: new policy.
4551 * @param: structure containing the new RT priority.
4552 *
4553 * Just like sched_setscheduler, only don't bother checking if the
4554 * current context has permission. For example, this is needed in
4555 * stop_machine(): we create temporary high priority worker threads,
4556 * but our caller might not have that capability.
4557 */
4558int sched_setscheduler_nocheck(struct task_struct *p, int policy,
4559 struct sched_param *param)
4560{
4561 return __sched_setscheduler(p, policy, param, false);
4562}
4563
Ingo Molnar95cdf3b2005-09-10 00:26:11 -07004564static int
4565do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004566{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004567 struct sched_param lparam;
4568 struct task_struct *p;
Ingo Molnar36c8b582006-07-03 00:25:41 -07004569 int retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004570
4571 if (!param || pid < 0)
4572 return -EINVAL;
4573 if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
4574 return -EFAULT;
Oleg Nesterov5fe1d752006-09-29 02:00:48 -07004575
4576 rcu_read_lock();
4577 retval = -ESRCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004578 p = find_process_by_pid(pid);
Oleg Nesterov5fe1d752006-09-29 02:00:48 -07004579 if (p != NULL)
4580 retval = sched_setscheduler(p, policy, &lparam);
4581 rcu_read_unlock();
Ingo Molnar36c8b582006-07-03 00:25:41 -07004582
Linus Torvalds1da177e2005-04-16 15:20:36 -07004583 return retval;
4584}
4585
4586/**
4587 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
4588 * @pid: the pid in question.
4589 * @policy: new policy.
4590 * @param: structure containing the new RT priority.
4591 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004592SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
4593 struct sched_param __user *, param)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004594{
Jason Baronc21761f2006-01-18 17:43:03 -08004595 /* negative values for policy are not valid */
4596 if (policy < 0)
4597 return -EINVAL;
4598
Linus Torvalds1da177e2005-04-16 15:20:36 -07004599 return do_sched_setscheduler(pid, policy, param);
4600}
4601
4602/**
4603 * sys_sched_setparam - set/change the RT priority of a thread
4604 * @pid: the pid in question.
4605 * @param: structure containing the new RT priority.
4606 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004607SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004608{
4609 return do_sched_setscheduler(pid, -1, param);
4610}
4611
4612/**
4613 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
4614 * @pid: the pid in question.
4615 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004616SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004617{
Ingo Molnar36c8b582006-07-03 00:25:41 -07004618 struct task_struct *p;
Andi Kleen3a5c3592007-10-15 17:00:14 +02004619 int retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004620
4621 if (pid < 0)
Andi Kleen3a5c3592007-10-15 17:00:14 +02004622 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004623
4624 retval = -ESRCH;
Thomas Gleixner5fe85be2009-12-09 10:14:58 +00004625 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004626 p = find_process_by_pid(pid);
4627 if (p) {
4628 retval = security_task_getscheduler(p);
4629 if (!retval)
Lennart Poetteringca94c442009-06-15 17:17:47 +02004630 retval = p->policy
4631 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004632 }
Thomas Gleixner5fe85be2009-12-09 10:14:58 +00004633 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004634 return retval;
4635}
4636
4637/**
Lennart Poetteringca94c442009-06-15 17:17:47 +02004638 * sys_sched_getparam - get the RT priority of a thread
Linus Torvalds1da177e2005-04-16 15:20:36 -07004639 * @pid: the pid in question.
4640 * @param: structure containing the RT priority.
4641 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004642SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004643{
4644 struct sched_param lp;
Ingo Molnar36c8b582006-07-03 00:25:41 -07004645 struct task_struct *p;
Andi Kleen3a5c3592007-10-15 17:00:14 +02004646 int retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004647
4648 if (!param || pid < 0)
Andi Kleen3a5c3592007-10-15 17:00:14 +02004649 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004650
Thomas Gleixner5fe85be2009-12-09 10:14:58 +00004651 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004652 p = find_process_by_pid(pid);
4653 retval = -ESRCH;
4654 if (!p)
4655 goto out_unlock;
4656
4657 retval = security_task_getscheduler(p);
4658 if (retval)
4659 goto out_unlock;
4660
4661 lp.sched_priority = p->rt_priority;
Thomas Gleixner5fe85be2009-12-09 10:14:58 +00004662 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004663
4664 /*
4665 * This one might sleep, we cannot do it with a spinlock held ...
4666 */
4667 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
4668
Linus Torvalds1da177e2005-04-16 15:20:36 -07004669 return retval;
4670
4671out_unlock:
Thomas Gleixner5fe85be2009-12-09 10:14:58 +00004672 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004673 return retval;
4674}
4675
Rusty Russell96f874e2008-11-25 02:35:14 +10304676long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004677{
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304678 cpumask_var_t cpus_allowed, new_mask;
Ingo Molnar36c8b582006-07-03 00:25:41 -07004679 struct task_struct *p;
4680 int retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004681
Gautham R Shenoy95402b32008-01-25 21:08:02 +01004682 get_online_cpus();
Thomas Gleixner23f5d142009-12-09 10:15:01 +00004683 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004684
4685 p = find_process_by_pid(pid);
4686 if (!p) {
Thomas Gleixner23f5d142009-12-09 10:15:01 +00004687 rcu_read_unlock();
Gautham R Shenoy95402b32008-01-25 21:08:02 +01004688 put_online_cpus();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004689 return -ESRCH;
4690 }
4691
Thomas Gleixner23f5d142009-12-09 10:15:01 +00004692 /* Prevent p going away */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004693 get_task_struct(p);
Thomas Gleixner23f5d142009-12-09 10:15:01 +00004694 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004695
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304696 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
4697 retval = -ENOMEM;
4698 goto out_put_task;
4699 }
4700 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
4701 retval = -ENOMEM;
4702 goto out_free_cpus_allowed;
4703 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004704 retval = -EPERM;
David Howellsc69e8d92008-11-14 10:39:19 +11004705 if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004706 goto out_unlock;
4707
David Quigleye7834f82006-06-23 02:03:59 -07004708 retval = security_task_setscheduler(p, 0, NULL);
4709 if (retval)
4710 goto out_unlock;
4711
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304712 cpuset_cpus_allowed(p, cpus_allowed);
4713 cpumask_and(new_mask, in_mask, cpus_allowed);
Paul Menage8707d8b2007-10-18 23:40:22 -07004714 again:
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304715 retval = set_cpus_allowed_ptr(p, new_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004716
Paul Menage8707d8b2007-10-18 23:40:22 -07004717 if (!retval) {
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304718 cpuset_cpus_allowed(p, cpus_allowed);
4719 if (!cpumask_subset(new_mask, cpus_allowed)) {
Paul Menage8707d8b2007-10-18 23:40:22 -07004720 /*
4721 * We must have raced with a concurrent cpuset
4722 * update. Just reset the cpus_allowed to the
4723 * cpuset's cpus_allowed
4724 */
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304725 cpumask_copy(new_mask, cpus_allowed);
Paul Menage8707d8b2007-10-18 23:40:22 -07004726 goto again;
4727 }
4728 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004729out_unlock:
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304730 free_cpumask_var(new_mask);
4731out_free_cpus_allowed:
4732 free_cpumask_var(cpus_allowed);
4733out_put_task:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004734 put_task_struct(p);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01004735 put_online_cpus();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004736 return retval;
4737}
4738
4739static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
Rusty Russell96f874e2008-11-25 02:35:14 +10304740 struct cpumask *new_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004741{
Rusty Russell96f874e2008-11-25 02:35:14 +10304742 if (len < cpumask_size())
4743 cpumask_clear(new_mask);
4744 else if (len > cpumask_size())
4745 len = cpumask_size();
4746
Linus Torvalds1da177e2005-04-16 15:20:36 -07004747 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
4748}
4749
4750/**
4751 * sys_sched_setaffinity - set the cpu affinity of a process
4752 * @pid: pid of the process
4753 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4754 * @user_mask_ptr: user-space pointer to the new cpu mask
4755 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004756SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
4757 unsigned long __user *, user_mask_ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004758{
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304759 cpumask_var_t new_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004760 int retval;
4761
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304762 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
4763 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004764
Rusty Russell5a16f3d2008-11-25 02:35:11 +10304765 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
4766 if (retval == 0)
4767 retval = sched_setaffinity(pid, new_mask);
4768 free_cpumask_var(new_mask);
4769 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004770}
4771
Rusty Russell96f874e2008-11-25 02:35:14 +10304772long sched_getaffinity(pid_t pid, struct cpumask *mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004773{
Ingo Molnar36c8b582006-07-03 00:25:41 -07004774 struct task_struct *p;
Thomas Gleixner31605682009-12-08 20:24:16 +00004775 unsigned long flags;
4776 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004777 int retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004778
Gautham R Shenoy95402b32008-01-25 21:08:02 +01004779 get_online_cpus();
Thomas Gleixner23f5d142009-12-09 10:15:01 +00004780 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004781
4782 retval = -ESRCH;
4783 p = find_process_by_pid(pid);
4784 if (!p)
4785 goto out_unlock;
4786
David Quigleye7834f82006-06-23 02:03:59 -07004787 retval = security_task_getscheduler(p);
4788 if (retval)
4789 goto out_unlock;
4790
Thomas Gleixner31605682009-12-08 20:24:16 +00004791 rq = task_rq_lock(p, &flags);
Rusty Russell96f874e2008-11-25 02:35:14 +10304792 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
Thomas Gleixner31605682009-12-08 20:24:16 +00004793 task_rq_unlock(rq, &flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004794
4795out_unlock:
Thomas Gleixner23f5d142009-12-09 10:15:01 +00004796 rcu_read_unlock();
Gautham R Shenoy95402b32008-01-25 21:08:02 +01004797 put_online_cpus();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004798
Ulrich Drepper9531b622007-08-09 11:16:46 +02004799 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004800}
4801
4802/**
4803 * sys_sched_getaffinity - get the cpu affinity of a process
4804 * @pid: pid of the process
4805 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4806 * @user_mask_ptr: user-space pointer to hold the current cpu mask
4807 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004808SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
4809 unsigned long __user *, user_mask_ptr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004810{
4811 int ret;
Rusty Russellf17c8602008-11-25 02:35:11 +10304812 cpumask_var_t mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004813
Anton Blanchard84fba5e2010-04-06 17:02:19 +10004814 if ((len * BITS_PER_BYTE) < nr_cpu_ids)
KOSAKI Motohirocd3d8032010-03-12 16:15:36 +09004815 return -EINVAL;
4816 if (len & (sizeof(unsigned long)-1))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004817 return -EINVAL;
4818
Rusty Russellf17c8602008-11-25 02:35:11 +10304819 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
4820 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004821
Rusty Russellf17c8602008-11-25 02:35:11 +10304822 ret = sched_getaffinity(pid, mask);
4823 if (ret == 0) {
KOSAKI Motohiro8bc037f2010-03-17 09:36:58 +09004824 size_t retlen = min_t(size_t, len, cpumask_size());
KOSAKI Motohirocd3d8032010-03-12 16:15:36 +09004825
4826 if (copy_to_user(user_mask_ptr, mask, retlen))
Rusty Russellf17c8602008-11-25 02:35:11 +10304827 ret = -EFAULT;
4828 else
KOSAKI Motohirocd3d8032010-03-12 16:15:36 +09004829 ret = retlen;
Rusty Russellf17c8602008-11-25 02:35:11 +10304830 }
4831 free_cpumask_var(mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004832
Rusty Russellf17c8602008-11-25 02:35:11 +10304833 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004834}
4835
4836/**
4837 * sys_sched_yield - yield the current processor to other threads.
4838 *
Ingo Molnardd41f592007-07-09 18:51:59 +02004839 * This function yields the current CPU to other tasks. If there are no
4840 * other threads running on this CPU then this function will return.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004841 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004842SYSCALL_DEFINE0(sched_yield)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004843{
Ingo Molnar70b97a72006-07-03 00:25:42 -07004844 struct rq *rq = this_rq_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004845
Ingo Molnar2d723762007-10-15 17:00:12 +02004846 schedstat_inc(rq, yld_count);
Dmitry Adamushko4530d7a2007-10-15 17:00:08 +02004847 current->sched_class->yield_task(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004848
4849 /*
4850 * Since we are going to call schedule() anyway, there's
4851 * no need to preempt or enable interrupts:
4852 */
4853 __release(rq->lock);
Ingo Molnar8a25d5d2006-07-03 00:24:54 -07004854 spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +01004855 do_raw_spin_unlock(&rq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004856 preempt_enable_no_resched();
4857
4858 schedule();
4859
4860 return 0;
4861}
4862
Peter Zijlstrad86ee482009-07-10 14:57:57 +02004863static inline int should_resched(void)
4864{
4865 return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
4866}
4867
Andrew Mortone7b38402006-06-30 01:56:00 -07004868static void __cond_resched(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004869{
Frederic Weisbeckere7aaaa62009-07-16 15:44:29 +02004870 add_preempt_count(PREEMPT_ACTIVE);
4871 schedule();
4872 sub_preempt_count(PREEMPT_ACTIVE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004873}
4874
Herbert Xu02b67cc2008-01-25 21:08:28 +01004875int __sched _cond_resched(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004876{
Peter Zijlstrad86ee482009-07-10 14:57:57 +02004877 if (should_resched()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004878 __cond_resched();
4879 return 1;
4880 }
4881 return 0;
4882}
Herbert Xu02b67cc2008-01-25 21:08:28 +01004883EXPORT_SYMBOL(_cond_resched);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004884
4885/*
Frederic Weisbecker613afbf2009-07-16 15:44:29 +02004886 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004887 * call schedule, and on return reacquire the lock.
4888 *
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01004889 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
Linus Torvalds1da177e2005-04-16 15:20:36 -07004890 * operations here to prevent schedule() from being called twice (once via
4891 * spin_unlock(), once by hand).
4892 */
Frederic Weisbecker613afbf2009-07-16 15:44:29 +02004893int __cond_resched_lock(spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004894{
Peter Zijlstrad86ee482009-07-10 14:57:57 +02004895 int resched = should_resched();
Jan Kara6df3cec2005-06-13 15:52:32 -07004896 int ret = 0;
4897
Peter Zijlstraf607c662009-07-20 19:16:29 +02004898 lockdep_assert_held(lock);
4899
Nick Piggin95c354f2008-01-30 13:31:20 +01004900 if (spin_needbreak(lock) || resched) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004901 spin_unlock(lock);
Peter Zijlstrad86ee482009-07-10 14:57:57 +02004902 if (resched)
Nick Piggin95c354f2008-01-30 13:31:20 +01004903 __cond_resched();
4904 else
4905 cpu_relax();
Jan Kara6df3cec2005-06-13 15:52:32 -07004906 ret = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004907 spin_lock(lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004908 }
Jan Kara6df3cec2005-06-13 15:52:32 -07004909 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004910}
Frederic Weisbecker613afbf2009-07-16 15:44:29 +02004911EXPORT_SYMBOL(__cond_resched_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004912
Frederic Weisbecker613afbf2009-07-16 15:44:29 +02004913int __sched __cond_resched_softirq(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004914{
4915 BUG_ON(!in_softirq());
4916
Peter Zijlstrad86ee482009-07-10 14:57:57 +02004917 if (should_resched()) {
Thomas Gleixner98d82562007-05-23 13:58:18 -07004918 local_bh_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004919 __cond_resched();
4920 local_bh_disable();
4921 return 1;
4922 }
4923 return 0;
4924}
Frederic Weisbecker613afbf2009-07-16 15:44:29 +02004925EXPORT_SYMBOL(__cond_resched_softirq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004926
Linus Torvalds1da177e2005-04-16 15:20:36 -07004927/**
4928 * yield - yield the current processor to other threads.
4929 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -08004930 * This is a shortcut for kernel-space yielding - it marks the
Linus Torvalds1da177e2005-04-16 15:20:36 -07004931 * thread runnable and calls sys_sched_yield().
4932 */
4933void __sched yield(void)
4934{
4935 set_current_state(TASK_RUNNING);
4936 sys_sched_yield();
4937}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004938EXPORT_SYMBOL(yield);
4939
4940/*
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01004941 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
Linus Torvalds1da177e2005-04-16 15:20:36 -07004942 * that process accounting knows that this is a task in IO wait state.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004943 */
4944void __sched io_schedule(void)
4945{
Hitoshi Mitake54d35f22009-06-29 14:44:57 +09004946 struct rq *rq = raw_rq();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004947
Shailabh Nagar0ff92242006-07-14 00:24:37 -07004948 delayacct_blkio_start();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004949 atomic_inc(&rq->nr_iowait);
Arjan van de Ven8f0dfc32009-07-20 11:26:58 -07004950 current->in_iowait = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004951 schedule();
Arjan van de Ven8f0dfc32009-07-20 11:26:58 -07004952 current->in_iowait = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004953 atomic_dec(&rq->nr_iowait);
Shailabh Nagar0ff92242006-07-14 00:24:37 -07004954 delayacct_blkio_end();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004955}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004956EXPORT_SYMBOL(io_schedule);
4957
4958long __sched io_schedule_timeout(long timeout)
4959{
Hitoshi Mitake54d35f22009-06-29 14:44:57 +09004960 struct rq *rq = raw_rq();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004961 long ret;
4962
Shailabh Nagar0ff92242006-07-14 00:24:37 -07004963 delayacct_blkio_start();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004964 atomic_inc(&rq->nr_iowait);
Arjan van de Ven8f0dfc32009-07-20 11:26:58 -07004965 current->in_iowait = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004966 ret = schedule_timeout(timeout);
Arjan van de Ven8f0dfc32009-07-20 11:26:58 -07004967 current->in_iowait = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004968 atomic_dec(&rq->nr_iowait);
Shailabh Nagar0ff92242006-07-14 00:24:37 -07004969 delayacct_blkio_end();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004970 return ret;
4971}
4972
4973/**
4974 * sys_sched_get_priority_max - return maximum RT priority.
4975 * @policy: scheduling class.
4976 *
4977 * this syscall returns the maximum rt_priority that can be used
4978 * by a given scheduling class.
4979 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01004980SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004981{
4982 int ret = -EINVAL;
4983
4984 switch (policy) {
4985 case SCHED_FIFO:
4986 case SCHED_RR:
4987 ret = MAX_USER_RT_PRIO-1;
4988 break;
4989 case SCHED_NORMAL:
Ingo Molnarb0a94992006-01-14 13:20:41 -08004990 case SCHED_BATCH:
Ingo Molnardd41f592007-07-09 18:51:59 +02004991 case SCHED_IDLE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07004992 ret = 0;
4993 break;
4994 }
4995 return ret;
4996}
4997
4998/**
4999 * sys_sched_get_priority_min - return minimum RT priority.
5000 * @policy: scheduling class.
5001 *
5002 * this syscall returns the minimum rt_priority that can be used
5003 * by a given scheduling class.
5004 */
Heiko Carstens5add95d2009-01-14 14:14:08 +01005005SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005006{
5007 int ret = -EINVAL;
5008
5009 switch (policy) {
5010 case SCHED_FIFO:
5011 case SCHED_RR:
5012 ret = 1;
5013 break;
5014 case SCHED_NORMAL:
Ingo Molnarb0a94992006-01-14 13:20:41 -08005015 case SCHED_BATCH:
Ingo Molnardd41f592007-07-09 18:51:59 +02005016 case SCHED_IDLE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07005017 ret = 0;
5018 }
5019 return ret;
5020}
5021
5022/**
5023 * sys_sched_rr_get_interval - return the default timeslice of a process.
5024 * @pid: pid of the process.
5025 * @interval: userspace pointer to the timeslice value.
5026 *
5027 * this syscall writes the default timeslice value of a given process
5028 * into the user-space timespec buffer. A value of '0' means infinity.
5029 */
Heiko Carstens17da2bd2009-01-14 14:14:10 +01005030SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
Heiko Carstens754fe8d2009-01-14 14:14:09 +01005031 struct timespec __user *, interval)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005032{
Ingo Molnar36c8b582006-07-03 00:25:41 -07005033 struct task_struct *p;
Dmitry Adamushkoa4ec24b2007-10-15 17:00:13 +02005034 unsigned int time_slice;
Thomas Gleixnerdba091b2009-12-09 09:32:03 +01005035 unsigned long flags;
5036 struct rq *rq;
Andi Kleen3a5c3592007-10-15 17:00:14 +02005037 int retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005038 struct timespec t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005039
5040 if (pid < 0)
Andi Kleen3a5c3592007-10-15 17:00:14 +02005041 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005042
5043 retval = -ESRCH;
Thomas Gleixner1a551ae2009-12-09 10:15:11 +00005044 rcu_read_lock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005045 p = find_process_by_pid(pid);
5046 if (!p)
5047 goto out_unlock;
5048
5049 retval = security_task_getscheduler(p);
5050 if (retval)
5051 goto out_unlock;
5052
Thomas Gleixnerdba091b2009-12-09 09:32:03 +01005053 rq = task_rq_lock(p, &flags);
5054 time_slice = p->sched_class->get_rr_interval(rq, p);
5055 task_rq_unlock(rq, &flags);
Dmitry Adamushkoa4ec24b2007-10-15 17:00:13 +02005056
Thomas Gleixner1a551ae2009-12-09 10:15:11 +00005057 rcu_read_unlock();
Dmitry Adamushkoa4ec24b2007-10-15 17:00:13 +02005058 jiffies_to_timespec(time_slice, &t);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005059 retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005060 return retval;
Andi Kleen3a5c3592007-10-15 17:00:14 +02005061
Linus Torvalds1da177e2005-04-16 15:20:36 -07005062out_unlock:
Thomas Gleixner1a551ae2009-12-09 10:15:11 +00005063 rcu_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005064 return retval;
5065}
5066
Steven Rostedt7c731e02008-05-12 21:20:41 +02005067static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
Ingo Molnar36c8b582006-07-03 00:25:41 -07005068
Ingo Molnar82a1fcb2008-01-25 21:08:02 +01005069void sched_show_task(struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005070{
Linus Torvalds1da177e2005-04-16 15:20:36 -07005071 unsigned long free = 0;
Ingo Molnar36c8b582006-07-03 00:25:41 -07005072 unsigned state;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005073
Linus Torvalds1da177e2005-04-16 15:20:36 -07005074 state = p->state ? __ffs(p->state) + 1 : 0;
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005075 printk(KERN_INFO "%-13.13s %c", p->comm,
Andreas Mohr2ed6e342006-07-10 04:43:52 -07005076 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
Ingo Molnar4bd77322007-07-11 21:21:47 +02005077#if BITS_PER_LONG == 32
Linus Torvalds1da177e2005-04-16 15:20:36 -07005078 if (state == TASK_RUNNING)
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005079 printk(KERN_CONT " running ");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005080 else
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005081 printk(KERN_CONT " %08lx ", thread_saved_pc(p));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005082#else
5083 if (state == TASK_RUNNING)
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005084 printk(KERN_CONT " running task ");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005085 else
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005086 printk(KERN_CONT " %016lx ", thread_saved_pc(p));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005087#endif
5088#ifdef CONFIG_DEBUG_STACK_USAGE
Eric Sandeen7c9f8862008-04-22 16:38:23 -05005089 free = stack_not_used(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005090#endif
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005091 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
David Rientjesaa47b7e2009-05-04 01:38:05 -07005092 task_pid_nr(p), task_pid_nr(p->real_parent),
5093 (unsigned long)task_thread_info(p)->flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005094
Nick Piggin5fb5e6d2008-01-25 21:08:34 +01005095 show_stack(p, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005096}
5097
Ingo Molnare59e2ae2006-12-06 20:35:59 -08005098void show_state_filter(unsigned long state_filter)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005099{
Ingo Molnar36c8b582006-07-03 00:25:41 -07005100 struct task_struct *g, *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005101
Ingo Molnar4bd77322007-07-11 21:21:47 +02005102#if BITS_PER_LONG == 32
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005103 printk(KERN_INFO
5104 " task PC stack pid father\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005105#else
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005106 printk(KERN_INFO
5107 " task PC stack pid father\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07005108#endif
5109 read_lock(&tasklist_lock);
5110 do_each_thread(g, p) {
5111 /*
5112 * reset the NMI-timeout, listing all files on a slow
5113 * console might take alot of time:
5114 */
5115 touch_nmi_watchdog();
Ingo Molnar39bc89f2007-04-25 20:50:03 -07005116 if (!state_filter || (p->state & state_filter))
Ingo Molnar82a1fcb2008-01-25 21:08:02 +01005117 sched_show_task(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005118 } while_each_thread(g, p);
5119
Jeremy Fitzhardinge04c91672007-05-08 00:28:05 -07005120 touch_all_softlockup_watchdogs();
5121
Ingo Molnardd41f592007-07-09 18:51:59 +02005122#ifdef CONFIG_SCHED_DEBUG
5123 sysrq_sched_debug_show();
5124#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07005125 read_unlock(&tasklist_lock);
Ingo Molnare59e2ae2006-12-06 20:35:59 -08005126 /*
5127 * Only show locks if all tasks are dumped:
5128 */
Shmulik Ladkani93335a22009-11-25 15:23:41 +02005129 if (!state_filter)
Ingo Molnare59e2ae2006-12-06 20:35:59 -08005130 debug_show_all_locks();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005131}
5132
Ingo Molnar1df21052007-07-09 18:51:58 +02005133void __cpuinit init_idle_bootup_task(struct task_struct *idle)
5134{
Ingo Molnardd41f592007-07-09 18:51:59 +02005135 idle->sched_class = &idle_sched_class;
Ingo Molnar1df21052007-07-09 18:51:58 +02005136}
5137
Ingo Molnarf340c0d2005-06-28 16:40:42 +02005138/**
5139 * init_idle - set up an idle thread for a given CPU
5140 * @idle: task in question
5141 * @cpu: cpu the idle task belongs to
5142 *
5143 * NOTE: this function does not set the idle thread's NEED_RESCHED
5144 * flag, to make booting more robust.
5145 */
Nick Piggin5c1e1762006-10-03 01:14:04 -07005146void __cpuinit init_idle(struct task_struct *idle, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005147{
Ingo Molnar70b97a72006-07-03 00:25:42 -07005148 struct rq *rq = cpu_rq(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005149 unsigned long flags;
5150
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005151 raw_spin_lock_irqsave(&rq->lock, flags);
Ingo Molnar5cbd54e2008-11-12 20:05:50 +01005152
Ingo Molnardd41f592007-07-09 18:51:59 +02005153 __sched_fork(idle);
Peter Zijlstra06b83b52009-12-16 18:04:35 +01005154 idle->state = TASK_RUNNING;
Ingo Molnardd41f592007-07-09 18:51:59 +02005155 idle->se.exec_start = sched_clock();
5156
Rusty Russell96f874e2008-11-25 02:35:14 +10305157 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
Ingo Molnardd41f592007-07-09 18:51:59 +02005158 __set_task_cpu(idle, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005159
Linus Torvalds1da177e2005-04-16 15:20:36 -07005160 rq->curr = rq->idle = idle;
Nick Piggin4866cde2005-06-25 14:57:23 -07005161#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
5162 idle->oncpu = 1;
5163#endif
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005164 raw_spin_unlock_irqrestore(&rq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005165
5166 /* Set the preempt count _outside_ the spinlocks! */
Linus Torvalds8e3e0762008-05-10 20:58:02 -07005167#if defined(CONFIG_PREEMPT)
5168 task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
5169#else
Al Viroa1261f52005-11-13 16:06:55 -08005170 task_thread_info(idle)->preempt_count = 0;
Linus Torvalds8e3e0762008-05-10 20:58:02 -07005171#endif
Ingo Molnardd41f592007-07-09 18:51:59 +02005172 /*
5173 * The idle tasks have their own, simple scheduling class:
5174 */
5175 idle->sched_class = &idle_sched_class;
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01005176 ftrace_graph_init_task(idle);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005177}
5178
5179/*
5180 * In a system that switches off the HZ timer nohz_cpu_mask
5181 * indicates which cpus entered this state. This is used
5182 * in the rcu update to wait only for active cpus. For system
5183 * which do not switch off the HZ timer nohz_cpu_mask should
Rusty Russell6a7b3dc2008-11-25 02:35:04 +10305184 * always be CPU_BITS_NONE.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005185 */
Rusty Russell6a7b3dc2008-11-25 02:35:04 +10305186cpumask_var_t nohz_cpu_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005187
Ingo Molnar19978ca2007-11-09 22:39:38 +01005188/*
5189 * Increase the granularity value when there are more CPUs,
5190 * because with more CPUs the 'effective latency' as visible
5191 * to users decreases. But the relationship is not linear,
5192 * so pick a second-best guess by going with the log2 of the
5193 * number of CPUs.
5194 *
5195 * This idea comes from the SD scheduler of Con Kolivas:
5196 */
Christian Ehrhardtacb4a842009-11-30 12:16:48 +01005197static int get_update_sysctl_factor(void)
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01005198{
Mike Galbraith4ca3ef72009-12-10 09:25:53 +01005199 unsigned int cpus = min_t(int, num_online_cpus(), 8);
Christian Ehrhardt1983a922009-11-30 12:16:47 +01005200 unsigned int factor;
5201
5202 switch (sysctl_sched_tunable_scaling) {
5203 case SCHED_TUNABLESCALING_NONE:
5204 factor = 1;
5205 break;
5206 case SCHED_TUNABLESCALING_LINEAR:
5207 factor = cpus;
5208 break;
5209 case SCHED_TUNABLESCALING_LOG:
5210 default:
5211 factor = 1 + ilog2(cpus);
5212 break;
5213 }
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01005214
Christian Ehrhardtacb4a842009-11-30 12:16:48 +01005215 return factor;
5216}
5217
5218static void update_sysctl(void)
5219{
5220 unsigned int factor = get_update_sysctl_factor();
5221
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01005222#define SET_SYSCTL(name) \
5223 (sysctl_##name = (factor) * normalized_sysctl_##name)
5224 SET_SYSCTL(sched_min_granularity);
5225 SET_SYSCTL(sched_latency);
5226 SET_SYSCTL(sched_wakeup_granularity);
5227 SET_SYSCTL(sched_shares_ratelimit);
5228#undef SET_SYSCTL
5229}
5230
Ingo Molnar19978ca2007-11-09 22:39:38 +01005231static inline void sched_init_granularity(void)
5232{
Christian Ehrhardt0bcdcf22009-11-30 12:16:46 +01005233 update_sysctl();
Ingo Molnar19978ca2007-11-09 22:39:38 +01005234}
5235
Linus Torvalds1da177e2005-04-16 15:20:36 -07005236#ifdef CONFIG_SMP
5237/*
5238 * This is how migration works:
5239 *
Tejun Heo969c7922010-05-06 18:49:21 +02005240 * 1) we invoke migration_cpu_stop() on the target CPU using
5241 * stop_one_cpu().
5242 * 2) stopper starts to run (implicitly forcing the migrated thread
5243 * off the CPU)
5244 * 3) it checks whether the migrated task is still in the wrong runqueue.
5245 * 4) if it's in the wrong runqueue then the migration thread removes
Linus Torvalds1da177e2005-04-16 15:20:36 -07005246 * it and puts it into the right queue.
Tejun Heo969c7922010-05-06 18:49:21 +02005247 * 5) stopper completes and stop_one_cpu() returns and the migration
5248 * is done.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005249 */
5250
5251/*
5252 * Change a given task's CPU affinity. Migrate the thread to a
5253 * proper CPU and schedule it away if the CPU it's executing on
5254 * is removed from the allowed bitmask.
5255 *
5256 * NOTE: the caller must have a valid reference to the task, the
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01005257 * task must not exit() & deallocate itself prematurely. The
Linus Torvalds1da177e2005-04-16 15:20:36 -07005258 * call is not atomic; no spinlocks may be held.
5259 */
Rusty Russell96f874e2008-11-25 02:35:14 +10305260int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005261{
5262 unsigned long flags;
Ingo Molnar70b97a72006-07-03 00:25:42 -07005263 struct rq *rq;
Tejun Heo969c7922010-05-06 18:49:21 +02005264 unsigned int dest_cpu;
Ingo Molnar48f24c42006-07-03 00:25:40 -07005265 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005266
Peter Zijlstra65cc8e42010-03-25 21:05:16 +01005267 /*
5268 * Serialize against TASK_WAKING so that ttwu() and wunt() can
5269 * drop the rq->lock and still rely on ->cpus_allowed.
5270 */
5271again:
5272 while (task_is_waking(p))
5273 cpu_relax();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005274 rq = task_rq_lock(p, &flags);
Peter Zijlstra65cc8e42010-03-25 21:05:16 +01005275 if (task_is_waking(p)) {
5276 task_rq_unlock(rq, &flags);
5277 goto again;
5278 }
Peter Zijlstrae2912002009-12-16 18:04:36 +01005279
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01005280 if (!cpumask_intersects(new_mask, cpu_active_mask)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005281 ret = -EINVAL;
5282 goto out;
5283 }
5284
David Rientjes9985b0b2008-06-05 12:57:11 -07005285 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
Rusty Russell96f874e2008-11-25 02:35:14 +10305286 !cpumask_equal(&p->cpus_allowed, new_mask))) {
David Rientjes9985b0b2008-06-05 12:57:11 -07005287 ret = -EINVAL;
5288 goto out;
5289 }
5290
Gregory Haskins73fe6aa2008-01-25 21:08:07 +01005291 if (p->sched_class->set_cpus_allowed)
Mike Traviscd8ba7c2008-03-26 14:23:49 -07005292 p->sched_class->set_cpus_allowed(p, new_mask);
Gregory Haskins73fe6aa2008-01-25 21:08:07 +01005293 else {
Rusty Russell96f874e2008-11-25 02:35:14 +10305294 cpumask_copy(&p->cpus_allowed, new_mask);
5295 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
Gregory Haskins73fe6aa2008-01-25 21:08:07 +01005296 }
5297
Linus Torvalds1da177e2005-04-16 15:20:36 -07005298 /* Can the task run on the task's current CPU? If so, we're done */
Rusty Russell96f874e2008-11-25 02:35:14 +10305299 if (cpumask_test_cpu(task_cpu(p), new_mask))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005300 goto out;
5301
Tejun Heo969c7922010-05-06 18:49:21 +02005302 dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
5303 if (migrate_task(p, dest_cpu)) {
5304 struct migration_arg arg = { p, dest_cpu };
Linus Torvalds1da177e2005-04-16 15:20:36 -07005305 /* Need help from migration thread: drop lock and wait. */
5306 task_rq_unlock(rq, &flags);
Tejun Heo969c7922010-05-06 18:49:21 +02005307 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005308 tlb_migrate_finish(p->mm);
5309 return 0;
5310 }
5311out:
5312 task_rq_unlock(rq, &flags);
Ingo Molnar48f24c42006-07-03 00:25:40 -07005313
Linus Torvalds1da177e2005-04-16 15:20:36 -07005314 return ret;
5315}
Mike Traviscd8ba7c2008-03-26 14:23:49 -07005316EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005317
5318/*
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01005319 * Move (not current) task off this cpu, onto dest cpu. We're doing
Linus Torvalds1da177e2005-04-16 15:20:36 -07005320 * this because either it can't run here any more (set_cpus_allowed()
5321 * away from this CPU, or CPU going down), or because we're
5322 * attempting to rebalance this task on exec (sched_exec).
5323 *
5324 * So we race with normal scheduler movements, but that's OK, as long
5325 * as the task is no longer on this CPU.
Kirill Korotaevefc30812006-06-27 02:54:32 -07005326 *
5327 * Returns non-zero if task was successfully migrated.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005328 */
Kirill Korotaevefc30812006-06-27 02:54:32 -07005329static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005330{
Ingo Molnar70b97a72006-07-03 00:25:42 -07005331 struct rq *rq_dest, *rq_src;
Peter Zijlstrae2912002009-12-16 18:04:36 +01005332 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005333
Max Krasnyanskye761b772008-07-15 04:43:49 -07005334 if (unlikely(!cpu_active(dest_cpu)))
Kirill Korotaevefc30812006-06-27 02:54:32 -07005335 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005336
5337 rq_src = cpu_rq(src_cpu);
5338 rq_dest = cpu_rq(dest_cpu);
5339
5340 double_rq_lock(rq_src, rq_dest);
5341 /* Already moved. */
5342 if (task_cpu(p) != src_cpu)
Linus Torvaldsb1e38732008-07-10 11:25:03 -07005343 goto done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005344 /* Affinity changed (again). */
Rusty Russell96f874e2008-11-25 02:35:14 +10305345 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
Linus Torvaldsb1e38732008-07-10 11:25:03 -07005346 goto fail;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005347
Peter Zijlstrae2912002009-12-16 18:04:36 +01005348 /*
5349 * If we're not on a rq, the next wake-up will ensure we're
5350 * placed properly.
5351 */
5352 if (p->se.on_rq) {
Ingo Molnar2e1cb742007-08-09 11:16:49 +02005353 deactivate_task(rq_src, p, 0);
Peter Zijlstrae2912002009-12-16 18:04:36 +01005354 set_task_cpu(p, dest_cpu);
Ingo Molnardd41f592007-07-09 18:51:59 +02005355 activate_task(rq_dest, p, 0);
Peter Zijlstra15afe092008-09-20 23:38:02 +02005356 check_preempt_curr(rq_dest, p, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005357 }
Linus Torvaldsb1e38732008-07-10 11:25:03 -07005358done:
Kirill Korotaevefc30812006-06-27 02:54:32 -07005359 ret = 1;
Linus Torvaldsb1e38732008-07-10 11:25:03 -07005360fail:
Linus Torvalds1da177e2005-04-16 15:20:36 -07005361 double_rq_unlock(rq_src, rq_dest);
Kirill Korotaevefc30812006-06-27 02:54:32 -07005362 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005363}
5364
5365/*
Tejun Heo969c7922010-05-06 18:49:21 +02005366 * migration_cpu_stop - this will be executed by a highprio stopper thread
5367 * and performs thread migration by bumping thread off CPU then
5368 * 'pushing' onto another runqueue.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005369 */
Tejun Heo969c7922010-05-06 18:49:21 +02005370static int migration_cpu_stop(void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005371{
Tejun Heo969c7922010-05-06 18:49:21 +02005372 struct migration_arg *arg = data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005373
Tejun Heo969c7922010-05-06 18:49:21 +02005374 /*
5375 * The original target cpu might have gone down and we might
5376 * be on another cpu but it doesn't matter.
5377 */
5378 local_irq_disable();
5379 __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
5380 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005381 return 0;
5382}
5383
5384#ifdef CONFIG_HOTPLUG_CPU
Kirill Korotaev054b9102006-12-10 02:20:11 -08005385/*
Robert P. J. Day3a4fa0a2007-10-19 23:10:43 +02005386 * Figure out where task on dead CPU should go, use force if necessary.
Kirill Korotaev054b9102006-12-10 02:20:11 -08005387 */
Oleg Nesterov6a1bdc12010-03-15 10:10:23 +01005388void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005389{
Oleg Nesterov1445c082010-03-15 10:10:10 +01005390 struct rq *rq = cpu_rq(dead_cpu);
5391 int needs_cpu, uninitialized_var(dest_cpu);
5392 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005393
Oleg Nesterov1445c082010-03-15 10:10:10 +01005394 local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005395
Oleg Nesterov1445c082010-03-15 10:10:10 +01005396 raw_spin_lock(&rq->lock);
5397 needs_cpu = (task_cpu(p) == dead_cpu) && (p->state != TASK_WAKING);
5398 if (needs_cpu)
5399 dest_cpu = select_fallback_rq(dead_cpu, p);
5400 raw_spin_unlock(&rq->lock);
Oleg Nesterovc1804d52010-03-15 10:10:14 +01005401 /*
5402 * It can only fail if we race with set_cpus_allowed(),
5403 * in the racer should migrate the task anyway.
5404 */
Oleg Nesterov1445c082010-03-15 10:10:10 +01005405 if (needs_cpu)
Oleg Nesterovc1804d52010-03-15 10:10:14 +01005406 __migrate_task(p, dead_cpu, dest_cpu);
Oleg Nesterov1445c082010-03-15 10:10:10 +01005407 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005408}
5409
5410/*
5411 * While a dead CPU has no uninterruptible tasks queued at this point,
5412 * it might still have a nonzero ->nr_uninterruptible counter, because
5413 * for performance reasons the counter is not stricly tracking tasks to
5414 * their home CPUs. So we just add the counter to another CPU's counter,
5415 * to keep the global sum constant after CPU-down:
5416 */
Ingo Molnar70b97a72006-07-03 00:25:42 -07005417static void migrate_nr_uninterruptible(struct rq *rq_src)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005418{
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01005419 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005420 unsigned long flags;
5421
5422 local_irq_save(flags);
5423 double_rq_lock(rq_src, rq_dest);
5424 rq_dest->nr_uninterruptible += rq_src->nr_uninterruptible;
5425 rq_src->nr_uninterruptible = 0;
5426 double_rq_unlock(rq_src, rq_dest);
5427 local_irq_restore(flags);
5428}
5429
5430/* Run through task list and migrate tasks from the dead cpu. */
5431static void migrate_live_tasks(int src_cpu)
5432{
Ingo Molnar48f24c42006-07-03 00:25:40 -07005433 struct task_struct *p, *t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005434
Oleg Nesterovf7b4cdd2007-10-16 23:30:56 -07005435 read_lock(&tasklist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005436
Ingo Molnar48f24c42006-07-03 00:25:40 -07005437 do_each_thread(t, p) {
5438 if (p == current)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005439 continue;
5440
Ingo Molnar48f24c42006-07-03 00:25:40 -07005441 if (task_cpu(p) == src_cpu)
5442 move_task_off_dead_cpu(src_cpu, p);
5443 } while_each_thread(t, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005444
Oleg Nesterovf7b4cdd2007-10-16 23:30:56 -07005445 read_unlock(&tasklist_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005446}
5447
Ingo Molnardd41f592007-07-09 18:51:59 +02005448/*
5449 * Schedules idle task to be the next runnable task on current CPU.
Dmitry Adamushko94bc9a72007-11-15 20:57:40 +01005450 * It does so by boosting its priority to highest possible.
5451 * Used by CPU offline code.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005452 */
5453void sched_idle_next(void)
5454{
Ingo Molnar48f24c42006-07-03 00:25:40 -07005455 int this_cpu = smp_processor_id();
Ingo Molnar70b97a72006-07-03 00:25:42 -07005456 struct rq *rq = cpu_rq(this_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005457 struct task_struct *p = rq->idle;
5458 unsigned long flags;
5459
5460 /* cpu has to be offline */
Ingo Molnar48f24c42006-07-03 00:25:40 -07005461 BUG_ON(cpu_online(this_cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -07005462
Ingo Molnar48f24c42006-07-03 00:25:40 -07005463 /*
5464 * Strictly not necessary since rest of the CPUs are stopped by now
5465 * and interrupts disabled on the current cpu.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005466 */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005467 raw_spin_lock_irqsave(&rq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005468
Ingo Molnardd41f592007-07-09 18:51:59 +02005469 __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
Ingo Molnar48f24c42006-07-03 00:25:40 -07005470
Dmitry Adamushko94bc9a72007-11-15 20:57:40 +01005471 activate_task(rq, p, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005472
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005473 raw_spin_unlock_irqrestore(&rq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005474}
5475
Ingo Molnar48f24c42006-07-03 00:25:40 -07005476/*
5477 * Ensures that the idle task is using init_mm right before its cpu goes
Linus Torvalds1da177e2005-04-16 15:20:36 -07005478 * offline.
5479 */
5480void idle_task_exit(void)
5481{
5482 struct mm_struct *mm = current->active_mm;
5483
5484 BUG_ON(cpu_online(smp_processor_id()));
5485
5486 if (mm != &init_mm)
5487 switch_mm(mm, &init_mm, current);
5488 mmdrop(mm);
5489}
5490
Kirill Korotaev054b9102006-12-10 02:20:11 -08005491/* called under rq->lock with disabled interrupts */
Ingo Molnar36c8b582006-07-03 00:25:41 -07005492static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005493{
Ingo Molnar70b97a72006-07-03 00:25:42 -07005494 struct rq *rq = cpu_rq(dead_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005495
5496 /* Must be exiting, otherwise would be on tasklist. */
Eugene Teo270f7222007-10-18 23:40:38 -07005497 BUG_ON(!p->exit_state);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005498
5499 /* Cannot have done final schedule yet: would have vanished. */
Oleg Nesterovc394cc92006-09-29 02:01:11 -07005500 BUG_ON(p->state == TASK_DEAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005501
Ingo Molnar48f24c42006-07-03 00:25:40 -07005502 get_task_struct(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005503
5504 /*
5505 * Drop lock around migration; if someone else moves it,
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01005506 * that's OK. No task can be added to this CPU, so iteration is
Linus Torvalds1da177e2005-04-16 15:20:36 -07005507 * fine.
5508 */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005509 raw_spin_unlock_irq(&rq->lock);
Ingo Molnar48f24c42006-07-03 00:25:40 -07005510 move_task_off_dead_cpu(dead_cpu, p);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005511 raw_spin_lock_irq(&rq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005512
Ingo Molnar48f24c42006-07-03 00:25:40 -07005513 put_task_struct(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005514}
5515
5516/* release_task() removes task from tasklist, so we won't find dead tasks. */
5517static void migrate_dead_tasks(unsigned int dead_cpu)
5518{
Ingo Molnar70b97a72006-07-03 00:25:42 -07005519 struct rq *rq = cpu_rq(dead_cpu);
Ingo Molnardd41f592007-07-09 18:51:59 +02005520 struct task_struct *next;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005521
Ingo Molnardd41f592007-07-09 18:51:59 +02005522 for ( ; ; ) {
5523 if (!rq->nr_running)
5524 break;
Wang Chenb67802e2009-03-02 13:55:26 +08005525 next = pick_next_task(rq);
Ingo Molnardd41f592007-07-09 18:51:59 +02005526 if (!next)
5527 break;
Dmitry Adamushko79c53792008-06-29 00:16:56 +02005528 next->sched_class->put_prev_task(rq, next);
Ingo Molnardd41f592007-07-09 18:51:59 +02005529 migrate_dead(dead_cpu, next);
Nick Piggine692ab52007-07-26 13:40:43 +02005530
Linus Torvalds1da177e2005-04-16 15:20:36 -07005531 }
5532}
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02005533
5534/*
5535 * remove the tasks which were accounted by rq from calc_load_tasks.
5536 */
5537static void calc_global_load_remove(struct rq *rq)
5538{
5539 atomic_long_sub(rq->calc_load_active, &calc_load_tasks);
Thomas Gleixnera468d382009-07-17 14:15:46 +02005540 rq->calc_load_active = 0;
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02005541}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005542#endif /* CONFIG_HOTPLUG_CPU */
5543
Nick Piggine692ab52007-07-26 13:40:43 +02005544#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
5545
5546static struct ctl_table sd_ctl_dir[] = {
Alexey Dobriyane0361852007-08-09 11:16:46 +02005547 {
5548 .procname = "sched_domain",
Eric W. Biedermanc57baf12007-08-23 15:18:02 +02005549 .mode = 0555,
Alexey Dobriyane0361852007-08-09 11:16:46 +02005550 },
Eric W. Biederman56992302009-11-05 15:38:40 -08005551 {}
Nick Piggine692ab52007-07-26 13:40:43 +02005552};
5553
5554static struct ctl_table sd_ctl_root[] = {
Alexey Dobriyane0361852007-08-09 11:16:46 +02005555 {
5556 .procname = "kernel",
Eric W. Biedermanc57baf12007-08-23 15:18:02 +02005557 .mode = 0555,
Alexey Dobriyane0361852007-08-09 11:16:46 +02005558 .child = sd_ctl_dir,
5559 },
Eric W. Biederman56992302009-11-05 15:38:40 -08005560 {}
Nick Piggine692ab52007-07-26 13:40:43 +02005561};
5562
5563static struct ctl_table *sd_alloc_ctl_entry(int n)
5564{
5565 struct ctl_table *entry =
Milton Miller5cf9f062007-10-15 17:00:19 +02005566 kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
Nick Piggine692ab52007-07-26 13:40:43 +02005567
Nick Piggine692ab52007-07-26 13:40:43 +02005568 return entry;
5569}
5570
Milton Miller6382bc92007-10-15 17:00:19 +02005571static void sd_free_ctl_entry(struct ctl_table **tablep)
5572{
Milton Millercd790072007-10-17 16:55:11 +02005573 struct ctl_table *entry;
Milton Miller6382bc92007-10-15 17:00:19 +02005574
Milton Millercd790072007-10-17 16:55:11 +02005575 /*
5576 * In the intermediate directories, both the child directory and
5577 * procname are dynamically allocated and could fail but the mode
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01005578 * will always be set. In the lowest directory the names are
Milton Millercd790072007-10-17 16:55:11 +02005579 * static strings and all have proc handlers.
5580 */
5581 for (entry = *tablep; entry->mode; entry++) {
Milton Miller6382bc92007-10-15 17:00:19 +02005582 if (entry->child)
5583 sd_free_ctl_entry(&entry->child);
Milton Millercd790072007-10-17 16:55:11 +02005584 if (entry->proc_handler == NULL)
5585 kfree(entry->procname);
5586 }
Milton Miller6382bc92007-10-15 17:00:19 +02005587
5588 kfree(*tablep);
5589 *tablep = NULL;
5590}
5591
Nick Piggine692ab52007-07-26 13:40:43 +02005592static void
Alexey Dobriyane0361852007-08-09 11:16:46 +02005593set_table_entry(struct ctl_table *entry,
Nick Piggine692ab52007-07-26 13:40:43 +02005594 const char *procname, void *data, int maxlen,
5595 mode_t mode, proc_handler *proc_handler)
5596{
Nick Piggine692ab52007-07-26 13:40:43 +02005597 entry->procname = procname;
5598 entry->data = data;
5599 entry->maxlen = maxlen;
5600 entry->mode = mode;
5601 entry->proc_handler = proc_handler;
5602}
5603
5604static struct ctl_table *
5605sd_alloc_ctl_domain_table(struct sched_domain *sd)
5606{
Ingo Molnara5d8c342008-10-09 11:35:51 +02005607 struct ctl_table *table = sd_alloc_ctl_entry(13);
Nick Piggine692ab52007-07-26 13:40:43 +02005608
Milton Millerad1cdc12007-10-15 17:00:19 +02005609 if (table == NULL)
5610 return NULL;
5611
Alexey Dobriyane0361852007-08-09 11:16:46 +02005612 set_table_entry(&table[0], "min_interval", &sd->min_interval,
Nick Piggine692ab52007-07-26 13:40:43 +02005613 sizeof(long), 0644, proc_doulongvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005614 set_table_entry(&table[1], "max_interval", &sd->max_interval,
Nick Piggine692ab52007-07-26 13:40:43 +02005615 sizeof(long), 0644, proc_doulongvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005616 set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
Nick Piggine692ab52007-07-26 13:40:43 +02005617 sizeof(int), 0644, proc_dointvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005618 set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
Nick Piggine692ab52007-07-26 13:40:43 +02005619 sizeof(int), 0644, proc_dointvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005620 set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
Nick Piggine692ab52007-07-26 13:40:43 +02005621 sizeof(int), 0644, proc_dointvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005622 set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
Nick Piggine692ab52007-07-26 13:40:43 +02005623 sizeof(int), 0644, proc_dointvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005624 set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
Nick Piggine692ab52007-07-26 13:40:43 +02005625 sizeof(int), 0644, proc_dointvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005626 set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
Nick Piggine692ab52007-07-26 13:40:43 +02005627 sizeof(int), 0644, proc_dointvec_minmax);
Alexey Dobriyane0361852007-08-09 11:16:46 +02005628 set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
Nick Piggine692ab52007-07-26 13:40:43 +02005629 sizeof(int), 0644, proc_dointvec_minmax);
Zou Nan haiace8b3d2007-10-15 17:00:14 +02005630 set_table_entry(&table[9], "cache_nice_tries",
Nick Piggine692ab52007-07-26 13:40:43 +02005631 &sd->cache_nice_tries,
5632 sizeof(int), 0644, proc_dointvec_minmax);
Zou Nan haiace8b3d2007-10-15 17:00:14 +02005633 set_table_entry(&table[10], "flags", &sd->flags,
Nick Piggine692ab52007-07-26 13:40:43 +02005634 sizeof(int), 0644, proc_dointvec_minmax);
Ingo Molnara5d8c342008-10-09 11:35:51 +02005635 set_table_entry(&table[11], "name", sd->name,
5636 CORENAME_MAX_SIZE, 0444, proc_dostring);
5637 /* &table[12] is terminator */
Nick Piggine692ab52007-07-26 13:40:43 +02005638
5639 return table;
5640}
5641
Ingo Molnar9a4e7152007-11-28 15:52:56 +01005642static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
Nick Piggine692ab52007-07-26 13:40:43 +02005643{
5644 struct ctl_table *entry, *table;
5645 struct sched_domain *sd;
5646 int domain_num = 0, i;
5647 char buf[32];
5648
5649 for_each_domain(cpu, sd)
5650 domain_num++;
5651 entry = table = sd_alloc_ctl_entry(domain_num + 1);
Milton Millerad1cdc12007-10-15 17:00:19 +02005652 if (table == NULL)
5653 return NULL;
Nick Piggine692ab52007-07-26 13:40:43 +02005654
5655 i = 0;
5656 for_each_domain(cpu, sd) {
5657 snprintf(buf, 32, "domain%d", i);
Nick Piggine692ab52007-07-26 13:40:43 +02005658 entry->procname = kstrdup(buf, GFP_KERNEL);
Eric W. Biedermanc57baf12007-08-23 15:18:02 +02005659 entry->mode = 0555;
Nick Piggine692ab52007-07-26 13:40:43 +02005660 entry->child = sd_alloc_ctl_domain_table(sd);
5661 entry++;
5662 i++;
5663 }
5664 return table;
5665}
5666
5667static struct ctl_table_header *sd_sysctl_header;
Milton Miller6382bc92007-10-15 17:00:19 +02005668static void register_sched_domain_sysctl(void)
Nick Piggine692ab52007-07-26 13:40:43 +02005669{
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01005670 int i, cpu_num = num_possible_cpus();
Nick Piggine692ab52007-07-26 13:40:43 +02005671 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
5672 char buf[32];
5673
Milton Miller73785472007-10-24 18:23:48 +02005674 WARN_ON(sd_ctl_dir[0].child);
5675 sd_ctl_dir[0].child = entry;
5676
Milton Millerad1cdc12007-10-15 17:00:19 +02005677 if (entry == NULL)
5678 return;
5679
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01005680 for_each_possible_cpu(i) {
Nick Piggine692ab52007-07-26 13:40:43 +02005681 snprintf(buf, 32, "cpu%d", i);
Nick Piggine692ab52007-07-26 13:40:43 +02005682 entry->procname = kstrdup(buf, GFP_KERNEL);
Eric W. Biedermanc57baf12007-08-23 15:18:02 +02005683 entry->mode = 0555;
Nick Piggine692ab52007-07-26 13:40:43 +02005684 entry->child = sd_alloc_ctl_cpu_table(i);
Milton Miller97b6ea72007-10-15 17:00:19 +02005685 entry++;
Nick Piggine692ab52007-07-26 13:40:43 +02005686 }
Milton Miller73785472007-10-24 18:23:48 +02005687
5688 WARN_ON(sd_sysctl_header);
Nick Piggine692ab52007-07-26 13:40:43 +02005689 sd_sysctl_header = register_sysctl_table(sd_ctl_root);
5690}
Milton Miller6382bc92007-10-15 17:00:19 +02005691
Milton Miller73785472007-10-24 18:23:48 +02005692/* may be called multiple times per register */
Milton Miller6382bc92007-10-15 17:00:19 +02005693static void unregister_sched_domain_sysctl(void)
5694{
Milton Miller73785472007-10-24 18:23:48 +02005695 if (sd_sysctl_header)
5696 unregister_sysctl_table(sd_sysctl_header);
Milton Miller6382bc92007-10-15 17:00:19 +02005697 sd_sysctl_header = NULL;
Milton Miller73785472007-10-24 18:23:48 +02005698 if (sd_ctl_dir[0].child)
5699 sd_free_ctl_entry(&sd_ctl_dir[0].child);
Milton Miller6382bc92007-10-15 17:00:19 +02005700}
Nick Piggine692ab52007-07-26 13:40:43 +02005701#else
Milton Miller6382bc92007-10-15 17:00:19 +02005702static void register_sched_domain_sysctl(void)
5703{
5704}
5705static void unregister_sched_domain_sysctl(void)
Nick Piggine692ab52007-07-26 13:40:43 +02005706{
5707}
5708#endif
5709
Gregory Haskins1f11eb62008-06-04 15:04:05 -04005710static void set_rq_online(struct rq *rq)
5711{
5712 if (!rq->online) {
5713 const struct sched_class *class;
5714
Rusty Russellc6c49272008-11-25 02:35:05 +10305715 cpumask_set_cpu(rq->cpu, rq->rd->online);
Gregory Haskins1f11eb62008-06-04 15:04:05 -04005716 rq->online = 1;
5717
5718 for_each_class(class) {
5719 if (class->rq_online)
5720 class->rq_online(rq);
5721 }
5722 }
5723}
5724
5725static void set_rq_offline(struct rq *rq)
5726{
5727 if (rq->online) {
5728 const struct sched_class *class;
5729
5730 for_each_class(class) {
5731 if (class->rq_offline)
5732 class->rq_offline(rq);
5733 }
5734
Rusty Russellc6c49272008-11-25 02:35:05 +10305735 cpumask_clear_cpu(rq->cpu, rq->rd->online);
Gregory Haskins1f11eb62008-06-04 15:04:05 -04005736 rq->online = 0;
5737 }
5738}
5739
Linus Torvalds1da177e2005-04-16 15:20:36 -07005740/*
5741 * migration_call - callback that gets triggered when a CPU is added.
5742 * Here we can start up the necessary migration thread for the new CPU.
5743 */
Ingo Molnar48f24c42006-07-03 00:25:40 -07005744static int __cpuinit
5745migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005746{
Ingo Molnar48f24c42006-07-03 00:25:40 -07005747 int cpu = (long)hcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005748 unsigned long flags;
Tejun Heo969c7922010-05-06 18:49:21 +02005749 struct rq *rq = cpu_rq(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005750
5751 switch (action) {
Gautham R Shenoy5be93612007-05-09 02:34:04 -07005752
Linus Torvalds1da177e2005-04-16 15:20:36 -07005753 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005754 case CPU_UP_PREPARE_FROZEN:
Thomas Gleixnera468d382009-07-17 14:15:46 +02005755 rq->calc_load_update = calc_load_update;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005756 break;
Ingo Molnar48f24c42006-07-03 00:25:40 -07005757
Linus Torvalds1da177e2005-04-16 15:20:36 -07005758 case CPU_ONLINE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005759 case CPU_ONLINE_FROZEN:
Gregory Haskins1f94ef52008-03-10 16:52:41 -04005760 /* Update our root-domain */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005761 raw_spin_lock_irqsave(&rq->lock, flags);
Gregory Haskins1f94ef52008-03-10 16:52:41 -04005762 if (rq->rd) {
Rusty Russellc6c49272008-11-25 02:35:05 +10305763 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
Gregory Haskins1f11eb62008-06-04 15:04:05 -04005764
5765 set_rq_online(rq);
Gregory Haskins1f94ef52008-03-10 16:52:41 -04005766 }
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005767 raw_spin_unlock_irqrestore(&rq->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005768 break;
Ingo Molnar48f24c42006-07-03 00:25:40 -07005769
Linus Torvalds1da177e2005-04-16 15:20:36 -07005770#ifdef CONFIG_HOTPLUG_CPU
Linus Torvalds1da177e2005-04-16 15:20:36 -07005771 case CPU_DEAD:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005772 case CPU_DEAD_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -07005773 migrate_live_tasks(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005774 /* Idle task back to normal (off runqueue, low prio) */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005775 raw_spin_lock_irq(&rq->lock);
Ingo Molnar2e1cb742007-08-09 11:16:49 +02005776 deactivate_task(rq, rq->idle, 0);
Ingo Molnardd41f592007-07-09 18:51:59 +02005777 __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
5778 rq->idle->sched_class = &idle_sched_class;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005779 migrate_dead_tasks(cpu);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005780 raw_spin_unlock_irq(&rq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005781 migrate_nr_uninterruptible(rq);
5782 BUG_ON(rq->nr_running != 0);
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02005783 calc_global_load_remove(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005784 break;
Gregory Haskins57d885f2008-01-25 21:08:18 +01005785
Gregory Haskins08f503b2008-03-10 17:59:11 -04005786 case CPU_DYING:
5787 case CPU_DYING_FROZEN:
Gregory Haskins57d885f2008-01-25 21:08:18 +01005788 /* Update our root-domain */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005789 raw_spin_lock_irqsave(&rq->lock, flags);
Gregory Haskins57d885f2008-01-25 21:08:18 +01005790 if (rq->rd) {
Rusty Russellc6c49272008-11-25 02:35:05 +10305791 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
Gregory Haskins1f11eb62008-06-04 15:04:05 -04005792 set_rq_offline(rq);
Gregory Haskins57d885f2008-01-25 21:08:18 +01005793 }
Thomas Gleixner05fa7852009-11-17 14:28:38 +01005794 raw_spin_unlock_irqrestore(&rq->lock, flags);
Gregory Haskins57d885f2008-01-25 21:08:18 +01005795 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005796#endif
5797 }
5798 return NOTIFY_OK;
5799}
5800
Paul Mackerrasf38b0822009-06-02 21:05:16 +10005801/*
5802 * Register at high priority so that task migration (migrate_all_tasks)
5803 * happens before everything else. This has to be lower priority than
Ingo Molnarcdd6c482009-09-21 12:02:48 +02005804 * the notifier in the perf_event subsystem, though.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005805 */
Chandra Seetharaman26c21432006-06-27 02:54:10 -07005806static struct notifier_block __cpuinitdata migration_notifier = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005807 .notifier_call = migration_call,
5808 .priority = 10
5809};
5810
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -07005811static int __init migration_init(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005812{
5813 void *cpu = (void *)(long)smp_processor_id();
Akinobu Mita07dccf32006-09-29 02:00:22 -07005814 int err;
Ingo Molnar48f24c42006-07-03 00:25:40 -07005815
5816 /* Start one for the boot CPU: */
Akinobu Mita07dccf32006-09-29 02:00:22 -07005817 err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
5818 BUG_ON(err == NOTIFY_BAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005819 migration_call(&migration_notifier, CPU_ONLINE, cpu);
5820 register_cpu_notifier(&migration_notifier);
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -07005821
Thomas Gleixnera004cd42009-07-21 09:54:05 +02005822 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005823}
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -07005824early_initcall(migration_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005825#endif
5826
5827#ifdef CONFIG_SMP
Christoph Lameter476f3532007-05-06 14:48:58 -07005828
Ingo Molnar3e9830d2007-10-15 17:00:13 +02005829#ifdef CONFIG_SCHED_DEBUG
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005830
Mike Travisf6630112009-11-17 18:22:15 -06005831static __read_mostly int sched_domain_debug_enabled;
5832
5833static int __init sched_domain_debug_setup(char *str)
5834{
5835 sched_domain_debug_enabled = 1;
5836
5837 return 0;
5838}
5839early_param("sched_debug", sched_domain_debug_setup);
5840
Mike Travis7c16ec52008-04-04 18:11:11 -07005841static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
Rusty Russell96f874e2008-11-25 02:35:14 +10305842 struct cpumask *groupmask)
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005843{
5844 struct sched_group *group = sd->groups;
Mike Travis434d53b2008-04-04 18:11:04 -07005845 char str[256];
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005846
Rusty Russell968ea6d2008-12-13 21:55:51 +10305847 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
Rusty Russell96f874e2008-11-25 02:35:14 +10305848 cpumask_clear(groupmask);
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005849
5850 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
5851
5852 if (!(sd->flags & SD_LOAD_BALANCE)) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005853 printk("does not load-balance\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005854 if (sd->parent)
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005855 printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5856 " has parent");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005857 return -1;
5858 }
5859
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005860 printk(KERN_CONT "span %s level %s\n", str, sd->name);
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005861
Rusty Russell758b2cd2008-11-25 02:35:04 +10305862 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005863 printk(KERN_ERR "ERROR: domain->span does not contain "
5864 "CPU%d\n", cpu);
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005865 }
Rusty Russell758b2cd2008-11-25 02:35:04 +10305866 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005867 printk(KERN_ERR "ERROR: domain->groups does not contain"
5868 " CPU%d\n", cpu);
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005869 }
5870
5871 printk(KERN_DEBUG "%*s groups:", level + 1, "");
5872 do {
5873 if (!group) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005874 printk("\n");
5875 printk(KERN_ERR "ERROR: group is NULL\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005876 break;
5877 }
5878
Peter Zijlstra18a38852009-09-01 10:34:39 +02005879 if (!group->cpu_power) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005880 printk(KERN_CONT "\n");
5881 printk(KERN_ERR "ERROR: domain->cpu_power not "
5882 "set\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005883 break;
5884 }
5885
Rusty Russell758b2cd2008-11-25 02:35:04 +10305886 if (!cpumask_weight(sched_group_cpus(group))) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005887 printk(KERN_CONT "\n");
5888 printk(KERN_ERR "ERROR: empty group\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005889 break;
5890 }
5891
Rusty Russell758b2cd2008-11-25 02:35:04 +10305892 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005893 printk(KERN_CONT "\n");
5894 printk(KERN_ERR "ERROR: repeated CPUs\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005895 break;
5896 }
5897
Rusty Russell758b2cd2008-11-25 02:35:04 +10305898 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005899
Rusty Russell968ea6d2008-12-13 21:55:51 +10305900 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
Gautham R Shenoy381512c2009-04-14 09:09:36 +05305901
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005902 printk(KERN_CONT " %s", str);
Peter Zijlstra18a38852009-09-01 10:34:39 +02005903 if (group->cpu_power != SCHED_LOAD_SCALE) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005904 printk(KERN_CONT " (cpu_power = %d)",
5905 group->cpu_power);
Gautham R Shenoy381512c2009-04-14 09:09:36 +05305906 }
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005907
5908 group = group->next;
5909 } while (group != sd->groups);
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005910 printk(KERN_CONT "\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005911
Rusty Russell758b2cd2008-11-25 02:35:04 +10305912 if (!cpumask_equal(sched_domain_span(sd), groupmask))
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005913 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005914
Rusty Russell758b2cd2008-11-25 02:35:04 +10305915 if (sd->parent &&
5916 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01005917 printk(KERN_ERR "ERROR: parent span is not a superset "
5918 "of domain->span\n");
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005919 return 0;
5920}
5921
Linus Torvalds1da177e2005-04-16 15:20:36 -07005922static void sched_domain_debug(struct sched_domain *sd, int cpu)
5923{
Rusty Russelld5dd3db2008-11-25 02:35:12 +10305924 cpumask_var_t groupmask;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005925 int level = 0;
5926
Mike Travisf6630112009-11-17 18:22:15 -06005927 if (!sched_domain_debug_enabled)
5928 return;
5929
Nick Piggin41c7ce92005-06-25 14:57:24 -07005930 if (!sd) {
5931 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
5932 return;
5933 }
5934
Linus Torvalds1da177e2005-04-16 15:20:36 -07005935 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
5936
Rusty Russelld5dd3db2008-11-25 02:35:12 +10305937 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
Mike Travis7c16ec52008-04-04 18:11:11 -07005938 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
5939 return;
5940 }
5941
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005942 for (;;) {
Mike Travis7c16ec52008-04-04 18:11:11 -07005943 if (sched_domain_debug_one(sd, cpu, level, groupmask))
Linus Torvalds1da177e2005-04-16 15:20:36 -07005944 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005945 level++;
5946 sd = sd->parent;
Miguel Ojeda Sandonis33859f72006-12-10 02:20:38 -08005947 if (!sd)
Ingo Molnar4dcf6af2007-10-24 18:23:48 +02005948 break;
5949 }
Rusty Russelld5dd3db2008-11-25 02:35:12 +10305950 free_cpumask_var(groupmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005951}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02005952#else /* !CONFIG_SCHED_DEBUG */
Ingo Molnar48f24c42006-07-03 00:25:40 -07005953# define sched_domain_debug(sd, cpu) do { } while (0)
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02005954#endif /* CONFIG_SCHED_DEBUG */
Linus Torvalds1da177e2005-04-16 15:20:36 -07005955
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07005956static int sd_degenerate(struct sched_domain *sd)
Suresh Siddha245af2c2005-06-25 14:57:25 -07005957{
Rusty Russell758b2cd2008-11-25 02:35:04 +10305958 if (cpumask_weight(sched_domain_span(sd)) == 1)
Suresh Siddha245af2c2005-06-25 14:57:25 -07005959 return 1;
5960
5961 /* Following flags need at least 2 groups */
5962 if (sd->flags & (SD_LOAD_BALANCE |
5963 SD_BALANCE_NEWIDLE |
5964 SD_BALANCE_FORK |
Siddha, Suresh B89c47102006-10-03 01:14:09 -07005965 SD_BALANCE_EXEC |
5966 SD_SHARE_CPUPOWER |
5967 SD_SHARE_PKG_RESOURCES)) {
Suresh Siddha245af2c2005-06-25 14:57:25 -07005968 if (sd->groups != sd->groups->next)
5969 return 0;
5970 }
5971
5972 /* Following flags don't use groups */
Peter Zijlstrac88d5912009-09-10 13:50:02 +02005973 if (sd->flags & (SD_WAKE_AFFINE))
Suresh Siddha245af2c2005-06-25 14:57:25 -07005974 return 0;
5975
5976 return 1;
5977}
5978
Ingo Molnar48f24c42006-07-03 00:25:40 -07005979static int
5980sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
Suresh Siddha245af2c2005-06-25 14:57:25 -07005981{
5982 unsigned long cflags = sd->flags, pflags = parent->flags;
5983
5984 if (sd_degenerate(parent))
5985 return 1;
5986
Rusty Russell758b2cd2008-11-25 02:35:04 +10305987 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
Suresh Siddha245af2c2005-06-25 14:57:25 -07005988 return 0;
5989
Suresh Siddha245af2c2005-06-25 14:57:25 -07005990 /* Flags needing groups don't count if only 1 group in parent */
5991 if (parent->groups == parent->groups->next) {
5992 pflags &= ~(SD_LOAD_BALANCE |
5993 SD_BALANCE_NEWIDLE |
5994 SD_BALANCE_FORK |
Siddha, Suresh B89c47102006-10-03 01:14:09 -07005995 SD_BALANCE_EXEC |
5996 SD_SHARE_CPUPOWER |
5997 SD_SHARE_PKG_RESOURCES);
Ken Chen54364992008-12-07 18:47:37 -08005998 if (nr_node_ids == 1)
5999 pflags &= ~SD_SERIALIZE;
Suresh Siddha245af2c2005-06-25 14:57:25 -07006000 }
6001 if (~cflags & pflags)
6002 return 0;
6003
6004 return 1;
6005}
6006
Rusty Russellc6c49272008-11-25 02:35:05 +10306007static void free_rootdomain(struct root_domain *rd)
6008{
Peter Zijlstra047106a2009-11-16 10:28:09 +01006009 synchronize_sched();
6010
Rusty Russell68e74562008-11-25 02:35:13 +10306011 cpupri_cleanup(&rd->cpupri);
6012
Rusty Russellc6c49272008-11-25 02:35:05 +10306013 free_cpumask_var(rd->rto_mask);
6014 free_cpumask_var(rd->online);
6015 free_cpumask_var(rd->span);
6016 kfree(rd);
6017}
6018
Gregory Haskins57d885f2008-01-25 21:08:18 +01006019static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6020{
Ingo Molnara0490fa2009-02-12 11:35:40 +01006021 struct root_domain *old_rd = NULL;
Gregory Haskins57d885f2008-01-25 21:08:18 +01006022 unsigned long flags;
Gregory Haskins57d885f2008-01-25 21:08:18 +01006023
Thomas Gleixner05fa7852009-11-17 14:28:38 +01006024 raw_spin_lock_irqsave(&rq->lock, flags);
Gregory Haskins57d885f2008-01-25 21:08:18 +01006025
6026 if (rq->rd) {
Ingo Molnara0490fa2009-02-12 11:35:40 +01006027 old_rd = rq->rd;
Gregory Haskins57d885f2008-01-25 21:08:18 +01006028
Rusty Russellc6c49272008-11-25 02:35:05 +10306029 if (cpumask_test_cpu(rq->cpu, old_rd->online))
Gregory Haskins1f11eb62008-06-04 15:04:05 -04006030 set_rq_offline(rq);
Gregory Haskins57d885f2008-01-25 21:08:18 +01006031
Rusty Russellc6c49272008-11-25 02:35:05 +10306032 cpumask_clear_cpu(rq->cpu, old_rd->span);
Gregory Haskinsdc938522008-01-25 21:08:26 +01006033
Ingo Molnara0490fa2009-02-12 11:35:40 +01006034 /*
6035 * If we dont want to free the old_rt yet then
6036 * set old_rd to NULL to skip the freeing later
6037 * in this function:
6038 */
6039 if (!atomic_dec_and_test(&old_rd->refcount))
6040 old_rd = NULL;
Gregory Haskins57d885f2008-01-25 21:08:18 +01006041 }
6042
6043 atomic_inc(&rd->refcount);
6044 rq->rd = rd;
6045
Rusty Russellc6c49272008-11-25 02:35:05 +10306046 cpumask_set_cpu(rq->cpu, rd->span);
Gregory Haskins00aec932009-07-30 10:57:23 -04006047 if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
Gregory Haskins1f11eb62008-06-04 15:04:05 -04006048 set_rq_online(rq);
Gregory Haskins57d885f2008-01-25 21:08:18 +01006049
Thomas Gleixner05fa7852009-11-17 14:28:38 +01006050 raw_spin_unlock_irqrestore(&rq->lock, flags);
Ingo Molnara0490fa2009-02-12 11:35:40 +01006051
6052 if (old_rd)
6053 free_rootdomain(old_rd);
Gregory Haskins57d885f2008-01-25 21:08:18 +01006054}
6055
Li Zefanfd5e1b52009-06-15 13:34:19 +08006056static int init_rootdomain(struct root_domain *rd, bool bootmem)
Gregory Haskins57d885f2008-01-25 21:08:18 +01006057{
Pekka Enberg36b7b6d2009-06-10 23:42:36 +03006058 gfp_t gfp = GFP_KERNEL;
6059
Gregory Haskins57d885f2008-01-25 21:08:18 +01006060 memset(rd, 0, sizeof(*rd));
6061
Pekka Enberg36b7b6d2009-06-10 23:42:36 +03006062 if (bootmem)
6063 gfp = GFP_NOWAIT;
Gregory Haskins6e0534f2008-05-12 21:21:01 +02006064
Pekka Enberg36b7b6d2009-06-10 23:42:36 +03006065 if (!alloc_cpumask_var(&rd->span, gfp))
Li Zefan0c910d22009-01-06 17:39:06 +08006066 goto out;
Pekka Enberg36b7b6d2009-06-10 23:42:36 +03006067 if (!alloc_cpumask_var(&rd->online, gfp))
Rusty Russellc6c49272008-11-25 02:35:05 +10306068 goto free_span;
Pekka Enberg36b7b6d2009-06-10 23:42:36 +03006069 if (!alloc_cpumask_var(&rd->rto_mask, gfp))
Rusty Russellc6c49272008-11-25 02:35:05 +10306070 goto free_online;
Gregory Haskins6e0534f2008-05-12 21:21:01 +02006071
Pekka Enberg0fb53022009-06-11 08:41:22 +03006072 if (cpupri_init(&rd->cpupri, bootmem) != 0)
Rusty Russell68e74562008-11-25 02:35:13 +10306073 goto free_rto_mask;
Rusty Russellc6c49272008-11-25 02:35:05 +10306074 return 0;
6075
Rusty Russell68e74562008-11-25 02:35:13 +10306076free_rto_mask:
6077 free_cpumask_var(rd->rto_mask);
Rusty Russellc6c49272008-11-25 02:35:05 +10306078free_online:
6079 free_cpumask_var(rd->online);
6080free_span:
6081 free_cpumask_var(rd->span);
Li Zefan0c910d22009-01-06 17:39:06 +08006082out:
Rusty Russellc6c49272008-11-25 02:35:05 +10306083 return -ENOMEM;
Gregory Haskins57d885f2008-01-25 21:08:18 +01006084}
6085
6086static void init_defrootdomain(void)
6087{
Rusty Russellc6c49272008-11-25 02:35:05 +10306088 init_rootdomain(&def_root_domain, true);
6089
Gregory Haskins57d885f2008-01-25 21:08:18 +01006090 atomic_set(&def_root_domain.refcount, 1);
6091}
6092
Gregory Haskinsdc938522008-01-25 21:08:26 +01006093static struct root_domain *alloc_rootdomain(void)
Gregory Haskins57d885f2008-01-25 21:08:18 +01006094{
6095 struct root_domain *rd;
6096
6097 rd = kmalloc(sizeof(*rd), GFP_KERNEL);
6098 if (!rd)
6099 return NULL;
6100
Rusty Russellc6c49272008-11-25 02:35:05 +10306101 if (init_rootdomain(rd, false) != 0) {
6102 kfree(rd);
6103 return NULL;
6104 }
Gregory Haskins57d885f2008-01-25 21:08:18 +01006105
6106 return rd;
6107}
6108
Linus Torvalds1da177e2005-04-16 15:20:36 -07006109/*
Ingo Molnar0eab9142008-01-25 21:08:19 +01006110 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
Linus Torvalds1da177e2005-04-16 15:20:36 -07006111 * hold the hotplug lock.
6112 */
Ingo Molnar0eab9142008-01-25 21:08:19 +01006113static void
6114cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006115{
Ingo Molnar70b97a72006-07-03 00:25:42 -07006116 struct rq *rq = cpu_rq(cpu);
Suresh Siddha245af2c2005-06-25 14:57:25 -07006117 struct sched_domain *tmp;
6118
Peter Zijlstra669c55e2010-04-16 14:59:29 +02006119 for (tmp = sd; tmp; tmp = tmp->parent)
6120 tmp->span_weight = cpumask_weight(sched_domain_span(tmp));
6121
Suresh Siddha245af2c2005-06-25 14:57:25 -07006122 /* Remove the sched domains which do not contribute to scheduling. */
Li Zefanf29c9b12008-11-06 09:45:16 +08006123 for (tmp = sd; tmp; ) {
Suresh Siddha245af2c2005-06-25 14:57:25 -07006124 struct sched_domain *parent = tmp->parent;
6125 if (!parent)
6126 break;
Li Zefanf29c9b12008-11-06 09:45:16 +08006127
Siddha, Suresh B1a848872006-10-03 01:14:08 -07006128 if (sd_parent_degenerate(tmp, parent)) {
Suresh Siddha245af2c2005-06-25 14:57:25 -07006129 tmp->parent = parent->parent;
Siddha, Suresh B1a848872006-10-03 01:14:08 -07006130 if (parent->parent)
6131 parent->parent->child = tmp;
Li Zefanf29c9b12008-11-06 09:45:16 +08006132 } else
6133 tmp = tmp->parent;
Suresh Siddha245af2c2005-06-25 14:57:25 -07006134 }
6135
Siddha, Suresh B1a848872006-10-03 01:14:08 -07006136 if (sd && sd_degenerate(sd)) {
Suresh Siddha245af2c2005-06-25 14:57:25 -07006137 sd = sd->parent;
Siddha, Suresh B1a848872006-10-03 01:14:08 -07006138 if (sd)
6139 sd->child = NULL;
6140 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07006141
6142 sched_domain_debug(sd, cpu);
6143
Gregory Haskins57d885f2008-01-25 21:08:18 +01006144 rq_attach_root(rq, rd);
Nick Piggin674311d2005-06-25 14:57:27 -07006145 rcu_assign_pointer(rq->sd, sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006146}
6147
6148/* cpus with isolated domains */
Rusty Russelldcc30a32008-11-25 02:35:12 +10306149static cpumask_var_t cpu_isolated_map;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006150
6151/* Setup the mask of cpus configured for isolated domains */
6152static int __init isolated_cpu_setup(char *str)
6153{
Rusty Russellbdddd292009-12-02 14:09:16 +10306154 alloc_bootmem_cpumask_var(&cpu_isolated_map);
Rusty Russell968ea6d2008-12-13 21:55:51 +10306155 cpulist_parse(str, cpu_isolated_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006156 return 1;
6157}
6158
Ingo Molnar8927f492007-10-15 17:00:13 +02006159__setup("isolcpus=", isolated_cpu_setup);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006160
6161/*
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006162 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6163 * to a function which identifies what group(along with sched group) a CPU
Rusty Russell96f874e2008-11-25 02:35:14 +10306164 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6165 * (due to the fact that we keep track of groups covered with a struct cpumask).
Linus Torvalds1da177e2005-04-16 15:20:36 -07006166 *
6167 * init_sched_build_groups will build a circular linked list of the groups
6168 * covered by the given span, and will set each group's ->cpumask correctly,
6169 * and ->cpu_power to 0.
6170 */
Siddha, Suresh Ba6160582006-10-03 01:14:06 -07006171static void
Rusty Russell96f874e2008-11-25 02:35:14 +10306172init_sched_build_groups(const struct cpumask *span,
6173 const struct cpumask *cpu_map,
6174 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
Mike Travis7c16ec52008-04-04 18:11:11 -07006175 struct sched_group **sg,
Rusty Russell96f874e2008-11-25 02:35:14 +10306176 struct cpumask *tmpmask),
6177 struct cpumask *covered, struct cpumask *tmpmask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006178{
6179 struct sched_group *first = NULL, *last = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006180 int i;
6181
Rusty Russell96f874e2008-11-25 02:35:14 +10306182 cpumask_clear(covered);
Mike Travis7c16ec52008-04-04 18:11:11 -07006183
Rusty Russellabcd0832008-11-25 02:35:02 +10306184 for_each_cpu(i, span) {
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006185 struct sched_group *sg;
Mike Travis7c16ec52008-04-04 18:11:11 -07006186 int group = group_fn(i, cpu_map, &sg, tmpmask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006187 int j;
6188
Rusty Russell758b2cd2008-11-25 02:35:04 +10306189 if (cpumask_test_cpu(i, covered))
Linus Torvalds1da177e2005-04-16 15:20:36 -07006190 continue;
6191
Rusty Russell758b2cd2008-11-25 02:35:04 +10306192 cpumask_clear(sched_group_cpus(sg));
Peter Zijlstra18a38852009-09-01 10:34:39 +02006193 sg->cpu_power = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006194
Rusty Russellabcd0832008-11-25 02:35:02 +10306195 for_each_cpu(j, span) {
Mike Travis7c16ec52008-04-04 18:11:11 -07006196 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006197 continue;
6198
Rusty Russell96f874e2008-11-25 02:35:14 +10306199 cpumask_set_cpu(j, covered);
Rusty Russell758b2cd2008-11-25 02:35:04 +10306200 cpumask_set_cpu(j, sched_group_cpus(sg));
Linus Torvalds1da177e2005-04-16 15:20:36 -07006201 }
6202 if (!first)
6203 first = sg;
6204 if (last)
6205 last->next = sg;
6206 last = sg;
6207 }
6208 last->next = first;
6209}
6210
John Hawkes9c1cfda2005-09-06 15:18:14 -07006211#define SD_NODES_PER_DOMAIN 16
Linus Torvalds1da177e2005-04-16 15:20:36 -07006212
John Hawkes9c1cfda2005-09-06 15:18:14 -07006213#ifdef CONFIG_NUMA
akpm@osdl.org198e2f12006-01-12 01:05:30 -08006214
John Hawkes9c1cfda2005-09-06 15:18:14 -07006215/**
6216 * find_next_best_node - find the next node to include in a sched_domain
6217 * @node: node whose sched_domain we're building
6218 * @used_nodes: nodes already in the sched_domain
6219 *
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01006220 * Find the next node to include in a given scheduling domain. Simply
John Hawkes9c1cfda2005-09-06 15:18:14 -07006221 * finds the closest node not already in the @used_nodes map.
6222 *
6223 * Should use nodemask_t.
6224 */
Mike Travisc5f59f02008-04-04 18:11:10 -07006225static int find_next_best_node(int node, nodemask_t *used_nodes)
John Hawkes9c1cfda2005-09-06 15:18:14 -07006226{
6227 int i, n, val, min_val, best_node = 0;
6228
6229 min_val = INT_MAX;
6230
Mike Travis076ac2a2008-05-12 21:21:12 +02006231 for (i = 0; i < nr_node_ids; i++) {
John Hawkes9c1cfda2005-09-06 15:18:14 -07006232 /* Start at @node */
Mike Travis076ac2a2008-05-12 21:21:12 +02006233 n = (node + i) % nr_node_ids;
John Hawkes9c1cfda2005-09-06 15:18:14 -07006234
6235 if (!nr_cpus_node(n))
6236 continue;
6237
6238 /* Skip already used nodes */
Mike Travisc5f59f02008-04-04 18:11:10 -07006239 if (node_isset(n, *used_nodes))
John Hawkes9c1cfda2005-09-06 15:18:14 -07006240 continue;
6241
6242 /* Simple min distance search */
6243 val = node_distance(node, n);
6244
6245 if (val < min_val) {
6246 min_val = val;
6247 best_node = n;
6248 }
6249 }
6250
Mike Travisc5f59f02008-04-04 18:11:10 -07006251 node_set(best_node, *used_nodes);
John Hawkes9c1cfda2005-09-06 15:18:14 -07006252 return best_node;
6253}
6254
6255/**
6256 * sched_domain_node_span - get a cpumask for a node's sched_domain
6257 * @node: node whose cpumask we're constructing
Randy Dunlap73486722008-04-22 10:07:22 -07006258 * @span: resulting cpumask
John Hawkes9c1cfda2005-09-06 15:18:14 -07006259 *
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01006260 * Given a node, construct a good cpumask for its sched_domain to span. It
John Hawkes9c1cfda2005-09-06 15:18:14 -07006261 * should be one that prevents unnecessary balancing, but also spreads tasks
6262 * out optimally.
6263 */
Rusty Russell96f874e2008-11-25 02:35:14 +10306264static void sched_domain_node_span(int node, struct cpumask *span)
John Hawkes9c1cfda2005-09-06 15:18:14 -07006265{
Mike Travisc5f59f02008-04-04 18:11:10 -07006266 nodemask_t used_nodes;
Ingo Molnar48f24c42006-07-03 00:25:40 -07006267 int i;
John Hawkes9c1cfda2005-09-06 15:18:14 -07006268
Mike Travis6ca09df2008-12-31 18:08:45 -08006269 cpumask_clear(span);
Mike Travisc5f59f02008-04-04 18:11:10 -07006270 nodes_clear(used_nodes);
John Hawkes9c1cfda2005-09-06 15:18:14 -07006271
Mike Travis6ca09df2008-12-31 18:08:45 -08006272 cpumask_or(span, span, cpumask_of_node(node));
Mike Travisc5f59f02008-04-04 18:11:10 -07006273 node_set(node, used_nodes);
John Hawkes9c1cfda2005-09-06 15:18:14 -07006274
6275 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
Mike Travisc5f59f02008-04-04 18:11:10 -07006276 int next_node = find_next_best_node(node, &used_nodes);
Ingo Molnar48f24c42006-07-03 00:25:40 -07006277
Mike Travis6ca09df2008-12-31 18:08:45 -08006278 cpumask_or(span, span, cpumask_of_node(next_node));
John Hawkes9c1cfda2005-09-06 15:18:14 -07006279 }
John Hawkes9c1cfda2005-09-06 15:18:14 -07006280}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02006281#endif /* CONFIG_NUMA */
John Hawkes9c1cfda2005-09-06 15:18:14 -07006282
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07006283int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
Ingo Molnar48f24c42006-07-03 00:25:40 -07006284
John Hawkes9c1cfda2005-09-06 15:18:14 -07006285/*
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306286 * The cpus mask in sched_group and sched_domain hangs off the end.
Ingo Molnar4200efd2009-05-19 09:22:19 +02006287 *
6288 * ( See the the comments in include/linux/sched.h:struct sched_group
6289 * and struct sched_domain. )
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306290 */
6291struct static_sched_group {
6292 struct sched_group sg;
6293 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
6294};
6295
6296struct static_sched_domain {
6297 struct sched_domain sd;
6298 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
6299};
6300
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006301struct s_data {
6302#ifdef CONFIG_NUMA
6303 int sd_allnodes;
6304 cpumask_var_t domainspan;
6305 cpumask_var_t covered;
6306 cpumask_var_t notcovered;
6307#endif
6308 cpumask_var_t nodemask;
6309 cpumask_var_t this_sibling_map;
6310 cpumask_var_t this_core_map;
6311 cpumask_var_t send_covered;
6312 cpumask_var_t tmpmask;
6313 struct sched_group **sched_group_nodes;
6314 struct root_domain *rd;
6315};
6316
Andreas Herrmann2109b992009-08-18 12:53:00 +02006317enum s_alloc {
6318 sa_sched_groups = 0,
6319 sa_rootdomain,
6320 sa_tmpmask,
6321 sa_send_covered,
6322 sa_this_core_map,
6323 sa_this_sibling_map,
6324 sa_nodemask,
6325 sa_sched_group_nodes,
6326#ifdef CONFIG_NUMA
6327 sa_notcovered,
6328 sa_covered,
6329 sa_domainspan,
6330#endif
6331 sa_none,
6332};
6333
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306334/*
Ingo Molnar48f24c42006-07-03 00:25:40 -07006335 * SMT sched-domains:
John Hawkes9c1cfda2005-09-06 15:18:14 -07006336 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006337#ifdef CONFIG_SCHED_SMT
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306338static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
Tejun Heo1871e522009-10-29 22:34:13 +09006339static DEFINE_PER_CPU(struct static_sched_group, sched_groups);
Ingo Molnar48f24c42006-07-03 00:25:40 -07006340
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01006341static int
Rusty Russell96f874e2008-11-25 02:35:14 +10306342cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
6343 struct sched_group **sg, struct cpumask *unused)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006344{
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006345 if (sg)
Tejun Heo1871e522009-10-29 22:34:13 +09006346 *sg = &per_cpu(sched_groups, cpu).sg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006347 return cpu;
6348}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02006349#endif /* CONFIG_SCHED_SMT */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006350
Ingo Molnar48f24c42006-07-03 00:25:40 -07006351/*
6352 * multi-core sched-domains:
6353 */
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006354#ifdef CONFIG_SCHED_MC
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306355static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
6356static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02006357#endif /* CONFIG_SCHED_MC */
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006358
6359#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01006360static int
Rusty Russell96f874e2008-11-25 02:35:14 +10306361cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
6362 struct sched_group **sg, struct cpumask *mask)
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006363{
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006364 int group;
Mike Travis7c16ec52008-04-04 18:11:11 -07006365
Rusty Russellc69fc562009-03-13 14:49:46 +10306366 cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
Rusty Russell96f874e2008-11-25 02:35:14 +10306367 group = cpumask_first(mask);
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006368 if (sg)
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306369 *sg = &per_cpu(sched_group_core, group).sg;
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006370 return group;
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006371}
6372#elif defined(CONFIG_SCHED_MC)
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01006373static int
Rusty Russell96f874e2008-11-25 02:35:14 +10306374cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
6375 struct sched_group **sg, struct cpumask *unused)
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006376{
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006377 if (sg)
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306378 *sg = &per_cpu(sched_group_core, cpu).sg;
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006379 return cpu;
6380}
6381#endif
6382
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306383static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
6384static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
Ingo Molnar48f24c42006-07-03 00:25:40 -07006385
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01006386static int
Rusty Russell96f874e2008-11-25 02:35:14 +10306387cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
6388 struct sched_group **sg, struct cpumask *mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006389{
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006390 int group;
Ingo Molnar48f24c42006-07-03 00:25:40 -07006391#ifdef CONFIG_SCHED_MC
Mike Travis6ca09df2008-12-31 18:08:45 -08006392 cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
Rusty Russell96f874e2008-11-25 02:35:14 +10306393 group = cpumask_first(mask);
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006394#elif defined(CONFIG_SCHED_SMT)
Rusty Russellc69fc562009-03-13 14:49:46 +10306395 cpumask_and(mask, topology_thread_cpumask(cpu), cpu_map);
Rusty Russell96f874e2008-11-25 02:35:14 +10306396 group = cpumask_first(mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006397#else
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006398 group = cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006399#endif
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006400 if (sg)
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306401 *sg = &per_cpu(sched_group_phys, group).sg;
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006402 return group;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006403}
6404
6405#ifdef CONFIG_NUMA
John Hawkes9c1cfda2005-09-06 15:18:14 -07006406/*
6407 * The init_sched_build_groups can't handle what we want to do with node
6408 * groups, so roll our own. Now each node has its own list of groups which
6409 * gets dynamically allocated.
6410 */
Rusty Russell62ea9ce2009-01-11 01:04:16 +01006411static DEFINE_PER_CPU(struct static_sched_domain, node_domains);
Mike Travis434d53b2008-04-04 18:11:04 -07006412static struct sched_group ***sched_group_nodes_bycpu;
John Hawkes9c1cfda2005-09-06 15:18:14 -07006413
Rusty Russell62ea9ce2009-01-11 01:04:16 +01006414static DEFINE_PER_CPU(struct static_sched_domain, allnodes_domains);
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306415static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
John Hawkes9c1cfda2005-09-06 15:18:14 -07006416
Rusty Russell96f874e2008-11-25 02:35:14 +10306417static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
6418 struct sched_group **sg,
6419 struct cpumask *nodemask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006420{
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006421 int group;
6422
Mike Travis6ca09df2008-12-31 18:08:45 -08006423 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
Rusty Russell96f874e2008-11-25 02:35:14 +10306424 group = cpumask_first(nodemask);
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006425
6426 if (sg)
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306427 *sg = &per_cpu(sched_group_allnodes, group).sg;
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006428 return group;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006429}
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006430
Siddha, Suresh B08069032006-03-27 01:15:23 -08006431static void init_numa_sched_groups_power(struct sched_group *group_head)
6432{
6433 struct sched_group *sg = group_head;
6434 int j;
6435
6436 if (!sg)
6437 return;
Andi Kleen3a5c3592007-10-15 17:00:14 +02006438 do {
Rusty Russell758b2cd2008-11-25 02:35:04 +10306439 for_each_cpu(j, sched_group_cpus(sg)) {
Andi Kleen3a5c3592007-10-15 17:00:14 +02006440 struct sched_domain *sd;
Siddha, Suresh B08069032006-03-27 01:15:23 -08006441
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306442 sd = &per_cpu(phys_domains, j).sd;
Miao Xie13318a72009-04-15 09:59:10 +08006443 if (j != group_first_cpu(sd->groups)) {
Andi Kleen3a5c3592007-10-15 17:00:14 +02006444 /*
6445 * Only add "power" once for each
6446 * physical package.
6447 */
6448 continue;
6449 }
6450
Peter Zijlstra18a38852009-09-01 10:34:39 +02006451 sg->cpu_power += sd->groups->cpu_power;
Siddha, Suresh B08069032006-03-27 01:15:23 -08006452 }
Andi Kleen3a5c3592007-10-15 17:00:14 +02006453 sg = sg->next;
6454 } while (sg != group_head);
Siddha, Suresh B08069032006-03-27 01:15:23 -08006455}
Andreas Herrmann0601a882009-08-18 13:01:11 +02006456
6457static int build_numa_sched_groups(struct s_data *d,
6458 const struct cpumask *cpu_map, int num)
6459{
6460 struct sched_domain *sd;
6461 struct sched_group *sg, *prev;
6462 int n, j;
6463
6464 cpumask_clear(d->covered);
6465 cpumask_and(d->nodemask, cpumask_of_node(num), cpu_map);
6466 if (cpumask_empty(d->nodemask)) {
6467 d->sched_group_nodes[num] = NULL;
6468 goto out;
6469 }
6470
6471 sched_domain_node_span(num, d->domainspan);
6472 cpumask_and(d->domainspan, d->domainspan, cpu_map);
6473
6474 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
6475 GFP_KERNEL, num);
6476 if (!sg) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01006477 printk(KERN_WARNING "Can not alloc domain group for node %d\n",
6478 num);
Andreas Herrmann0601a882009-08-18 13:01:11 +02006479 return -ENOMEM;
6480 }
6481 d->sched_group_nodes[num] = sg;
6482
6483 for_each_cpu(j, d->nodemask) {
6484 sd = &per_cpu(node_domains, j).sd;
6485 sd->groups = sg;
6486 }
6487
Peter Zijlstra18a38852009-09-01 10:34:39 +02006488 sg->cpu_power = 0;
Andreas Herrmann0601a882009-08-18 13:01:11 +02006489 cpumask_copy(sched_group_cpus(sg), d->nodemask);
6490 sg->next = sg;
6491 cpumask_or(d->covered, d->covered, d->nodemask);
6492
6493 prev = sg;
6494 for (j = 0; j < nr_node_ids; j++) {
6495 n = (num + j) % nr_node_ids;
6496 cpumask_complement(d->notcovered, d->covered);
6497 cpumask_and(d->tmpmask, d->notcovered, cpu_map);
6498 cpumask_and(d->tmpmask, d->tmpmask, d->domainspan);
6499 if (cpumask_empty(d->tmpmask))
6500 break;
6501 cpumask_and(d->tmpmask, d->tmpmask, cpumask_of_node(n));
6502 if (cpumask_empty(d->tmpmask))
6503 continue;
6504 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
6505 GFP_KERNEL, num);
6506 if (!sg) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01006507 printk(KERN_WARNING
6508 "Can not alloc domain group for node %d\n", j);
Andreas Herrmann0601a882009-08-18 13:01:11 +02006509 return -ENOMEM;
6510 }
Peter Zijlstra18a38852009-09-01 10:34:39 +02006511 sg->cpu_power = 0;
Andreas Herrmann0601a882009-08-18 13:01:11 +02006512 cpumask_copy(sched_group_cpus(sg), d->tmpmask);
6513 sg->next = prev->next;
6514 cpumask_or(d->covered, d->covered, d->tmpmask);
6515 prev->next = sg;
6516 prev = sg;
6517 }
6518out:
6519 return 0;
6520}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02006521#endif /* CONFIG_NUMA */
Linus Torvalds1da177e2005-04-16 15:20:36 -07006522
Siddha, Suresh Ba6160582006-10-03 01:14:06 -07006523#ifdef CONFIG_NUMA
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006524/* Free memory allocated for various sched_group structures */
Rusty Russell96f874e2008-11-25 02:35:14 +10306525static void free_sched_groups(const struct cpumask *cpu_map,
6526 struct cpumask *nodemask)
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006527{
Siddha, Suresh Ba6160582006-10-03 01:14:06 -07006528 int cpu, i;
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006529
Rusty Russellabcd0832008-11-25 02:35:02 +10306530 for_each_cpu(cpu, cpu_map) {
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006531 struct sched_group **sched_group_nodes
6532 = sched_group_nodes_bycpu[cpu];
6533
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006534 if (!sched_group_nodes)
6535 continue;
6536
Mike Travis076ac2a2008-05-12 21:21:12 +02006537 for (i = 0; i < nr_node_ids; i++) {
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006538 struct sched_group *oldsg, *sg = sched_group_nodes[i];
6539
Mike Travis6ca09df2008-12-31 18:08:45 -08006540 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
Rusty Russell96f874e2008-11-25 02:35:14 +10306541 if (cpumask_empty(nodemask))
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006542 continue;
6543
6544 if (sg == NULL)
6545 continue;
6546 sg = sg->next;
6547next_sg:
6548 oldsg = sg;
6549 sg = sg->next;
6550 kfree(oldsg);
6551 if (oldsg != sched_group_nodes[i])
6552 goto next_sg;
6553 }
6554 kfree(sched_group_nodes);
6555 sched_group_nodes_bycpu[cpu] = NULL;
6556 }
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006557}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02006558#else /* !CONFIG_NUMA */
Rusty Russell96f874e2008-11-25 02:35:14 +10306559static void free_sched_groups(const struct cpumask *cpu_map,
6560 struct cpumask *nodemask)
Siddha, Suresh Ba6160582006-10-03 01:14:06 -07006561{
6562}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02006563#endif /* CONFIG_NUMA */
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006564
Linus Torvalds1da177e2005-04-16 15:20:36 -07006565/*
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006566 * Initialize sched groups cpu_power.
6567 *
6568 * cpu_power indicates the capacity of sched group, which is used while
6569 * distributing the load between different sched groups in a sched domain.
6570 * Typically cpu_power for all the groups in a sched domain will be same unless
6571 * there are asymmetries in the topology. If there are asymmetries, group
6572 * having more cpu_power will pickup more load compared to the group having
6573 * less cpu_power.
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006574 */
6575static void init_sched_groups_power(int cpu, struct sched_domain *sd)
6576{
6577 struct sched_domain *child;
6578 struct sched_group *group;
Peter Zijlstraf93e65c2009-09-01 10:34:32 +02006579 long power;
6580 int weight;
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006581
6582 WARN_ON(!sd || !sd->groups);
6583
Miao Xie13318a72009-04-15 09:59:10 +08006584 if (cpu != group_first_cpu(sd->groups))
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006585 return;
6586
6587 child = sd->child;
6588
Peter Zijlstra18a38852009-09-01 10:34:39 +02006589 sd->groups->cpu_power = 0;
Eric Dumazet5517d862007-05-08 00:32:57 -07006590
Peter Zijlstraf93e65c2009-09-01 10:34:32 +02006591 if (!child) {
6592 power = SCHED_LOAD_SCALE;
6593 weight = cpumask_weight(sched_domain_span(sd));
6594 /*
6595 * SMT siblings share the power of a single core.
Peter Zijlstraa52bfd72009-09-01 10:34:35 +02006596 * Usually multiple threads get a better yield out of
6597 * that one core than a single thread would have,
6598 * reflect that in sd->smt_gain.
Peter Zijlstraf93e65c2009-09-01 10:34:32 +02006599 */
Peter Zijlstraa52bfd72009-09-01 10:34:35 +02006600 if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
6601 power *= sd->smt_gain;
Peter Zijlstraf93e65c2009-09-01 10:34:32 +02006602 power /= weight;
Peter Zijlstraa52bfd72009-09-01 10:34:35 +02006603 power >>= SCHED_LOAD_SHIFT;
6604 }
Peter Zijlstra18a38852009-09-01 10:34:39 +02006605 sd->groups->cpu_power += power;
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006606 return;
6607 }
6608
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006609 /*
Peter Zijlstraf93e65c2009-09-01 10:34:32 +02006610 * Add cpu_power of each child group to this groups cpu_power.
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006611 */
6612 group = child->groups;
6613 do {
Peter Zijlstra18a38852009-09-01 10:34:39 +02006614 sd->groups->cpu_power += group->cpu_power;
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006615 group = group->next;
6616 } while (group != child->groups);
6617}
6618
6619/*
Mike Travis7c16ec52008-04-04 18:11:11 -07006620 * Initializers for schedule domains
6621 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
6622 */
6623
Ingo Molnara5d8c342008-10-09 11:35:51 +02006624#ifdef CONFIG_SCHED_DEBUG
6625# define SD_INIT_NAME(sd, type) sd->name = #type
6626#else
6627# define SD_INIT_NAME(sd, type) do { } while (0)
6628#endif
6629
Mike Travis7c16ec52008-04-04 18:11:11 -07006630#define SD_INIT(sd, type) sd_init_##type(sd)
Ingo Molnara5d8c342008-10-09 11:35:51 +02006631
Mike Travis7c16ec52008-04-04 18:11:11 -07006632#define SD_INIT_FUNC(type) \
6633static noinline void sd_init_##type(struct sched_domain *sd) \
6634{ \
6635 memset(sd, 0, sizeof(*sd)); \
6636 *sd = SD_##type##_INIT; \
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09006637 sd->level = SD_LV_##type; \
Ingo Molnara5d8c342008-10-09 11:35:51 +02006638 SD_INIT_NAME(sd, type); \
Mike Travis7c16ec52008-04-04 18:11:11 -07006639}
6640
6641SD_INIT_FUNC(CPU)
6642#ifdef CONFIG_NUMA
6643 SD_INIT_FUNC(ALLNODES)
6644 SD_INIT_FUNC(NODE)
6645#endif
6646#ifdef CONFIG_SCHED_SMT
6647 SD_INIT_FUNC(SIBLING)
6648#endif
6649#ifdef CONFIG_SCHED_MC
6650 SD_INIT_FUNC(MC)
6651#endif
6652
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09006653static int default_relax_domain_level = -1;
6654
6655static int __init setup_relax_domain_level(char *str)
6656{
Li Zefan30e0e172008-05-13 10:27:17 +08006657 unsigned long val;
6658
6659 val = simple_strtoul(str, NULL, 0);
6660 if (val < SD_LV_MAX)
6661 default_relax_domain_level = val;
6662
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09006663 return 1;
6664}
6665__setup("relax_domain_level=", setup_relax_domain_level);
6666
6667static void set_domain_attribute(struct sched_domain *sd,
6668 struct sched_domain_attr *attr)
6669{
6670 int request;
6671
6672 if (!attr || attr->relax_domain_level < 0) {
6673 if (default_relax_domain_level < 0)
6674 return;
6675 else
6676 request = default_relax_domain_level;
6677 } else
6678 request = attr->relax_domain_level;
6679 if (request < sd->level) {
6680 /* turn off idle balance on this domain */
Peter Zijlstrac88d5912009-09-10 13:50:02 +02006681 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09006682 } else {
6683 /* turn on idle balance on this domain */
Peter Zijlstrac88d5912009-09-10 13:50:02 +02006684 sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09006685 }
6686}
6687
Andreas Herrmann2109b992009-08-18 12:53:00 +02006688static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
6689 const struct cpumask *cpu_map)
6690{
6691 switch (what) {
6692 case sa_sched_groups:
6693 free_sched_groups(cpu_map, d->tmpmask); /* fall through */
6694 d->sched_group_nodes = NULL;
6695 case sa_rootdomain:
6696 free_rootdomain(d->rd); /* fall through */
6697 case sa_tmpmask:
6698 free_cpumask_var(d->tmpmask); /* fall through */
6699 case sa_send_covered:
6700 free_cpumask_var(d->send_covered); /* fall through */
6701 case sa_this_core_map:
6702 free_cpumask_var(d->this_core_map); /* fall through */
6703 case sa_this_sibling_map:
6704 free_cpumask_var(d->this_sibling_map); /* fall through */
6705 case sa_nodemask:
6706 free_cpumask_var(d->nodemask); /* fall through */
6707 case sa_sched_group_nodes:
6708#ifdef CONFIG_NUMA
6709 kfree(d->sched_group_nodes); /* fall through */
6710 case sa_notcovered:
6711 free_cpumask_var(d->notcovered); /* fall through */
6712 case sa_covered:
6713 free_cpumask_var(d->covered); /* fall through */
6714 case sa_domainspan:
6715 free_cpumask_var(d->domainspan); /* fall through */
6716#endif
6717 case sa_none:
6718 break;
6719 }
6720}
6721
6722static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
6723 const struct cpumask *cpu_map)
6724{
6725#ifdef CONFIG_NUMA
6726 if (!alloc_cpumask_var(&d->domainspan, GFP_KERNEL))
6727 return sa_none;
6728 if (!alloc_cpumask_var(&d->covered, GFP_KERNEL))
6729 return sa_domainspan;
6730 if (!alloc_cpumask_var(&d->notcovered, GFP_KERNEL))
6731 return sa_covered;
6732 /* Allocate the per-node list of sched groups */
6733 d->sched_group_nodes = kcalloc(nr_node_ids,
6734 sizeof(struct sched_group *), GFP_KERNEL);
6735 if (!d->sched_group_nodes) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01006736 printk(KERN_WARNING "Can not alloc sched group node list\n");
Andreas Herrmann2109b992009-08-18 12:53:00 +02006737 return sa_notcovered;
6738 }
6739 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = d->sched_group_nodes;
6740#endif
6741 if (!alloc_cpumask_var(&d->nodemask, GFP_KERNEL))
6742 return sa_sched_group_nodes;
6743 if (!alloc_cpumask_var(&d->this_sibling_map, GFP_KERNEL))
6744 return sa_nodemask;
6745 if (!alloc_cpumask_var(&d->this_core_map, GFP_KERNEL))
6746 return sa_this_sibling_map;
6747 if (!alloc_cpumask_var(&d->send_covered, GFP_KERNEL))
6748 return sa_this_core_map;
6749 if (!alloc_cpumask_var(&d->tmpmask, GFP_KERNEL))
6750 return sa_send_covered;
6751 d->rd = alloc_rootdomain();
6752 if (!d->rd) {
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01006753 printk(KERN_WARNING "Cannot alloc root domain\n");
Andreas Herrmann2109b992009-08-18 12:53:00 +02006754 return sa_tmpmask;
6755 }
6756 return sa_rootdomain;
6757}
6758
Andreas Herrmann7f4588f2009-08-18 12:54:06 +02006759static struct sched_domain *__build_numa_sched_domains(struct s_data *d,
6760 const struct cpumask *cpu_map, struct sched_domain_attr *attr, int i)
6761{
6762 struct sched_domain *sd = NULL;
6763#ifdef CONFIG_NUMA
6764 struct sched_domain *parent;
6765
6766 d->sd_allnodes = 0;
6767 if (cpumask_weight(cpu_map) >
6768 SD_NODES_PER_DOMAIN * cpumask_weight(d->nodemask)) {
6769 sd = &per_cpu(allnodes_domains, i).sd;
6770 SD_INIT(sd, ALLNODES);
6771 set_domain_attribute(sd, attr);
6772 cpumask_copy(sched_domain_span(sd), cpu_map);
6773 cpu_to_allnodes_group(i, cpu_map, &sd->groups, d->tmpmask);
6774 d->sd_allnodes = 1;
6775 }
6776 parent = sd;
6777
6778 sd = &per_cpu(node_domains, i).sd;
6779 SD_INIT(sd, NODE);
6780 set_domain_attribute(sd, attr);
6781 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
6782 sd->parent = parent;
6783 if (parent)
6784 parent->child = sd;
6785 cpumask_and(sched_domain_span(sd), sched_domain_span(sd), cpu_map);
6786#endif
6787 return sd;
6788}
6789
Andreas Herrmann87cce662009-08-18 12:54:55 +02006790static struct sched_domain *__build_cpu_sched_domain(struct s_data *d,
6791 const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6792 struct sched_domain *parent, int i)
6793{
6794 struct sched_domain *sd;
6795 sd = &per_cpu(phys_domains, i).sd;
6796 SD_INIT(sd, CPU);
6797 set_domain_attribute(sd, attr);
6798 cpumask_copy(sched_domain_span(sd), d->nodemask);
6799 sd->parent = parent;
6800 if (parent)
6801 parent->child = sd;
6802 cpu_to_phys_group(i, cpu_map, &sd->groups, d->tmpmask);
6803 return sd;
6804}
6805
Andreas Herrmann410c4082009-08-18 12:56:14 +02006806static struct sched_domain *__build_mc_sched_domain(struct s_data *d,
6807 const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6808 struct sched_domain *parent, int i)
6809{
6810 struct sched_domain *sd = parent;
6811#ifdef CONFIG_SCHED_MC
6812 sd = &per_cpu(core_domains, i).sd;
6813 SD_INIT(sd, MC);
6814 set_domain_attribute(sd, attr);
6815 cpumask_and(sched_domain_span(sd), cpu_map, cpu_coregroup_mask(i));
6816 sd->parent = parent;
6817 parent->child = sd;
6818 cpu_to_core_group(i, cpu_map, &sd->groups, d->tmpmask);
6819#endif
6820 return sd;
6821}
6822
Andreas Herrmannd8173532009-08-18 12:57:03 +02006823static struct sched_domain *__build_smt_sched_domain(struct s_data *d,
6824 const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6825 struct sched_domain *parent, int i)
6826{
6827 struct sched_domain *sd = parent;
6828#ifdef CONFIG_SCHED_SMT
6829 sd = &per_cpu(cpu_domains, i).sd;
6830 SD_INIT(sd, SIBLING);
6831 set_domain_attribute(sd, attr);
6832 cpumask_and(sched_domain_span(sd), cpu_map, topology_thread_cpumask(i));
6833 sd->parent = parent;
6834 parent->child = sd;
6835 cpu_to_cpu_group(i, cpu_map, &sd->groups, d->tmpmask);
6836#endif
6837 return sd;
6838}
6839
Andreas Herrmann0e8e85c2009-08-18 12:57:51 +02006840static void build_sched_groups(struct s_data *d, enum sched_domain_level l,
6841 const struct cpumask *cpu_map, int cpu)
6842{
6843 switch (l) {
6844#ifdef CONFIG_SCHED_SMT
6845 case SD_LV_SIBLING: /* set up CPU (sibling) groups */
6846 cpumask_and(d->this_sibling_map, cpu_map,
6847 topology_thread_cpumask(cpu));
6848 if (cpu == cpumask_first(d->this_sibling_map))
6849 init_sched_build_groups(d->this_sibling_map, cpu_map,
6850 &cpu_to_cpu_group,
6851 d->send_covered, d->tmpmask);
6852 break;
6853#endif
Andreas Herrmanna2af04c2009-08-18 12:58:38 +02006854#ifdef CONFIG_SCHED_MC
6855 case SD_LV_MC: /* set up multi-core groups */
6856 cpumask_and(d->this_core_map, cpu_map, cpu_coregroup_mask(cpu));
6857 if (cpu == cpumask_first(d->this_core_map))
6858 init_sched_build_groups(d->this_core_map, cpu_map,
6859 &cpu_to_core_group,
6860 d->send_covered, d->tmpmask);
6861 break;
6862#endif
Andreas Herrmann86548092009-08-18 12:59:28 +02006863 case SD_LV_CPU: /* set up physical groups */
6864 cpumask_and(d->nodemask, cpumask_of_node(cpu), cpu_map);
6865 if (!cpumask_empty(d->nodemask))
6866 init_sched_build_groups(d->nodemask, cpu_map,
6867 &cpu_to_phys_group,
6868 d->send_covered, d->tmpmask);
6869 break;
Andreas Herrmannde616e32009-08-18 13:00:13 +02006870#ifdef CONFIG_NUMA
6871 case SD_LV_ALLNODES:
6872 init_sched_build_groups(cpu_map, cpu_map, &cpu_to_allnodes_group,
6873 d->send_covered, d->tmpmask);
6874 break;
6875#endif
Andreas Herrmann0e8e85c2009-08-18 12:57:51 +02006876 default:
6877 break;
6878 }
6879}
6880
Mike Travis7c16ec52008-04-04 18:11:11 -07006881/*
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07006882 * Build sched domains for a given set of cpus and attach the sched domains
6883 * to the individual cpus
Linus Torvalds1da177e2005-04-16 15:20:36 -07006884 */
Rusty Russell96f874e2008-11-25 02:35:14 +10306885static int __build_sched_domains(const struct cpumask *cpu_map,
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09006886 struct sched_domain_attr *attr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07006887{
Andreas Herrmann2109b992009-08-18 12:53:00 +02006888 enum s_alloc alloc_state = sa_none;
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006889 struct s_data d;
Andreas Herrmann294b0c92009-08-18 13:02:29 +02006890 struct sched_domain *sd;
Andreas Herrmann2109b992009-08-18 12:53:00 +02006891 int i;
John Hawkesd1b55132005-09-06 15:18:14 -07006892#ifdef CONFIG_NUMA
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006893 d.sd_allnodes = 0;
Rusty Russell3404c8d2008-11-25 02:35:03 +10306894#endif
6895
Andreas Herrmann2109b992009-08-18 12:53:00 +02006896 alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
6897 if (alloc_state != sa_rootdomain)
6898 goto error;
6899 alloc_state = sa_sched_groups;
Mike Travis7c16ec52008-04-04 18:11:11 -07006900
Linus Torvalds1da177e2005-04-16 15:20:36 -07006901 /*
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07006902 * Set up domains for cpus specified by the cpu_map.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006903 */
Rusty Russellabcd0832008-11-25 02:35:02 +10306904 for_each_cpu(i, cpu_map) {
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006905 cpumask_and(d.nodemask, cpumask_of_node(cpu_to_node(i)),
6906 cpu_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006907
Andreas Herrmann7f4588f2009-08-18 12:54:06 +02006908 sd = __build_numa_sched_domains(&d, cpu_map, attr, i);
Andreas Herrmann87cce662009-08-18 12:54:55 +02006909 sd = __build_cpu_sched_domain(&d, cpu_map, attr, sd, i);
Andreas Herrmann410c4082009-08-18 12:56:14 +02006910 sd = __build_mc_sched_domain(&d, cpu_map, attr, sd, i);
Andreas Herrmannd8173532009-08-18 12:57:03 +02006911 sd = __build_smt_sched_domain(&d, cpu_map, attr, sd, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006912 }
6913
Rusty Russellabcd0832008-11-25 02:35:02 +10306914 for_each_cpu(i, cpu_map) {
Andreas Herrmann0e8e85c2009-08-18 12:57:51 +02006915 build_sched_groups(&d, SD_LV_SIBLING, cpu_map, i);
Andreas Herrmanna2af04c2009-08-18 12:58:38 +02006916 build_sched_groups(&d, SD_LV_MC, cpu_map, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006917 }
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006918
Linus Torvalds1da177e2005-04-16 15:20:36 -07006919 /* Set up physical groups */
Andreas Herrmann86548092009-08-18 12:59:28 +02006920 for (i = 0; i < nr_node_ids; i++)
6921 build_sched_groups(&d, SD_LV_CPU, cpu_map, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006922
6923#ifdef CONFIG_NUMA
6924 /* Set up node groups */
Andreas Herrmannde616e32009-08-18 13:00:13 +02006925 if (d.sd_allnodes)
6926 build_sched_groups(&d, SD_LV_ALLNODES, cpu_map, 0);
John Hawkes9c1cfda2005-09-06 15:18:14 -07006927
Andreas Herrmann0601a882009-08-18 13:01:11 +02006928 for (i = 0; i < nr_node_ids; i++)
6929 if (build_numa_sched_groups(&d, cpu_map, i))
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006930 goto error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006931#endif
6932
6933 /* Calculate CPU power for physical packages and nodes */
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07006934#ifdef CONFIG_SCHED_SMT
Rusty Russellabcd0832008-11-25 02:35:02 +10306935 for_each_cpu(i, cpu_map) {
Andreas Herrmann294b0c92009-08-18 13:02:29 +02006936 sd = &per_cpu(cpu_domains, i).sd;
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006937 init_sched_groups_power(i, sd);
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07006938 }
6939#endif
6940#ifdef CONFIG_SCHED_MC
Rusty Russellabcd0832008-11-25 02:35:02 +10306941 for_each_cpu(i, cpu_map) {
Andreas Herrmann294b0c92009-08-18 13:02:29 +02006942 sd = &per_cpu(core_domains, i).sd;
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006943 init_sched_groups_power(i, sd);
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07006944 }
6945#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07006946
Rusty Russellabcd0832008-11-25 02:35:02 +10306947 for_each_cpu(i, cpu_map) {
Andreas Herrmann294b0c92009-08-18 13:02:29 +02006948 sd = &per_cpu(phys_domains, i).sd;
Siddha, Suresh B89c47102006-10-03 01:14:09 -07006949 init_sched_groups_power(i, sd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006950 }
6951
John Hawkes9c1cfda2005-09-06 15:18:14 -07006952#ifdef CONFIG_NUMA
Mike Travis076ac2a2008-05-12 21:21:12 +02006953 for (i = 0; i < nr_node_ids; i++)
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006954 init_numa_sched_groups_power(d.sched_group_nodes[i]);
John Hawkes9c1cfda2005-09-06 15:18:14 -07006955
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006956 if (d.sd_allnodes) {
Siddha, Suresh B6711cab2006-12-10 02:20:07 -08006957 struct sched_group *sg;
Siddha, Suresh Bf712c0c2006-07-30 03:02:59 -07006958
Rusty Russell96f874e2008-11-25 02:35:14 +10306959 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006960 d.tmpmask);
Siddha, Suresh Bf712c0c2006-07-30 03:02:59 -07006961 init_numa_sched_groups_power(sg);
6962 }
John Hawkes9c1cfda2005-09-06 15:18:14 -07006963#endif
6964
Linus Torvalds1da177e2005-04-16 15:20:36 -07006965 /* Attach the domains */
Rusty Russellabcd0832008-11-25 02:35:02 +10306966 for_each_cpu(i, cpu_map) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07006967#ifdef CONFIG_SCHED_SMT
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306968 sd = &per_cpu(cpu_domains, i).sd;
Siddha, Suresh B1e9f28f2006-03-27 01:15:22 -08006969#elif defined(CONFIG_SCHED_MC)
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306970 sd = &per_cpu(core_domains, i).sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006971#else
Rusty Russell6c99e9a2008-11-25 02:35:04 +10306972 sd = &per_cpu(phys_domains, i).sd;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006973#endif
Andreas Herrmann49a02c52009-08-18 12:51:52 +02006974 cpu_attach_domain(sd, d.rd, i);
Linus Torvalds1da177e2005-04-16 15:20:36 -07006975 }
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006976
Andreas Herrmann2109b992009-08-18 12:53:00 +02006977 d.sched_group_nodes = NULL; /* don't free this we still need it */
6978 __free_domain_allocs(&d, sa_tmpmask, cpu_map);
6979 return 0;
Rusty Russell3404c8d2008-11-25 02:35:03 +10306980
Srivatsa Vaddagiri51888ca2006-06-27 02:54:38 -07006981error:
Andreas Herrmann2109b992009-08-18 12:53:00 +02006982 __free_domain_allocs(&d, alloc_state, cpu_map);
6983 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07006984}
Paul Jackson029190c2007-10-18 23:40:20 -07006985
Rusty Russell96f874e2008-11-25 02:35:14 +10306986static int build_sched_domains(const struct cpumask *cpu_map)
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09006987{
6988 return __build_sched_domains(cpu_map, NULL);
6989}
6990
Rusty Russellacc3f5d2009-11-03 14:53:40 +10306991static cpumask_var_t *doms_cur; /* current sched domains */
Paul Jackson029190c2007-10-18 23:40:20 -07006992static int ndoms_cur; /* number of sched domains in 'doms_cur' */
Ingo Molnar4285f5942008-05-16 17:47:14 +02006993static struct sched_domain_attr *dattr_cur;
6994 /* attribues of custom domains in 'doms_cur' */
Paul Jackson029190c2007-10-18 23:40:20 -07006995
6996/*
6997 * Special case: If a kmalloc of a doms_cur partition (array of
Rusty Russell42128232008-11-25 02:35:12 +10306998 * cpumask) fails, then fallback to a single sched domain,
6999 * as determined by the single cpumask fallback_doms.
Paul Jackson029190c2007-10-18 23:40:20 -07007000 */
Rusty Russell42128232008-11-25 02:35:12 +10307001static cpumask_var_t fallback_doms;
Paul Jackson029190c2007-10-18 23:40:20 -07007002
Heiko Carstensee79d1b2008-12-09 18:49:50 +01007003/*
7004 * arch_update_cpu_topology lets virtualized architectures update the
7005 * cpu core maps. It is supposed to return 1 if the topology changed
7006 * or 0 if it stayed the same.
7007 */
7008int __attribute__((weak)) arch_update_cpu_topology(void)
Heiko Carstens22e52b02008-03-12 18:31:59 +01007009{
Heiko Carstensee79d1b2008-12-09 18:49:50 +01007010 return 0;
Heiko Carstens22e52b02008-03-12 18:31:59 +01007011}
7012
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307013cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
7014{
7015 int i;
7016 cpumask_var_t *doms;
7017
7018 doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
7019 if (!doms)
7020 return NULL;
7021 for (i = 0; i < ndoms; i++) {
7022 if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
7023 free_sched_domains(doms, i);
7024 return NULL;
7025 }
7026 }
7027 return doms;
7028}
7029
7030void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
7031{
7032 unsigned int i;
7033 for (i = 0; i < ndoms; i++)
7034 free_cpumask_var(doms[i]);
7035 kfree(doms);
7036}
7037
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007038/*
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01007039 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
Paul Jackson029190c2007-10-18 23:40:20 -07007040 * For now this just excludes isolated cpus, but could be used to
7041 * exclude other special cases in the future.
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007042 */
Rusty Russell96f874e2008-11-25 02:35:14 +10307043static int arch_init_sched_domains(const struct cpumask *cpu_map)
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007044{
Milton Miller73785472007-10-24 18:23:48 +02007045 int err;
7046
Heiko Carstens22e52b02008-03-12 18:31:59 +01007047 arch_update_cpu_topology();
Paul Jackson029190c2007-10-18 23:40:20 -07007048 ndoms_cur = 1;
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307049 doms_cur = alloc_sched_domains(ndoms_cur);
Paul Jackson029190c2007-10-18 23:40:20 -07007050 if (!doms_cur)
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307051 doms_cur = &fallback_doms;
7052 cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007053 dattr_cur = NULL;
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307054 err = build_sched_domains(doms_cur[0]);
Milton Miller6382bc92007-10-15 17:00:19 +02007055 register_sched_domain_sysctl();
Milton Miller73785472007-10-24 18:23:48 +02007056
7057 return err;
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007058}
7059
Rusty Russell96f874e2008-11-25 02:35:14 +10307060static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7061 struct cpumask *tmpmask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007062{
Mike Travis7c16ec52008-04-04 18:11:11 -07007063 free_sched_groups(cpu_map, tmpmask);
John Hawkes9c1cfda2005-09-06 15:18:14 -07007064}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007065
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007066/*
7067 * Detach sched domains from a group of cpus specified in cpu_map
7068 * These cpus will now be attached to the NULL domain
7069 */
Rusty Russell96f874e2008-11-25 02:35:14 +10307070static void detach_destroy_domains(const struct cpumask *cpu_map)
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007071{
Rusty Russell96f874e2008-11-25 02:35:14 +10307072 /* Save because hotplug lock held. */
7073 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007074 int i;
7075
Rusty Russellabcd0832008-11-25 02:35:02 +10307076 for_each_cpu(i, cpu_map)
Gregory Haskins57d885f2008-01-25 21:08:18 +01007077 cpu_attach_domain(NULL, &def_root_domain, i);
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007078 synchronize_sched();
Rusty Russell96f874e2008-11-25 02:35:14 +10307079 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
Dinakar Guniguntala1a20ff22005-06-25 14:57:33 -07007080}
7081
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007082/* handle null as "default" */
7083static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7084 struct sched_domain_attr *new, int idx_new)
7085{
7086 struct sched_domain_attr tmp;
7087
7088 /* fast path */
7089 if (!new && !cur)
7090 return 1;
7091
7092 tmp = SD_ATTR_INIT;
7093 return !memcmp(cur ? (cur + idx_cur) : &tmp,
7094 new ? (new + idx_new) : &tmp,
7095 sizeof(struct sched_domain_attr));
7096}
7097
Paul Jackson029190c2007-10-18 23:40:20 -07007098/*
7099 * Partition sched domains as specified by the 'ndoms_new'
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01007100 * cpumasks in the array doms_new[] of cpumasks. This compares
Paul Jackson029190c2007-10-18 23:40:20 -07007101 * doms_new[] to the current sched domain partitioning, doms_cur[].
7102 * It destroys each deleted domain and builds each new domain.
7103 *
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307104 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01007105 * The masks don't intersect (don't overlap.) We should setup one
7106 * sched domain for each mask. CPUs not in any of the cpumasks will
7107 * not be load balanced. If the same cpumask appears both in the
Paul Jackson029190c2007-10-18 23:40:20 -07007108 * current 'doms_cur' domains and in the new 'doms_new', we can leave
7109 * it as it is.
7110 *
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307111 * The passed in 'doms_new' should be allocated using
7112 * alloc_sched_domains. This routine takes ownership of it and will
7113 * free_sched_domains it when done with it. If the caller failed the
7114 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
7115 * and partition_sched_domains() will fallback to the single partition
7116 * 'fallback_doms', it also forces the domains to be rebuilt.
Paul Jackson029190c2007-10-18 23:40:20 -07007117 *
Rusty Russell96f874e2008-11-25 02:35:14 +10307118 * If doms_new == NULL it will be replaced with cpu_online_mask.
Li Zefan700018e2008-11-18 14:02:03 +08007119 * ndoms_new == 0 is a special case for destroying existing domains,
7120 * and it will not create the default domain.
Max Krasnyanskydfb512e2008-08-29 13:11:41 -07007121 *
Paul Jackson029190c2007-10-18 23:40:20 -07007122 * Call with hotplug lock held
7123 */
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307124void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007125 struct sched_domain_attr *dattr_new)
Paul Jackson029190c2007-10-18 23:40:20 -07007126{
Max Krasnyanskydfb512e2008-08-29 13:11:41 -07007127 int i, j, n;
Heiko Carstensd65bd5e2008-12-09 18:49:51 +01007128 int new_topology;
Paul Jackson029190c2007-10-18 23:40:20 -07007129
Heiko Carstens712555e2008-04-28 11:33:07 +02007130 mutex_lock(&sched_domains_mutex);
Srivatsa Vaddagiria1835612008-01-25 21:08:00 +01007131
Milton Miller73785472007-10-24 18:23:48 +02007132 /* always unregister in case we don't destroy any domains */
7133 unregister_sched_domain_sysctl();
7134
Heiko Carstensd65bd5e2008-12-09 18:49:51 +01007135 /* Let architecture update cpu core mappings. */
7136 new_topology = arch_update_cpu_topology();
7137
Max Krasnyanskydfb512e2008-08-29 13:11:41 -07007138 n = doms_new ? ndoms_new : 0;
Paul Jackson029190c2007-10-18 23:40:20 -07007139
7140 /* Destroy deleted domains */
7141 for (i = 0; i < ndoms_cur; i++) {
Heiko Carstensd65bd5e2008-12-09 18:49:51 +01007142 for (j = 0; j < n && !new_topology; j++) {
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307143 if (cpumask_equal(doms_cur[i], doms_new[j])
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007144 && dattrs_equal(dattr_cur, i, dattr_new, j))
Paul Jackson029190c2007-10-18 23:40:20 -07007145 goto match1;
7146 }
7147 /* no match - a current sched domain not in new doms_new[] */
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307148 detach_destroy_domains(doms_cur[i]);
Paul Jackson029190c2007-10-18 23:40:20 -07007149match1:
7150 ;
7151 }
7152
Max Krasnyanskye761b772008-07-15 04:43:49 -07007153 if (doms_new == NULL) {
7154 ndoms_cur = 0;
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307155 doms_new = &fallback_doms;
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01007156 cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
Li Zefanfaa2f982008-11-04 16:20:23 +08007157 WARN_ON_ONCE(dattr_new);
Max Krasnyanskye761b772008-07-15 04:43:49 -07007158 }
7159
Paul Jackson029190c2007-10-18 23:40:20 -07007160 /* Build new domains */
7161 for (i = 0; i < ndoms_new; i++) {
Heiko Carstensd65bd5e2008-12-09 18:49:51 +01007162 for (j = 0; j < ndoms_cur && !new_topology; j++) {
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307163 if (cpumask_equal(doms_new[i], doms_cur[j])
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007164 && dattrs_equal(dattr_new, i, dattr_cur, j))
Paul Jackson029190c2007-10-18 23:40:20 -07007165 goto match2;
7166 }
7167 /* no match - add a new doms_new */
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307168 __build_sched_domains(doms_new[i],
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007169 dattr_new ? dattr_new + i : NULL);
Paul Jackson029190c2007-10-18 23:40:20 -07007170match2:
7171 ;
7172 }
7173
7174 /* Remember the new sched domains */
Rusty Russellacc3f5d2009-11-03 14:53:40 +10307175 if (doms_cur != &fallback_doms)
7176 free_sched_domains(doms_cur, ndoms_cur);
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007177 kfree(dattr_cur); /* kfree(NULL) is safe */
Paul Jackson029190c2007-10-18 23:40:20 -07007178 doms_cur = doms_new;
Hidetoshi Seto1d3504f2008-04-15 14:04:23 +09007179 dattr_cur = dattr_new;
Paul Jackson029190c2007-10-18 23:40:20 -07007180 ndoms_cur = ndoms_new;
Milton Miller73785472007-10-24 18:23:48 +02007181
7182 register_sched_domain_sysctl();
Srivatsa Vaddagiria1835612008-01-25 21:08:00 +01007183
Heiko Carstens712555e2008-04-28 11:33:07 +02007184 mutex_unlock(&sched_domains_mutex);
Paul Jackson029190c2007-10-18 23:40:20 -07007185}
7186
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007187#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
Li Zefanc70f22d2009-01-05 19:07:50 +08007188static void arch_reinit_sched_domains(void)
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007189{
Gautham R Shenoy95402b32008-01-25 21:08:02 +01007190 get_online_cpus();
Max Krasnyanskydfb512e2008-08-29 13:11:41 -07007191
7192 /* Destroy domains first to force the rebuild */
7193 partition_sched_domains(0, NULL, NULL);
7194
Max Krasnyanskye761b772008-07-15 04:43:49 -07007195 rebuild_sched_domains();
Gautham R Shenoy95402b32008-01-25 21:08:02 +01007196 put_online_cpus();
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007197}
7198
7199static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7200{
Gautham R Shenoyafb8a9b2008-12-18 23:26:09 +05307201 unsigned int level = 0;
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007202
Gautham R Shenoyafb8a9b2008-12-18 23:26:09 +05307203 if (sscanf(buf, "%u", &level) != 1)
7204 return -EINVAL;
7205
7206 /*
7207 * level is always be positive so don't check for
7208 * level < POWERSAVINGS_BALANCE_NONE which is 0
7209 * What happens on 0 or 1 byte write,
7210 * need to check for count as well?
7211 */
7212
7213 if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007214 return -EINVAL;
7215
7216 if (smt)
Gautham R Shenoyafb8a9b2008-12-18 23:26:09 +05307217 sched_smt_power_savings = level;
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007218 else
Gautham R Shenoyafb8a9b2008-12-18 23:26:09 +05307219 sched_mc_power_savings = level;
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007220
Li Zefanc70f22d2009-01-05 19:07:50 +08007221 arch_reinit_sched_domains();
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007222
Li Zefanc70f22d2009-01-05 19:07:50 +08007223 return count;
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007224}
7225
Adrian Bunk6707de002007-08-12 18:08:19 +02007226#ifdef CONFIG_SCHED_MC
Andi Kleenf718cd42008-07-29 22:33:52 -07007227static ssize_t sched_mc_power_savings_show(struct sysdev_class *class,
Andi Kleenc9be0a32010-01-05 12:47:58 +01007228 struct sysdev_class_attribute *attr,
Andi Kleenf718cd42008-07-29 22:33:52 -07007229 char *page)
Adrian Bunk6707de002007-08-12 18:08:19 +02007230{
7231 return sprintf(page, "%u\n", sched_mc_power_savings);
7232}
Andi Kleenf718cd42008-07-29 22:33:52 -07007233static ssize_t sched_mc_power_savings_store(struct sysdev_class *class,
Andi Kleenc9be0a32010-01-05 12:47:58 +01007234 struct sysdev_class_attribute *attr,
Adrian Bunk6707de002007-08-12 18:08:19 +02007235 const char *buf, size_t count)
7236{
7237 return sched_power_savings_store(buf, count, 0);
7238}
Andi Kleenf718cd42008-07-29 22:33:52 -07007239static SYSDEV_CLASS_ATTR(sched_mc_power_savings, 0644,
7240 sched_mc_power_savings_show,
7241 sched_mc_power_savings_store);
Adrian Bunk6707de002007-08-12 18:08:19 +02007242#endif
7243
7244#ifdef CONFIG_SCHED_SMT
Andi Kleenf718cd42008-07-29 22:33:52 -07007245static ssize_t sched_smt_power_savings_show(struct sysdev_class *dev,
Andi Kleenc9be0a32010-01-05 12:47:58 +01007246 struct sysdev_class_attribute *attr,
Andi Kleenf718cd42008-07-29 22:33:52 -07007247 char *page)
Adrian Bunk6707de002007-08-12 18:08:19 +02007248{
7249 return sprintf(page, "%u\n", sched_smt_power_savings);
7250}
Andi Kleenf718cd42008-07-29 22:33:52 -07007251static ssize_t sched_smt_power_savings_store(struct sysdev_class *dev,
Andi Kleenc9be0a32010-01-05 12:47:58 +01007252 struct sysdev_class_attribute *attr,
Adrian Bunk6707de002007-08-12 18:08:19 +02007253 const char *buf, size_t count)
7254{
7255 return sched_power_savings_store(buf, count, 1);
7256}
Andi Kleenf718cd42008-07-29 22:33:52 -07007257static SYSDEV_CLASS_ATTR(sched_smt_power_savings, 0644,
7258 sched_smt_power_savings_show,
Adrian Bunk6707de002007-08-12 18:08:19 +02007259 sched_smt_power_savings_store);
7260#endif
7261
Li Zefan39aac642009-01-05 19:18:02 +08007262int __init sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007263{
7264 int err = 0;
Ingo Molnar48f24c42006-07-03 00:25:40 -07007265
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007266#ifdef CONFIG_SCHED_SMT
7267 if (smt_capable())
7268 err = sysfs_create_file(&cls->kset.kobj,
7269 &attr_sched_smt_power_savings.attr);
7270#endif
7271#ifdef CONFIG_SCHED_MC
7272 if (!err && mc_capable())
7273 err = sysfs_create_file(&cls->kset.kobj,
7274 &attr_sched_mc_power_savings.attr);
7275#endif
7276 return err;
7277}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007278#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
Siddha, Suresh B5c45bf22006-06-27 02:54:42 -07007279
Max Krasnyanskye761b772008-07-15 04:43:49 -07007280#ifndef CONFIG_CPUSETS
Linus Torvalds1da177e2005-04-16 15:20:36 -07007281/*
Max Krasnyanskye761b772008-07-15 04:43:49 -07007282 * Add online and remove offline CPUs from the scheduler domains.
7283 * When cpusets are enabled they take over this function.
Linus Torvalds1da177e2005-04-16 15:20:36 -07007284 */
7285static int update_sched_domains(struct notifier_block *nfb,
7286 unsigned long action, void *hcpu)
7287{
Max Krasnyanskye761b772008-07-15 04:43:49 -07007288 switch (action) {
7289 case CPU_ONLINE:
7290 case CPU_ONLINE_FROZEN:
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01007291 case CPU_DOWN_PREPARE:
7292 case CPU_DOWN_PREPARE_FROZEN:
7293 case CPU_DOWN_FAILED:
7294 case CPU_DOWN_FAILED_FROZEN:
Max Krasnyanskydfb512e2008-08-29 13:11:41 -07007295 partition_sched_domains(1, NULL, NULL);
Max Krasnyanskye761b772008-07-15 04:43:49 -07007296 return NOTIFY_OK;
7297
7298 default:
7299 return NOTIFY_DONE;
7300 }
7301}
7302#endif
7303
7304static int update_runtime(struct notifier_block *nfb,
7305 unsigned long action, void *hcpu)
7306{
Peter Zijlstra7def2be2008-06-05 14:49:58 +02007307 int cpu = (int)(long)hcpu;
7308
Linus Torvalds1da177e2005-04-16 15:20:36 -07007309 switch (action) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07007310 case CPU_DOWN_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07007311 case CPU_DOWN_PREPARE_FROZEN:
Peter Zijlstra7def2be2008-06-05 14:49:58 +02007312 disable_runtime(cpu_rq(cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -07007313 return NOTIFY_OK;
7314
Linus Torvalds1da177e2005-04-16 15:20:36 -07007315 case CPU_DOWN_FAILED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07007316 case CPU_DOWN_FAILED_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -07007317 case CPU_ONLINE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07007318 case CPU_ONLINE_FROZEN:
Peter Zijlstra7def2be2008-06-05 14:49:58 +02007319 enable_runtime(cpu_rq(cpu));
Max Krasnyanskye761b772008-07-15 04:43:49 -07007320 return NOTIFY_OK;
7321
Linus Torvalds1da177e2005-04-16 15:20:36 -07007322 default:
7323 return NOTIFY_DONE;
7324 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007325}
Linus Torvalds1da177e2005-04-16 15:20:36 -07007326
7327void __init sched_init_smp(void)
7328{
Rusty Russelldcc30a32008-11-25 02:35:12 +10307329 cpumask_var_t non_isolated_cpus;
7330
7331 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
Yong Zhangcb5fd132009-09-14 20:20:16 +08007332 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
Nick Piggin5c1e1762006-10-03 01:14:04 -07007333
Mike Travis434d53b2008-04-04 18:11:04 -07007334#if defined(CONFIG_NUMA)
7335 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
7336 GFP_KERNEL);
7337 BUG_ON(sched_group_nodes_bycpu == NULL);
7338#endif
Gautham R Shenoy95402b32008-01-25 21:08:02 +01007339 get_online_cpus();
Heiko Carstens712555e2008-04-28 11:33:07 +02007340 mutex_lock(&sched_domains_mutex);
Peter Zijlstra6ad4c182009-11-25 13:31:39 +01007341 arch_init_sched_domains(cpu_active_mask);
Rusty Russelldcc30a32008-11-25 02:35:12 +10307342 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
7343 if (cpumask_empty(non_isolated_cpus))
7344 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
Heiko Carstens712555e2008-04-28 11:33:07 +02007345 mutex_unlock(&sched_domains_mutex);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01007346 put_online_cpus();
Max Krasnyanskye761b772008-07-15 04:43:49 -07007347
7348#ifndef CONFIG_CPUSETS
Linus Torvalds1da177e2005-04-16 15:20:36 -07007349 /* XXX: Theoretical race here - CPU may be hotplugged now */
7350 hotcpu_notifier(update_sched_domains, 0);
Max Krasnyanskye761b772008-07-15 04:43:49 -07007351#endif
7352
7353 /* RT runtime code needs to handle some hotplug events */
7354 hotcpu_notifier(update_runtime, 0);
7355
Peter Zijlstrab328ca12008-04-29 10:02:46 +02007356 init_hrtick();
Nick Piggin5c1e1762006-10-03 01:14:04 -07007357
7358 /* Move init over to a non-isolated CPU */
Rusty Russelldcc30a32008-11-25 02:35:12 +10307359 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
Nick Piggin5c1e1762006-10-03 01:14:04 -07007360 BUG();
Ingo Molnar19978ca2007-11-09 22:39:38 +01007361 sched_init_granularity();
Rusty Russelldcc30a32008-11-25 02:35:12 +10307362 free_cpumask_var(non_isolated_cpus);
Rusty Russell42128232008-11-25 02:35:12 +10307363
Rusty Russell0e3900e2008-11-25 02:35:13 +10307364 init_sched_rt_class();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007365}
7366#else
7367void __init sched_init_smp(void)
7368{
Ingo Molnar19978ca2007-11-09 22:39:38 +01007369 sched_init_granularity();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007370}
7371#endif /* CONFIG_SMP */
7372
Arun R Bharadwajcd1bb942009-04-16 12:15:34 +05307373const_debug unsigned int sysctl_timer_migration = 1;
7374
Linus Torvalds1da177e2005-04-16 15:20:36 -07007375int in_sched_functions(unsigned long addr)
7376{
Linus Torvalds1da177e2005-04-16 15:20:36 -07007377 return in_lock_functions(addr) ||
7378 (addr >= (unsigned long)__sched_text_start
7379 && addr < (unsigned long)__sched_text_end);
7380}
7381
Alexey Dobriyana9957442007-10-15 17:00:13 +02007382static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq)
Ingo Molnardd41f592007-07-09 18:51:59 +02007383{
7384 cfs_rq->tasks_timeline = RB_ROOT;
Peter Zijlstra4a55bd52008-04-19 19:45:00 +02007385 INIT_LIST_HEAD(&cfs_rq->tasks);
Ingo Molnardd41f592007-07-09 18:51:59 +02007386#ifdef CONFIG_FAIR_GROUP_SCHED
7387 cfs_rq->rq = rq;
7388#endif
Peter Zijlstra67e9fb22007-10-15 17:00:10 +02007389 cfs_rq->min_vruntime = (u64)(-(1LL << 20));
Ingo Molnardd41f592007-07-09 18:51:59 +02007390}
7391
Peter Zijlstrafa85ae22008-01-25 21:08:29 +01007392static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
7393{
7394 struct rt_prio_array *array;
7395 int i;
7396
7397 array = &rt_rq->active;
7398 for (i = 0; i < MAX_RT_PRIO; i++) {
7399 INIT_LIST_HEAD(array->queue + i);
7400 __clear_bit(i, array->bitmap);
7401 }
7402 /* delimiter for bitsearch: */
7403 __set_bit(MAX_RT_PRIO, array->bitmap);
7404
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007405#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
Gregory Haskinse864c492008-12-29 09:39:49 -05007406 rt_rq->highest_prio.curr = MAX_RT_PRIO;
Gregory Haskins398a1532009-01-14 09:10:04 -05007407#ifdef CONFIG_SMP
Gregory Haskinse864c492008-12-29 09:39:49 -05007408 rt_rq->highest_prio.next = MAX_RT_PRIO;
Peter Zijlstra48d5e252008-01-25 21:08:31 +01007409#endif
Peter Zijlstrafa85ae22008-01-25 21:08:29 +01007410#endif
7411#ifdef CONFIG_SMP
7412 rt_rq->rt_nr_migratory = 0;
Peter Zijlstrafa85ae22008-01-25 21:08:29 +01007413 rt_rq->overloaded = 0;
Thomas Gleixner05fa7852009-11-17 14:28:38 +01007414 plist_head_init_raw(&rt_rq->pushable_tasks, &rq->lock);
Peter Zijlstrafa85ae22008-01-25 21:08:29 +01007415#endif
7416
7417 rt_rq->rt_time = 0;
7418 rt_rq->rt_throttled = 0;
Peter Zijlstraac086bc2008-04-19 19:44:58 +02007419 rt_rq->rt_runtime = 0;
Thomas Gleixner0986b112009-11-17 15:32:06 +01007420 raw_spin_lock_init(&rt_rq->rt_runtime_lock);
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007421
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007422#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra23b0fdf2008-02-13 15:45:39 +01007423 rt_rq->rt_nr_boosted = 0;
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007424 rt_rq->rq = rq;
7425#endif
Peter Zijlstrafa85ae22008-01-25 21:08:29 +01007426}
7427
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007428#ifdef CONFIG_FAIR_GROUP_SCHED
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007429static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
7430 struct sched_entity *se, int cpu, int add,
7431 struct sched_entity *parent)
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007432{
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007433 struct rq *rq = cpu_rq(cpu);
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007434 tg->cfs_rq[cpu] = cfs_rq;
7435 init_cfs_rq(cfs_rq, rq);
7436 cfs_rq->tg = tg;
7437 if (add)
7438 list_add(&cfs_rq->leaf_cfs_rq_list, &rq->leaf_cfs_rq_list);
7439
7440 tg->se[cpu] = se;
Dhaval Giani354d60c2008-04-19 19:44:59 +02007441 /* se could be NULL for init_task_group */
7442 if (!se)
7443 return;
7444
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007445 if (!parent)
7446 se->cfs_rq = &rq->cfs;
7447 else
7448 se->cfs_rq = parent->my_q;
7449
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007450 se->my_q = cfs_rq;
7451 se->load.weight = tg->shares;
Peter Zijlstrae05510d2008-05-05 23:56:17 +02007452 se->load.inv_weight = 0;
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007453 se->parent = parent;
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007454}
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007455#endif
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007456
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007457#ifdef CONFIG_RT_GROUP_SCHED
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007458static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
7459 struct sched_rt_entity *rt_se, int cpu, int add,
7460 struct sched_rt_entity *parent)
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007461{
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007462 struct rq *rq = cpu_rq(cpu);
7463
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007464 tg->rt_rq[cpu] = rt_rq;
7465 init_rt_rq(rt_rq, rq);
7466 rt_rq->tg = tg;
Peter Zijlstraac086bc2008-04-19 19:44:58 +02007467 rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007468 if (add)
7469 list_add(&rt_rq->leaf_rt_rq_list, &rq->leaf_rt_rq_list);
7470
7471 tg->rt_se[cpu] = rt_se;
Dhaval Giani354d60c2008-04-19 19:44:59 +02007472 if (!rt_se)
7473 return;
7474
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007475 if (!parent)
7476 rt_se->rt_rq = &rq->rt;
7477 else
7478 rt_se->rt_rq = parent->my_q;
7479
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007480 rt_se->my_q = rt_rq;
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007481 rt_se->parent = parent;
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007482 INIT_LIST_HEAD(&rt_se->run_list);
7483}
7484#endif
7485
Linus Torvalds1da177e2005-04-16 15:20:36 -07007486void __init sched_init(void)
7487{
Ingo Molnardd41f592007-07-09 18:51:59 +02007488 int i, j;
Mike Travis434d53b2008-04-04 18:11:04 -07007489 unsigned long alloc_size = 0, ptr;
7490
7491#ifdef CONFIG_FAIR_GROUP_SCHED
7492 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7493#endif
7494#ifdef CONFIG_RT_GROUP_SCHED
7495 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7496#endif
Rusty Russelldf7c8e82009-03-19 15:22:20 +10307497#ifdef CONFIG_CPUMASK_OFFSTACK
Rusty Russell8c083f02009-03-19 15:22:20 +10307498 alloc_size += num_possible_cpus() * cpumask_size();
Rusty Russelldf7c8e82009-03-19 15:22:20 +10307499#endif
Mike Travis434d53b2008-04-04 18:11:04 -07007500 if (alloc_size) {
Pekka Enberg36b7b6d2009-06-10 23:42:36 +03007501 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
Mike Travis434d53b2008-04-04 18:11:04 -07007502
7503#ifdef CONFIG_FAIR_GROUP_SCHED
7504 init_task_group.se = (struct sched_entity **)ptr;
7505 ptr += nr_cpu_ids * sizeof(void **);
7506
7507 init_task_group.cfs_rq = (struct cfs_rq **)ptr;
7508 ptr += nr_cpu_ids * sizeof(void **);
Peter Zijlstraeff766a2008-04-19 19:45:00 +02007509
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007510#endif /* CONFIG_FAIR_GROUP_SCHED */
Mike Travis434d53b2008-04-04 18:11:04 -07007511#ifdef CONFIG_RT_GROUP_SCHED
7512 init_task_group.rt_se = (struct sched_rt_entity **)ptr;
7513 ptr += nr_cpu_ids * sizeof(void **);
7514
7515 init_task_group.rt_rq = (struct rt_rq **)ptr;
Peter Zijlstraeff766a2008-04-19 19:45:00 +02007516 ptr += nr_cpu_ids * sizeof(void **);
7517
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007518#endif /* CONFIG_RT_GROUP_SCHED */
Rusty Russelldf7c8e82009-03-19 15:22:20 +10307519#ifdef CONFIG_CPUMASK_OFFSTACK
7520 for_each_possible_cpu(i) {
7521 per_cpu(load_balance_tmpmask, i) = (void *)ptr;
7522 ptr += cpumask_size();
7523 }
7524#endif /* CONFIG_CPUMASK_OFFSTACK */
Mike Travis434d53b2008-04-04 18:11:04 -07007525 }
Ingo Molnardd41f592007-07-09 18:51:59 +02007526
Gregory Haskins57d885f2008-01-25 21:08:18 +01007527#ifdef CONFIG_SMP
7528 init_defrootdomain();
7529#endif
7530
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02007531 init_rt_bandwidth(&def_rt_bandwidth,
7532 global_rt_period(), global_rt_runtime());
7533
7534#ifdef CONFIG_RT_GROUP_SCHED
7535 init_rt_bandwidth(&init_task_group.rt_bandwidth,
7536 global_rt_period(), global_rt_runtime());
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007537#endif /* CONFIG_RT_GROUP_SCHED */
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02007538
Dhaval Giani7c941432010-01-20 13:26:18 +01007539#ifdef CONFIG_CGROUP_SCHED
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007540 list_add(&init_task_group.list, &task_groups);
Peter Zijlstraf473aa52008-04-19 19:45:00 +02007541 INIT_LIST_HEAD(&init_task_group.children);
7542
Dhaval Giani7c941432010-01-20 13:26:18 +01007543#endif /* CONFIG_CGROUP_SCHED */
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007544
Jiri Kosina4a6cc4b2009-10-29 00:26:00 +09007545#if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP
7546 update_shares_data = __alloc_percpu(nr_cpu_ids * sizeof(unsigned long),
7547 __alignof__(unsigned long));
7548#endif
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08007549 for_each_possible_cpu(i) {
Ingo Molnar70b97a72006-07-03 00:25:42 -07007550 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007551
7552 rq = cpu_rq(i);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01007553 raw_spin_lock_init(&rq->lock);
Nick Piggin78979862005-06-25 14:57:13 -07007554 rq->nr_running = 0;
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02007555 rq->calc_load_active = 0;
7556 rq->calc_load_update = jiffies + LOAD_FREQ;
Ingo Molnardd41f592007-07-09 18:51:59 +02007557 init_cfs_rq(&rq->cfs, rq);
Peter Zijlstrafa85ae22008-01-25 21:08:29 +01007558 init_rt_rq(&rq->rt, rq);
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007559#ifdef CONFIG_FAIR_GROUP_SCHED
7560 init_task_group.shares = init_task_group_load;
7561 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
Dhaval Giani354d60c2008-04-19 19:44:59 +02007562#ifdef CONFIG_CGROUP_SCHED
7563 /*
7564 * How much cpu bandwidth does init_task_group get?
7565 *
7566 * In case of task-groups formed thr' the cgroup filesystem, it
7567 * gets 100% of the cpu resources in the system. This overall
7568 * system cpu resource is divided among the tasks of
7569 * init_task_group and its child task-groups in a fair manner,
7570 * based on each entity's (task or task-group's) weight
7571 * (se->load.weight).
7572 *
7573 * In other words, if init_task_group has 10 tasks of weight
7574 * 1024) and two child groups A0 and A1 (of weight 1024 each),
7575 * then A0's share of the cpu resource is:
7576 *
Ingo Molnar0d905bc2009-05-04 19:13:30 +02007577 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
Dhaval Giani354d60c2008-04-19 19:44:59 +02007578 *
7579 * We achieve this by letting init_task_group's tasks sit
7580 * directly in rq->cfs (i.e init_task_group->se[] = NULL).
7581 */
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007582 init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007583#endif
Dhaval Giani354d60c2008-04-19 19:44:59 +02007584#endif /* CONFIG_FAIR_GROUP_SCHED */
7585
7586 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007587#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007588 INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
Dhaval Giani354d60c2008-04-19 19:44:59 +02007589#ifdef CONFIG_CGROUP_SCHED
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007590 init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
Dhaval Giani354d60c2008-04-19 19:44:59 +02007591#endif
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007592#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07007593
Ingo Molnardd41f592007-07-09 18:51:59 +02007594 for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
7595 rq->cpu_load[j] = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007596#ifdef CONFIG_SMP
Nick Piggin41c7ce92005-06-25 14:57:24 -07007597 rq->sd = NULL;
Gregory Haskins57d885f2008-01-25 21:08:18 +01007598 rq->rd = NULL;
Gregory Haskins3f029d32009-07-29 11:08:47 -04007599 rq->post_schedule = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007600 rq->active_balance = 0;
Ingo Molnardd41f592007-07-09 18:51:59 +02007601 rq->next_balance = jiffies;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007602 rq->push_cpu = 0;
Christoph Lameter0a2966b2006-09-25 23:30:51 -07007603 rq->cpu = i;
Gregory Haskins1f11eb62008-06-04 15:04:05 -04007604 rq->online = 0;
Mike Galbraitheae0c9d2009-11-10 03:50:02 +01007605 rq->idle_stamp = 0;
7606 rq->avg_idle = 2*sysctl_sched_migration_cost;
Gregory Haskinsdc938522008-01-25 21:08:26 +01007607 rq_attach_root(rq, &def_root_domain);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007608#endif
Peter Zijlstra8f4d37e2008-01-25 21:08:29 +01007609 init_rq_hrtick(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007610 atomic_set(&rq->nr_iowait, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007611 }
7612
Peter Williams2dd73a42006-06-27 02:54:34 -07007613 set_load_weight(&init_task);
Heiko Carstensb50f60c2006-07-30 03:03:52 -07007614
Avi Kivitye107be32007-07-26 13:40:43 +02007615#ifdef CONFIG_PREEMPT_NOTIFIERS
7616 INIT_HLIST_HEAD(&init_task.preempt_notifiers);
7617#endif
7618
Christoph Lameterc9819f42006-12-10 02:20:25 -08007619#ifdef CONFIG_SMP
Carlos R. Mafra962cf362008-05-15 11:15:37 -03007620 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
Christoph Lameterc9819f42006-12-10 02:20:25 -08007621#endif
7622
Heiko Carstensb50f60c2006-07-30 03:03:52 -07007623#ifdef CONFIG_RT_MUTEXES
Thomas Gleixner1d615482009-11-17 14:54:03 +01007624 plist_head_init_raw(&init_task.pi_waiters, &init_task.pi_lock);
Heiko Carstensb50f60c2006-07-30 03:03:52 -07007625#endif
7626
Linus Torvalds1da177e2005-04-16 15:20:36 -07007627 /*
7628 * The boot idle thread does lazy MMU switching as well:
7629 */
7630 atomic_inc(&init_mm.mm_count);
7631 enter_lazy_tlb(&init_mm, current);
7632
7633 /*
7634 * Make us the idle thread. Technically, schedule() should not be
7635 * called from this thread, however somewhere below it might be,
7636 * but because we are the idle thread, we just pick up running again
7637 * when this runqueue becomes "idle".
7638 */
7639 init_idle(current, smp_processor_id());
Thomas Gleixnerdce48a82009-04-11 10:43:41 +02007640
7641 calc_load_update = jiffies + LOAD_FREQ;
7642
Ingo Molnardd41f592007-07-09 18:51:59 +02007643 /*
7644 * During early bootup we pretend to be a normal task:
7645 */
7646 current->sched_class = &fair_sched_class;
Ingo Molnar6892b752008-02-13 14:02:36 +01007647
Rusty Russell6a7b3dc2008-11-25 02:35:04 +10307648 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
Rusty Russell49557e62009-11-02 20:37:20 +10307649 zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT);
Rusty Russellbf4d83f2008-11-25 09:57:51 +10307650#ifdef CONFIG_SMP
Rusty Russell7d1e6a92008-11-25 02:35:09 +10307651#ifdef CONFIG_NO_HZ
Rusty Russell49557e62009-11-02 20:37:20 +10307652 zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT);
Pekka Enberg4bdddf82009-06-11 08:35:27 +03007653 alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT);
Rusty Russell7d1e6a92008-11-25 02:35:09 +10307654#endif
Rusty Russellbdddd292009-12-02 14:09:16 +10307655 /* May be allocated at isolcpus cmdline parse time */
7656 if (cpu_isolated_map == NULL)
7657 zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
Rusty Russellbf4d83f2008-11-25 09:57:51 +10307658#endif /* SMP */
Rusty Russell6a7b3dc2008-11-25 02:35:04 +10307659
Ingo Molnarcdd6c482009-09-21 12:02:48 +02007660 perf_event_init();
Ingo Molnar0d905bc2009-05-04 19:13:30 +02007661
Ingo Molnar6892b752008-02-13 14:02:36 +01007662 scheduler_running = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007663}
7664
7665#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
Frederic Weisbeckere4aafea2009-07-16 15:44:29 +02007666static inline int preempt_count_equals(int preempt_offset)
7667{
Frederic Weisbecker234da7b2009-12-16 20:21:05 +01007668 int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
Frederic Weisbeckere4aafea2009-07-16 15:44:29 +02007669
7670 return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
7671}
7672
Simon Kagstromd8948372009-12-23 11:08:18 +01007673void __might_sleep(const char *file, int line, int preempt_offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007674{
Ingo Molnar48f24c42006-07-03 00:25:40 -07007675#ifdef in_atomic
Linus Torvalds1da177e2005-04-16 15:20:36 -07007676 static unsigned long prev_jiffy; /* ratelimiting */
7677
Frederic Weisbeckere4aafea2009-07-16 15:44:29 +02007678 if ((preempt_count_equals(preempt_offset) && !irqs_disabled()) ||
7679 system_state != SYSTEM_RUNNING || oops_in_progress)
Ingo Molnaraef745f2008-08-28 11:34:43 +02007680 return;
7681 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
7682 return;
7683 prev_jiffy = jiffies;
7684
Peter Zijlstra3df0fc52009-12-20 14:23:57 +01007685 printk(KERN_ERR
7686 "BUG: sleeping function called from invalid context at %s:%d\n",
7687 file, line);
7688 printk(KERN_ERR
7689 "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
7690 in_atomic(), irqs_disabled(),
7691 current->pid, current->comm);
Ingo Molnaraef745f2008-08-28 11:34:43 +02007692
7693 debug_show_held_locks(current);
7694 if (irqs_disabled())
7695 print_irqtrace_events(current);
7696 dump_stack();
Linus Torvalds1da177e2005-04-16 15:20:36 -07007697#endif
7698}
7699EXPORT_SYMBOL(__might_sleep);
7700#endif
7701
7702#ifdef CONFIG_MAGIC_SYSRQ
Andi Kleen3a5e4dc2007-10-15 17:00:15 +02007703static void normalize_task(struct rq *rq, struct task_struct *p)
7704{
7705 int on_rq;
Peter Zijlstra3e51f332008-05-03 18:29:28 +02007706
Andi Kleen3a5e4dc2007-10-15 17:00:15 +02007707 on_rq = p->se.on_rq;
7708 if (on_rq)
7709 deactivate_task(rq, p, 0);
7710 __setscheduler(rq, p, SCHED_NORMAL, 0);
7711 if (on_rq) {
7712 activate_task(rq, p, 0);
7713 resched_task(rq->curr);
7714 }
7715}
7716
Linus Torvalds1da177e2005-04-16 15:20:36 -07007717void normalize_rt_tasks(void)
7718{
Ingo Molnara0f98a12007-06-17 18:37:45 +02007719 struct task_struct *g, *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007720 unsigned long flags;
Ingo Molnar70b97a72006-07-03 00:25:42 -07007721 struct rq *rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07007722
Peter Zijlstra4cf5d772008-02-13 15:45:39 +01007723 read_lock_irqsave(&tasklist_lock, flags);
Ingo Molnara0f98a12007-06-17 18:37:45 +02007724 do_each_thread(g, p) {
Ingo Molnar178be792007-10-15 17:00:18 +02007725 /*
7726 * Only normalize user tasks:
7727 */
7728 if (!p->mm)
7729 continue;
7730
Ingo Molnardd41f592007-07-09 18:51:59 +02007731 p->se.exec_start = 0;
Ingo Molnar6cfb0d52007-08-02 17:41:40 +02007732#ifdef CONFIG_SCHEDSTATS
Lucas De Marchi41acab82010-03-10 23:37:45 -03007733 p->se.statistics.wait_start = 0;
7734 p->se.statistics.sleep_start = 0;
7735 p->se.statistics.block_start = 0;
Ingo Molnar6cfb0d52007-08-02 17:41:40 +02007736#endif
Ingo Molnardd41f592007-07-09 18:51:59 +02007737
7738 if (!rt_task(p)) {
7739 /*
7740 * Renice negative nice level userspace
7741 * tasks back to 0:
7742 */
7743 if (TASK_NICE(p) < 0 && p->mm)
7744 set_user_nice(p, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007745 continue;
Ingo Molnardd41f592007-07-09 18:51:59 +02007746 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07007747
Thomas Gleixner1d615482009-11-17 14:54:03 +01007748 raw_spin_lock(&p->pi_lock);
Ingo Molnarb29739f2006-06-27 02:54:51 -07007749 rq = __task_rq_lock(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007750
Ingo Molnar178be792007-10-15 17:00:18 +02007751 normalize_task(rq, p);
Andi Kleen3a5e4dc2007-10-15 17:00:15 +02007752
Ingo Molnarb29739f2006-06-27 02:54:51 -07007753 __task_rq_unlock(rq);
Thomas Gleixner1d615482009-11-17 14:54:03 +01007754 raw_spin_unlock(&p->pi_lock);
Ingo Molnara0f98a12007-06-17 18:37:45 +02007755 } while_each_thread(g, p);
7756
Peter Zijlstra4cf5d772008-02-13 15:45:39 +01007757 read_unlock_irqrestore(&tasklist_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07007758}
7759
7760#endif /* CONFIG_MAGIC_SYSRQ */
Linus Torvalds1df5c102005-09-12 07:59:21 -07007761
7762#ifdef CONFIG_IA64
7763/*
7764 * These functions are only useful for the IA64 MCA handling.
7765 *
7766 * They can only be called when the whole system has been
7767 * stopped - every CPU needs to be quiescent, and no scheduling
7768 * activity can take place. Using them for anything else would
7769 * be a serious bug, and as a result, they aren't even visible
7770 * under any other configuration.
7771 */
7772
7773/**
7774 * curr_task - return the current task for a given cpu.
7775 * @cpu: the processor in question.
7776 *
7777 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7778 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07007779struct task_struct *curr_task(int cpu)
Linus Torvalds1df5c102005-09-12 07:59:21 -07007780{
7781 return cpu_curr(cpu);
7782}
7783
7784/**
7785 * set_curr_task - set the current task for a given cpu.
7786 * @cpu: the processor in question.
7787 * @p: the task pointer to set.
7788 *
7789 * Description: This function must only be used when non-maskable interrupts
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01007790 * are serviced on a separate stack. It allows the architecture to switch the
7791 * notion of the current task on a cpu in a non-blocking manner. This function
Linus Torvalds1df5c102005-09-12 07:59:21 -07007792 * must be called with all CPU's synchronized, and interrupts disabled, the
7793 * and caller must save the original value of the current task (see
7794 * curr_task() above) and restore that value before reenabling interrupts and
7795 * re-starting the system.
7796 *
7797 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7798 */
Ingo Molnar36c8b582006-07-03 00:25:41 -07007799void set_curr_task(int cpu, struct task_struct *p)
Linus Torvalds1df5c102005-09-12 07:59:21 -07007800{
7801 cpu_curr(cpu) = p;
7802}
7803
7804#endif
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007805
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007806#ifdef CONFIG_FAIR_GROUP_SCHED
7807static void free_fair_sched_group(struct task_group *tg)
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007808{
7809 int i;
7810
7811 for_each_possible_cpu(i) {
7812 if (tg->cfs_rq)
7813 kfree(tg->cfs_rq[i]);
7814 if (tg->se)
7815 kfree(tg->se[i]);
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007816 }
7817
7818 kfree(tg->cfs_rq);
7819 kfree(tg->se);
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007820}
7821
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007822static
7823int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007824{
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007825 struct cfs_rq *cfs_rq;
Li Zefaneab17222008-10-29 17:03:22 +08007826 struct sched_entity *se;
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02007827 struct rq *rq;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007828 int i;
7829
Mike Travis434d53b2008-04-04 18:11:04 -07007830 tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007831 if (!tg->cfs_rq)
7832 goto err;
Mike Travis434d53b2008-04-04 18:11:04 -07007833 tg->se = kzalloc(sizeof(se) * nr_cpu_ids, GFP_KERNEL);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007834 if (!tg->se)
7835 goto err;
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007836
7837 tg->shares = NICE_0_LOAD;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007838
7839 for_each_possible_cpu(i) {
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02007840 rq = cpu_rq(i);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007841
Li Zefaneab17222008-10-29 17:03:22 +08007842 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
7843 GFP_KERNEL, cpu_to_node(i));
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007844 if (!cfs_rq)
7845 goto err;
7846
Li Zefaneab17222008-10-29 17:03:22 +08007847 se = kzalloc_node(sizeof(struct sched_entity),
7848 GFP_KERNEL, cpu_to_node(i));
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007849 if (!se)
Phil Carmodydfc12eb2009-12-10 14:29:37 +02007850 goto err_free_rq;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007851
Li Zefaneab17222008-10-29 17:03:22 +08007852 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007853 }
7854
7855 return 1;
7856
Phil Carmodydfc12eb2009-12-10 14:29:37 +02007857 err_free_rq:
7858 kfree(cfs_rq);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007859 err:
7860 return 0;
7861}
7862
7863static inline void register_fair_sched_group(struct task_group *tg, int cpu)
7864{
7865 list_add_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list,
7866 &cpu_rq(cpu)->leaf_cfs_rq_list);
7867}
7868
7869static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
7870{
7871 list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
7872}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007873#else /* !CONFG_FAIR_GROUP_SCHED */
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007874static inline void free_fair_sched_group(struct task_group *tg)
7875{
7876}
7877
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007878static inline
7879int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007880{
7881 return 1;
7882}
7883
7884static inline void register_fair_sched_group(struct task_group *tg, int cpu)
7885{
7886}
7887
7888static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
7889{
7890}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007891#endif /* CONFIG_FAIR_GROUP_SCHED */
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01007892
7893#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007894static void free_rt_sched_group(struct task_group *tg)
7895{
7896 int i;
7897
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02007898 destroy_rt_bandwidth(&tg->rt_bandwidth);
7899
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007900 for_each_possible_cpu(i) {
7901 if (tg->rt_rq)
7902 kfree(tg->rt_rq[i]);
7903 if (tg->rt_se)
7904 kfree(tg->rt_se[i]);
7905 }
7906
7907 kfree(tg->rt_rq);
7908 kfree(tg->rt_se);
7909}
7910
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007911static
7912int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007913{
7914 struct rt_rq *rt_rq;
Li Zefaneab17222008-10-29 17:03:22 +08007915 struct sched_rt_entity *rt_se;
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007916 struct rq *rq;
7917 int i;
7918
Mike Travis434d53b2008-04-04 18:11:04 -07007919 tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007920 if (!tg->rt_rq)
7921 goto err;
Mike Travis434d53b2008-04-04 18:11:04 -07007922 tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007923 if (!tg->rt_se)
7924 goto err;
7925
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02007926 init_rt_bandwidth(&tg->rt_bandwidth,
7927 ktime_to_ns(def_rt_bandwidth.rt_period), 0);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007928
7929 for_each_possible_cpu(i) {
7930 rq = cpu_rq(i);
7931
Li Zefaneab17222008-10-29 17:03:22 +08007932 rt_rq = kzalloc_node(sizeof(struct rt_rq),
7933 GFP_KERNEL, cpu_to_node(i));
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007934 if (!rt_rq)
7935 goto err;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007936
Li Zefaneab17222008-10-29 17:03:22 +08007937 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
7938 GFP_KERNEL, cpu_to_node(i));
Peter Zijlstra6f505b12008-01-25 21:08:30 +01007939 if (!rt_se)
Phil Carmodydfc12eb2009-12-10 14:29:37 +02007940 goto err_free_rq;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007941
Li Zefaneab17222008-10-29 17:03:22 +08007942 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02007943 }
7944
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007945 return 1;
7946
Phil Carmodydfc12eb2009-12-10 14:29:37 +02007947 err_free_rq:
7948 kfree(rt_rq);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007949 err:
7950 return 0;
7951}
7952
7953static inline void register_rt_sched_group(struct task_group *tg, int cpu)
7954{
7955 list_add_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list,
7956 &cpu_rq(cpu)->leaf_rt_rq_list);
7957}
7958
7959static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
7960{
7961 list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
7962}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007963#else /* !CONFIG_RT_GROUP_SCHED */
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007964static inline void free_rt_sched_group(struct task_group *tg)
7965{
7966}
7967
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007968static inline
7969int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007970{
7971 return 1;
7972}
7973
7974static inline void register_rt_sched_group(struct task_group *tg, int cpu)
7975{
7976}
7977
7978static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
7979{
7980}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02007981#endif /* CONFIG_RT_GROUP_SCHED */
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007982
Dhaval Giani7c941432010-01-20 13:26:18 +01007983#ifdef CONFIG_CGROUP_SCHED
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007984static void free_sched_group(struct task_group *tg)
7985{
7986 free_fair_sched_group(tg);
7987 free_rt_sched_group(tg);
7988 kfree(tg);
7989}
7990
7991/* allocate runqueue etc for a new task group */
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02007992struct task_group *sched_create_group(struct task_group *parent)
Peter Zijlstrabccbe082008-02-13 15:45:40 +01007993{
7994 struct task_group *tg;
7995 unsigned long flags;
7996 int i;
7997
7998 tg = kzalloc(sizeof(*tg), GFP_KERNEL);
7999 if (!tg)
8000 return ERR_PTR(-ENOMEM);
8001
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02008002 if (!alloc_fair_sched_group(tg, parent))
Peter Zijlstrabccbe082008-02-13 15:45:40 +01008003 goto err;
8004
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02008005 if (!alloc_rt_sched_group(tg, parent))
Peter Zijlstrabccbe082008-02-13 15:45:40 +01008006 goto err;
8007
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008008 spin_lock_irqsave(&task_group_lock, flags);
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008009 for_each_possible_cpu(i) {
Peter Zijlstrabccbe082008-02-13 15:45:40 +01008010 register_fair_sched_group(tg, i);
8011 register_rt_sched_group(tg, i);
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008012 }
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008013 list_add_rcu(&tg->list, &task_groups);
Peter Zijlstraf473aa52008-04-19 19:45:00 +02008014
8015 WARN_ON(!parent); /* root should already exist */
8016
8017 tg->parent = parent;
Peter Zijlstraf473aa52008-04-19 19:45:00 +02008018 INIT_LIST_HEAD(&tg->children);
Zhang, Yanmin09f27242030-08-14 15:56:40 +08008019 list_add_rcu(&tg->siblings, &parent->children);
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008020 spin_unlock_irqrestore(&task_group_lock, flags);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008021
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008022 return tg;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008023
8024err:
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008025 free_sched_group(tg);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008026 return ERR_PTR(-ENOMEM);
8027}
8028
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008029/* rcu callback to free various structures associated with a task group */
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008030static void free_sched_group_rcu(struct rcu_head *rhp)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008031{
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008032 /* now it should be safe to free those cfs_rqs */
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008033 free_sched_group(container_of(rhp, struct task_group, rcu));
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008034}
8035
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008036/* Destroy runqueue etc associated with a task group */
Ingo Molnar4cf86d72007-10-15 17:00:14 +02008037void sched_destroy_group(struct task_group *tg)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008038{
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008039 unsigned long flags;
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008040 int i;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008041
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008042 spin_lock_irqsave(&task_group_lock, flags);
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008043 for_each_possible_cpu(i) {
Peter Zijlstrabccbe082008-02-13 15:45:40 +01008044 unregister_fair_sched_group(tg, i);
8045 unregister_rt_sched_group(tg, i);
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008046 }
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008047 list_del_rcu(&tg->list);
Peter Zijlstraf473aa52008-04-19 19:45:00 +02008048 list_del_rcu(&tg->siblings);
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008049 spin_unlock_irqrestore(&task_group_lock, flags);
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008050
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008051 /* wait for possible concurrent references to cfs_rqs complete */
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008052 call_rcu(&tg->rcu, free_sched_group_rcu);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008053}
8054
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008055/* change task's runqueue when it moves between groups.
Ingo Molnar3a252012007-10-15 17:00:12 +02008056 * The caller of this function should have put the task in its new group
8057 * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
8058 * reflect its new group.
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008059 */
8060void sched_move_task(struct task_struct *tsk)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008061{
8062 int on_rq, running;
8063 unsigned long flags;
8064 struct rq *rq;
8065
8066 rq = task_rq_lock(tsk, &flags);
8067
Dmitry Adamushko051a1d12007-12-18 15:21:13 +01008068 running = task_current(rq, tsk);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008069 on_rq = tsk->se.on_rq;
8070
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07008071 if (on_rq)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008072 dequeue_task(rq, tsk, 0);
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07008073 if (unlikely(running))
8074 tsk->sched_class->put_prev_task(rq, tsk);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008075
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008076 set_task_rq(tsk, task_cpu(tsk));
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008077
Peter Zijlstra810b3812008-02-29 15:21:01 -05008078#ifdef CONFIG_FAIR_GROUP_SCHED
8079 if (tsk->sched_class->moved_group)
Peter Zijlstra88ec22d2009-12-16 18:04:41 +01008080 tsk->sched_class->moved_group(tsk, on_rq);
Peter Zijlstra810b3812008-02-29 15:21:01 -05008081#endif
8082
Hiroshi Shimamoto0e1f3482008-03-10 11:01:20 -07008083 if (unlikely(running))
8084 tsk->sched_class->set_curr_task(rq);
8085 if (on_rq)
Peter Zijlstra371fd7e2010-03-24 16:38:48 +01008086 enqueue_task(rq, tsk, 0);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008087
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008088 task_rq_unlock(rq, &flags);
8089}
Dhaval Giani7c941432010-01-20 13:26:18 +01008090#endif /* CONFIG_CGROUP_SCHED */
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008091
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008092#ifdef CONFIG_FAIR_GROUP_SCHED
Peter Zijlstrac09595f2008-06-27 13:41:14 +02008093static void __set_se_shares(struct sched_entity *se, unsigned long shares)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008094{
8095 struct cfs_rq *cfs_rq = se->cfs_rq;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008096 int on_rq;
8097
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008098 on_rq = se->on_rq;
Peter Zijlstra62fb1852008-02-25 17:34:02 +01008099 if (on_rq)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008100 dequeue_entity(cfs_rq, se, 0);
8101
8102 se->load.weight = shares;
Peter Zijlstrae05510d2008-05-05 23:56:17 +02008103 se->load.inv_weight = 0;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008104
Peter Zijlstra62fb1852008-02-25 17:34:02 +01008105 if (on_rq)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008106 enqueue_entity(cfs_rq, se, 0);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02008107}
Peter Zijlstra62fb1852008-02-25 17:34:02 +01008108
Peter Zijlstrac09595f2008-06-27 13:41:14 +02008109static void set_se_shares(struct sched_entity *se, unsigned long shares)
8110{
8111 struct cfs_rq *cfs_rq = se->cfs_rq;
8112 struct rq *rq = cfs_rq->rq;
8113 unsigned long flags;
8114
Thomas Gleixner05fa7852009-11-17 14:28:38 +01008115 raw_spin_lock_irqsave(&rq->lock, flags);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02008116 __set_se_shares(se, shares);
Thomas Gleixner05fa7852009-11-17 14:28:38 +01008117 raw_spin_unlock_irqrestore(&rq->lock, flags);
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008118}
8119
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008120static DEFINE_MUTEX(shares_mutex);
8121
Ingo Molnar4cf86d72007-10-15 17:00:14 +02008122int sched_group_set_shares(struct task_group *tg, unsigned long shares)
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008123{
8124 int i;
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008125 unsigned long flags;
Ingo Molnarc61935f2008-01-22 11:24:58 +01008126
Peter Zijlstra62fb1852008-02-25 17:34:02 +01008127 /*
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02008128 * We can't change the weight of the root cgroup.
8129 */
8130 if (!tg->se[0])
8131 return -EINVAL;
8132
Peter Zijlstra18d95a22008-04-19 19:45:00 +02008133 if (shares < MIN_SHARES)
8134 shares = MIN_SHARES;
Miao Xiecb4ad1f2008-04-28 12:54:56 +08008135 else if (shares > MAX_SHARES)
8136 shares = MAX_SHARES;
Peter Zijlstra62fb1852008-02-25 17:34:02 +01008137
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008138 mutex_lock(&shares_mutex);
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008139 if (tg->shares == shares)
Dhaval Giani5cb350b2007-10-15 17:00:14 +02008140 goto done;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008141
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008142 spin_lock_irqsave(&task_group_lock, flags);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01008143 for_each_possible_cpu(i)
8144 unregister_fair_sched_group(tg, i);
Peter Zijlstraf473aa52008-04-19 19:45:00 +02008145 list_del_rcu(&tg->siblings);
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008146 spin_unlock_irqrestore(&task_group_lock, flags);
Srivatsa Vaddagiri6b2d7702008-01-25 21:08:00 +01008147
8148 /* wait for any ongoing reference to this group to finish */
8149 synchronize_sched();
8150
8151 /*
8152 * Now we are free to modify the group's share on each cpu
8153 * w/o tripping rebalance_share or load_balance_fair.
8154 */
8155 tg->shares = shares;
Peter Zijlstrac09595f2008-06-27 13:41:14 +02008156 for_each_possible_cpu(i) {
8157 /*
8158 * force a rebalance
8159 */
8160 cfs_rq_set_shares(tg->cfs_rq[i], 0);
Miao Xiecb4ad1f2008-04-28 12:54:56 +08008161 set_se_shares(tg->se[i], shares);
Peter Zijlstrac09595f2008-06-27 13:41:14 +02008162 }
Srivatsa Vaddagiri6b2d7702008-01-25 21:08:00 +01008163
8164 /*
8165 * Enable load balance activity on this group, by inserting it back on
8166 * each cpu's rq->leaf_cfs_rq_list.
8167 */
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008168 spin_lock_irqsave(&task_group_lock, flags);
Peter Zijlstrabccbe082008-02-13 15:45:40 +01008169 for_each_possible_cpu(i)
8170 register_fair_sched_group(tg, i);
Peter Zijlstraf473aa52008-04-19 19:45:00 +02008171 list_add_rcu(&tg->siblings, &tg->parent->children);
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008172 spin_unlock_irqrestore(&task_group_lock, flags);
Dhaval Giani5cb350b2007-10-15 17:00:14 +02008173done:
Peter Zijlstra8ed36992008-02-13 15:45:39 +01008174 mutex_unlock(&shares_mutex);
Srivatsa Vaddagiri9b5b7752007-10-15 17:00:09 +02008175 return 0;
Srivatsa Vaddagiri29f59db2007-10-15 17:00:07 +02008176}
8177
Dhaval Giani5cb350b2007-10-15 17:00:14 +02008178unsigned long sched_group_shares(struct task_group *tg)
8179{
8180 return tg->shares;
8181}
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008182#endif
Dhaval Giani5cb350b2007-10-15 17:00:14 +02008183
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008184#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008185/*
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008186 * Ensure that the real time constraints are schedulable.
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008187 */
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008188static DEFINE_MUTEX(rt_constraints_mutex);
8189
8190static unsigned long to_ratio(u64 period, u64 runtime)
8191{
8192 if (runtime == RUNTIME_INF)
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008193 return 1ULL << 20;
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008194
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008195 return div64_u64(runtime << 20, period);
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008196}
8197
Dhaval Giani521f1a242008-02-28 15:21:56 +05308198/* Must be called with tasklist_lock held */
8199static inline int tg_has_rt_tasks(struct task_group *tg)
8200{
8201 struct task_struct *g, *p;
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008202
Dhaval Giani521f1a242008-02-28 15:21:56 +05308203 do_each_thread(g, p) {
8204 if (rt_task(p) && rt_rq_of_se(&p->rt)->tg == tg)
8205 return 1;
8206 } while_each_thread(g, p);
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008207
Dhaval Giani521f1a242008-02-28 15:21:56 +05308208 return 0;
8209}
8210
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008211struct rt_schedulable_data {
8212 struct task_group *tg;
8213 u64 rt_period;
8214 u64 rt_runtime;
8215};
8216
8217static int tg_schedulable(struct task_group *tg, void *data)
8218{
8219 struct rt_schedulable_data *d = data;
8220 struct task_group *child;
8221 unsigned long total, sum = 0;
8222 u64 period, runtime;
8223
8224 period = ktime_to_ns(tg->rt_bandwidth.rt_period);
8225 runtime = tg->rt_bandwidth.rt_runtime;
8226
8227 if (tg == d->tg) {
8228 period = d->rt_period;
8229 runtime = d->rt_runtime;
8230 }
8231
Peter Zijlstra4653f802008-09-23 15:33:44 +02008232 /*
8233 * Cannot have more runtime than the period.
8234 */
8235 if (runtime > period && runtime != RUNTIME_INF)
8236 return -EINVAL;
8237
8238 /*
8239 * Ensure we don't starve existing RT tasks.
8240 */
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008241 if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
8242 return -EBUSY;
8243
8244 total = to_ratio(period, runtime);
8245
Peter Zijlstra4653f802008-09-23 15:33:44 +02008246 /*
8247 * Nobody can have more than the global setting allows.
8248 */
8249 if (total > to_ratio(global_rt_period(), global_rt_runtime()))
8250 return -EINVAL;
8251
8252 /*
8253 * The sum of our children's runtime should not exceed our own.
8254 */
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008255 list_for_each_entry_rcu(child, &tg->children, siblings) {
8256 period = ktime_to_ns(child->rt_bandwidth.rt_period);
8257 runtime = child->rt_bandwidth.rt_runtime;
8258
8259 if (child == d->tg) {
8260 period = d->rt_period;
8261 runtime = d->rt_runtime;
8262 }
8263
8264 sum += to_ratio(period, runtime);
8265 }
8266
8267 if (sum > total)
8268 return -EINVAL;
8269
8270 return 0;
8271}
8272
8273static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
8274{
8275 struct rt_schedulable_data data = {
8276 .tg = tg,
8277 .rt_period = period,
8278 .rt_runtime = runtime,
8279 };
8280
8281 return walk_tg_tree(tg_schedulable, tg_nop, &data);
8282}
8283
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008284static int tg_set_bandwidth(struct task_group *tg,
8285 u64 rt_period, u64 rt_runtime)
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008286{
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008287 int i, err = 0;
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008288
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008289 mutex_lock(&rt_constraints_mutex);
Dhaval Giani521f1a242008-02-28 15:21:56 +05308290 read_lock(&tasklist_lock);
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008291 err = __rt_schedulable(tg, rt_period, rt_runtime);
8292 if (err)
Dhaval Giani521f1a242008-02-28 15:21:56 +05308293 goto unlock;
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008294
Thomas Gleixner0986b112009-11-17 15:32:06 +01008295 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008296 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
8297 tg->rt_bandwidth.rt_runtime = rt_runtime;
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008298
8299 for_each_possible_cpu(i) {
8300 struct rt_rq *rt_rq = tg->rt_rq[i];
8301
Thomas Gleixner0986b112009-11-17 15:32:06 +01008302 raw_spin_lock(&rt_rq->rt_runtime_lock);
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008303 rt_rq->rt_runtime = rt_runtime;
Thomas Gleixner0986b112009-11-17 15:32:06 +01008304 raw_spin_unlock(&rt_rq->rt_runtime_lock);
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008305 }
Thomas Gleixner0986b112009-11-17 15:32:06 +01008306 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008307 unlock:
Dhaval Giani521f1a242008-02-28 15:21:56 +05308308 read_unlock(&tasklist_lock);
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008309 mutex_unlock(&rt_constraints_mutex);
8310
8311 return err;
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008312}
8313
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008314int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
8315{
8316 u64 rt_runtime, rt_period;
8317
8318 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
8319 rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
8320 if (rt_runtime_us < 0)
8321 rt_runtime = RUNTIME_INF;
8322
8323 return tg_set_bandwidth(tg, rt_period, rt_runtime);
8324}
8325
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008326long sched_group_rt_runtime(struct task_group *tg)
8327{
8328 u64 rt_runtime_us;
8329
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008330 if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008331 return -1;
8332
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008333 rt_runtime_us = tg->rt_bandwidth.rt_runtime;
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008334 do_div(rt_runtime_us, NSEC_PER_USEC);
8335 return rt_runtime_us;
8336}
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008337
8338int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
8339{
8340 u64 rt_runtime, rt_period;
8341
8342 rt_period = (u64)rt_period_us * NSEC_PER_USEC;
8343 rt_runtime = tg->rt_bandwidth.rt_runtime;
8344
Raistlin619b0482008-06-26 18:54:09 +02008345 if (rt_period == 0)
8346 return -EINVAL;
8347
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008348 return tg_set_bandwidth(tg, rt_period, rt_runtime);
8349}
8350
8351long sched_group_rt_period(struct task_group *tg)
8352{
8353 u64 rt_period_us;
8354
8355 rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
8356 do_div(rt_period_us, NSEC_PER_USEC);
8357 return rt_period_us;
8358}
8359
8360static int sched_rt_global_constraints(void)
8361{
Peter Zijlstra4653f802008-09-23 15:33:44 +02008362 u64 runtime, period;
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008363 int ret = 0;
8364
Hiroshi Shimamotoec5d4982008-09-10 17:00:19 -07008365 if (sysctl_sched_rt_period <= 0)
8366 return -EINVAL;
8367
Peter Zijlstra4653f802008-09-23 15:33:44 +02008368 runtime = global_rt_runtime();
8369 period = global_rt_period();
8370
8371 /*
8372 * Sanity check on the sysctl variables.
8373 */
8374 if (runtime > period && runtime != RUNTIME_INF)
8375 return -EINVAL;
Peter Zijlstra10b612f2008-06-19 14:22:27 +02008376
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008377 mutex_lock(&rt_constraints_mutex);
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008378 read_lock(&tasklist_lock);
Peter Zijlstra4653f802008-09-23 15:33:44 +02008379 ret = __rt_schedulable(NULL, 0, 0);
Peter Zijlstra9a7e0b12008-08-19 12:33:06 +02008380 read_unlock(&tasklist_lock);
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008381 mutex_unlock(&rt_constraints_mutex);
8382
8383 return ret;
8384}
Dhaval Giani54e99122009-02-27 15:13:54 +05308385
8386int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
8387{
8388 /* Don't accept realtime tasks when there is no way for them to run */
8389 if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
8390 return 0;
8391
8392 return 1;
8393}
8394
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02008395#else /* !CONFIG_RT_GROUP_SCHED */
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008396static int sched_rt_global_constraints(void)
8397{
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008398 unsigned long flags;
8399 int i;
8400
Hiroshi Shimamotoec5d4982008-09-10 17:00:19 -07008401 if (sysctl_sched_rt_period <= 0)
8402 return -EINVAL;
8403
Peter Zijlstra60aa6052009-05-05 17:50:21 +02008404 /*
8405 * There's always some RT tasks in the root group
8406 * -- migration, kstopmachine etc..
8407 */
8408 if (sysctl_sched_rt_runtime == 0)
8409 return -EBUSY;
8410
Thomas Gleixner0986b112009-11-17 15:32:06 +01008411 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008412 for_each_possible_cpu(i) {
8413 struct rt_rq *rt_rq = &cpu_rq(i)->rt;
8414
Thomas Gleixner0986b112009-11-17 15:32:06 +01008415 raw_spin_lock(&rt_rq->rt_runtime_lock);
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008416 rt_rq->rt_runtime = global_rt_runtime();
Thomas Gleixner0986b112009-11-17 15:32:06 +01008417 raw_spin_unlock(&rt_rq->rt_runtime_lock);
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008418 }
Thomas Gleixner0986b112009-11-17 15:32:06 +01008419 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
Peter Zijlstraac086bc2008-04-19 19:44:58 +02008420
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008421 return 0;
8422}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02008423#endif /* CONFIG_RT_GROUP_SCHED */
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008424
8425int sched_rt_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008426 void __user *buffer, size_t *lenp,
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008427 loff_t *ppos)
8428{
8429 int ret;
8430 int old_period, old_runtime;
8431 static DEFINE_MUTEX(mutex);
8432
8433 mutex_lock(&mutex);
8434 old_period = sysctl_sched_rt_period;
8435 old_runtime = sysctl_sched_rt_runtime;
8436
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07008437 ret = proc_dointvec(table, write, buffer, lenp, ppos);
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008438
8439 if (!ret && write) {
8440 ret = sched_rt_global_constraints();
8441 if (ret) {
8442 sysctl_sched_rt_period = old_period;
8443 sysctl_sched_rt_runtime = old_runtime;
8444 } else {
8445 def_rt_bandwidth.rt_runtime = global_rt_runtime();
8446 def_rt_bandwidth.rt_period =
8447 ns_to_ktime(global_rt_period());
8448 }
8449 }
8450 mutex_unlock(&mutex);
8451
8452 return ret;
8453}
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008454
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008455#ifdef CONFIG_CGROUP_SCHED
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008456
8457/* return corresponding task_group object of a cgroup */
Paul Menage2b01dfe2007-10-24 18:23:50 +02008458static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008459{
Paul Menage2b01dfe2007-10-24 18:23:50 +02008460 return container_of(cgroup_subsys_state(cgrp, cpu_cgroup_subsys_id),
8461 struct task_group, css);
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008462}
8463
8464static struct cgroup_subsys_state *
Paul Menage2b01dfe2007-10-24 18:23:50 +02008465cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008466{
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02008467 struct task_group *tg, *parent;
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008468
Paul Menage2b01dfe2007-10-24 18:23:50 +02008469 if (!cgrp->parent) {
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008470 /* This is early initialization for the top cgroup */
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008471 return &init_task_group.css;
8472 }
8473
Dhaval Gianiec7dc8a2008-04-19 19:44:59 +02008474 parent = cgroup_tg(cgrp->parent);
8475 tg = sched_create_group(parent);
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008476 if (IS_ERR(tg))
8477 return ERR_PTR(-ENOMEM);
8478
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008479 return &tg->css;
8480}
8481
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01008482static void
8483cpu_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008484{
Paul Menage2b01dfe2007-10-24 18:23:50 +02008485 struct task_group *tg = cgroup_tg(cgrp);
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008486
8487 sched_destroy_group(tg);
8488}
8489
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01008490static int
Ben Blumbe367d02009-09-23 15:56:31 -07008491cpu_cgroup_can_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008492{
Peter Zijlstrab68aa232008-02-13 15:45:40 +01008493#ifdef CONFIG_RT_GROUP_SCHED
Dhaval Giani54e99122009-02-27 15:13:54 +05308494 if (!sched_rt_can_attach(cgroup_tg(cgrp), tsk))
Peter Zijlstrab68aa232008-02-13 15:45:40 +01008495 return -EINVAL;
8496#else
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008497 /* We don't support RT-tasks being in separate groups */
8498 if (tsk->sched_class != &fair_sched_class)
8499 return -EINVAL;
Peter Zijlstrab68aa232008-02-13 15:45:40 +01008500#endif
Ben Blumbe367d02009-09-23 15:56:31 -07008501 return 0;
8502}
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008503
Ben Blumbe367d02009-09-23 15:56:31 -07008504static int
8505cpu_cgroup_can_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
8506 struct task_struct *tsk, bool threadgroup)
8507{
8508 int retval = cpu_cgroup_can_attach_task(cgrp, tsk);
8509 if (retval)
8510 return retval;
8511 if (threadgroup) {
8512 struct task_struct *c;
8513 rcu_read_lock();
8514 list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
8515 retval = cpu_cgroup_can_attach_task(cgrp, c);
8516 if (retval) {
8517 rcu_read_unlock();
8518 return retval;
8519 }
8520 }
8521 rcu_read_unlock();
8522 }
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008523 return 0;
8524}
8525
8526static void
Paul Menage2b01dfe2007-10-24 18:23:50 +02008527cpu_cgroup_attach(struct cgroup_subsys *ss, struct cgroup *cgrp,
Ben Blumbe367d02009-09-23 15:56:31 -07008528 struct cgroup *old_cont, struct task_struct *tsk,
8529 bool threadgroup)
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008530{
8531 sched_move_task(tsk);
Ben Blumbe367d02009-09-23 15:56:31 -07008532 if (threadgroup) {
8533 struct task_struct *c;
8534 rcu_read_lock();
8535 list_for_each_entry_rcu(c, &tsk->thread_group, thread_group) {
8536 sched_move_task(c);
8537 }
8538 rcu_read_unlock();
8539 }
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008540}
8541
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008542#ifdef CONFIG_FAIR_GROUP_SCHED
Paul Menagef4c753b2008-04-29 00:59:56 -07008543static int cpu_shares_write_u64(struct cgroup *cgrp, struct cftype *cftype,
Paul Menage2b01dfe2007-10-24 18:23:50 +02008544 u64 shareval)
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008545{
Paul Menage2b01dfe2007-10-24 18:23:50 +02008546 return sched_group_set_shares(cgroup_tg(cgrp), shareval);
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008547}
8548
Paul Menagef4c753b2008-04-29 00:59:56 -07008549static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008550{
Paul Menage2b01dfe2007-10-24 18:23:50 +02008551 struct task_group *tg = cgroup_tg(cgrp);
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008552
8553 return (u64) tg->shares;
8554}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02008555#endif /* CONFIG_FAIR_GROUP_SCHED */
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008556
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008557#ifdef CONFIG_RT_GROUP_SCHED
Mirco Tischler0c708142008-05-14 16:05:46 -07008558static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
Paul Menage06ecb272008-04-29 01:00:06 -07008559 s64 val)
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008560{
Paul Menage06ecb272008-04-29 01:00:06 -07008561 return sched_group_set_rt_runtime(cgroup_tg(cgrp), val);
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008562}
8563
Paul Menage06ecb272008-04-29 01:00:06 -07008564static s64 cpu_rt_runtime_read(struct cgroup *cgrp, struct cftype *cft)
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008565{
Paul Menage06ecb272008-04-29 01:00:06 -07008566 return sched_group_rt_runtime(cgroup_tg(cgrp));
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008567}
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008568
8569static int cpu_rt_period_write_uint(struct cgroup *cgrp, struct cftype *cftype,
8570 u64 rt_period_us)
8571{
8572 return sched_group_set_rt_period(cgroup_tg(cgrp), rt_period_us);
8573}
8574
8575static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
8576{
8577 return sched_group_rt_period(cgroup_tg(cgrp));
8578}
Dhaval Giani6d6bc0a2008-05-30 14:23:45 +02008579#endif /* CONFIG_RT_GROUP_SCHED */
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008580
Paul Menagefe5c7cc2007-10-29 21:18:11 +01008581static struct cftype cpu_files[] = {
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008582#ifdef CONFIG_FAIR_GROUP_SCHED
Paul Menagefe5c7cc2007-10-29 21:18:11 +01008583 {
8584 .name = "shares",
Paul Menagef4c753b2008-04-29 00:59:56 -07008585 .read_u64 = cpu_shares_read_u64,
8586 .write_u64 = cpu_shares_write_u64,
Paul Menagefe5c7cc2007-10-29 21:18:11 +01008587 },
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008588#endif
8589#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008590 {
Peter Zijlstra9f0c1e52008-02-13 15:45:39 +01008591 .name = "rt_runtime_us",
Paul Menage06ecb272008-04-29 01:00:06 -07008592 .read_s64 = cpu_rt_runtime_read,
8593 .write_s64 = cpu_rt_runtime_write,
Peter Zijlstra6f505b12008-01-25 21:08:30 +01008594 },
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008595 {
8596 .name = "rt_period_us",
Paul Menagef4c753b2008-04-29 00:59:56 -07008597 .read_u64 = cpu_rt_period_read_uint,
8598 .write_u64 = cpu_rt_period_write_uint,
Peter Zijlstrad0b27fa2008-04-19 19:44:57 +02008599 },
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008600#endif
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008601};
8602
8603static int cpu_cgroup_populate(struct cgroup_subsys *ss, struct cgroup *cont)
8604{
Paul Menagefe5c7cc2007-10-29 21:18:11 +01008605 return cgroup_add_files(cont, ss, cpu_files, ARRAY_SIZE(cpu_files));
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008606}
8607
8608struct cgroup_subsys cpu_cgroup_subsys = {
Ingo Molnar38605ca2007-10-29 21:18:11 +01008609 .name = "cpu",
8610 .create = cpu_cgroup_create,
8611 .destroy = cpu_cgroup_destroy,
8612 .can_attach = cpu_cgroup_can_attach,
8613 .attach = cpu_cgroup_attach,
8614 .populate = cpu_cgroup_populate,
8615 .subsys_id = cpu_cgroup_subsys_id,
Srivatsa Vaddagiri68318b82007-10-18 23:41:03 -07008616 .early_init = 1,
8617};
8618
Peter Zijlstra052f1dc2008-02-13 15:45:40 +01008619#endif /* CONFIG_CGROUP_SCHED */
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008620
8621#ifdef CONFIG_CGROUP_CPUACCT
8622
8623/*
8624 * CPU accounting code for task groups.
8625 *
8626 * Based on the work by Paul Menage (menage@google.com) and Balbir Singh
8627 * (balbir@in.ibm.com).
8628 */
8629
Bharata B Rao934352f2008-11-10 20:41:13 +05308630/* track cpu usage of a group of tasks and its child groups */
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008631struct cpuacct {
8632 struct cgroup_subsys_state css;
8633 /* cpuusage holds pointer to a u64-type object on every cpu */
Tejun Heo43cf38e2010-02-02 14:38:57 +09008634 u64 __percpu *cpuusage;
Bharata B Raoef12fef2009-03-31 10:02:22 +05308635 struct percpu_counter cpustat[CPUACCT_STAT_NSTATS];
Bharata B Rao934352f2008-11-10 20:41:13 +05308636 struct cpuacct *parent;
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008637};
8638
8639struct cgroup_subsys cpuacct_subsys;
8640
8641/* return cpu accounting group corresponding to this container */
Dhaval Giani32cd7562008-02-29 10:02:43 +05308642static inline struct cpuacct *cgroup_ca(struct cgroup *cgrp)
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008643{
Dhaval Giani32cd7562008-02-29 10:02:43 +05308644 return container_of(cgroup_subsys_state(cgrp, cpuacct_subsys_id),
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008645 struct cpuacct, css);
8646}
8647
8648/* return cpu accounting group to which this task belongs */
8649static inline struct cpuacct *task_ca(struct task_struct *tsk)
8650{
8651 return container_of(task_subsys_state(tsk, cpuacct_subsys_id),
8652 struct cpuacct, css);
8653}
8654
8655/* create a new cpu accounting group */
8656static struct cgroup_subsys_state *cpuacct_create(
Dhaval Giani32cd7562008-02-29 10:02:43 +05308657 struct cgroup_subsys *ss, struct cgroup *cgrp)
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008658{
8659 struct cpuacct *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
Bharata B Raoef12fef2009-03-31 10:02:22 +05308660 int i;
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008661
8662 if (!ca)
Bharata B Raoef12fef2009-03-31 10:02:22 +05308663 goto out;
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008664
8665 ca->cpuusage = alloc_percpu(u64);
Bharata B Raoef12fef2009-03-31 10:02:22 +05308666 if (!ca->cpuusage)
8667 goto out_free_ca;
8668
8669 for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
8670 if (percpu_counter_init(&ca->cpustat[i], 0))
8671 goto out_free_counters;
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008672
Bharata B Rao934352f2008-11-10 20:41:13 +05308673 if (cgrp->parent)
8674 ca->parent = cgroup_ca(cgrp->parent);
8675
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008676 return &ca->css;
Bharata B Raoef12fef2009-03-31 10:02:22 +05308677
8678out_free_counters:
8679 while (--i >= 0)
8680 percpu_counter_destroy(&ca->cpustat[i]);
8681 free_percpu(ca->cpuusage);
8682out_free_ca:
8683 kfree(ca);
8684out:
8685 return ERR_PTR(-ENOMEM);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008686}
8687
8688/* destroy an existing cpu accounting group */
Ingo Molnar41a2d6c2007-12-05 15:46:09 +01008689static void
Dhaval Giani32cd7562008-02-29 10:02:43 +05308690cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008691{
Dhaval Giani32cd7562008-02-29 10:02:43 +05308692 struct cpuacct *ca = cgroup_ca(cgrp);
Bharata B Raoef12fef2009-03-31 10:02:22 +05308693 int i;
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008694
Bharata B Raoef12fef2009-03-31 10:02:22 +05308695 for (i = 0; i < CPUACCT_STAT_NSTATS; i++)
8696 percpu_counter_destroy(&ca->cpustat[i]);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008697 free_percpu(ca->cpuusage);
8698 kfree(ca);
8699}
8700
Ken Chen720f5492008-12-15 22:02:01 -08008701static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
8702{
Rusty Russellb36128c2009-02-20 16:29:08 +09008703 u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
Ken Chen720f5492008-12-15 22:02:01 -08008704 u64 data;
8705
8706#ifndef CONFIG_64BIT
8707 /*
8708 * Take rq->lock to make 64-bit read safe on 32-bit platforms.
8709 */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01008710 raw_spin_lock_irq(&cpu_rq(cpu)->lock);
Ken Chen720f5492008-12-15 22:02:01 -08008711 data = *cpuusage;
Thomas Gleixner05fa7852009-11-17 14:28:38 +01008712 raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
Ken Chen720f5492008-12-15 22:02:01 -08008713#else
8714 data = *cpuusage;
8715#endif
8716
8717 return data;
8718}
8719
8720static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
8721{
Rusty Russellb36128c2009-02-20 16:29:08 +09008722 u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
Ken Chen720f5492008-12-15 22:02:01 -08008723
8724#ifndef CONFIG_64BIT
8725 /*
8726 * Take rq->lock to make 64-bit write safe on 32-bit platforms.
8727 */
Thomas Gleixner05fa7852009-11-17 14:28:38 +01008728 raw_spin_lock_irq(&cpu_rq(cpu)->lock);
Ken Chen720f5492008-12-15 22:02:01 -08008729 *cpuusage = val;
Thomas Gleixner05fa7852009-11-17 14:28:38 +01008730 raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
Ken Chen720f5492008-12-15 22:02:01 -08008731#else
8732 *cpuusage = val;
8733#endif
8734}
8735
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008736/* return total cpu usage (in nanoseconds) of a group */
Dhaval Giani32cd7562008-02-29 10:02:43 +05308737static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008738{
Dhaval Giani32cd7562008-02-29 10:02:43 +05308739 struct cpuacct *ca = cgroup_ca(cgrp);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008740 u64 totalcpuusage = 0;
8741 int i;
8742
Ken Chen720f5492008-12-15 22:02:01 -08008743 for_each_present_cpu(i)
8744 totalcpuusage += cpuacct_cpuusage_read(ca, i);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008745
8746 return totalcpuusage;
8747}
8748
Dhaval Giani0297b802008-02-29 10:02:44 +05308749static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
8750 u64 reset)
8751{
8752 struct cpuacct *ca = cgroup_ca(cgrp);
8753 int err = 0;
8754 int i;
8755
8756 if (reset) {
8757 err = -EINVAL;
8758 goto out;
8759 }
8760
Ken Chen720f5492008-12-15 22:02:01 -08008761 for_each_present_cpu(i)
8762 cpuacct_cpuusage_write(ca, i, 0);
Dhaval Giani0297b802008-02-29 10:02:44 +05308763
Dhaval Giani0297b802008-02-29 10:02:44 +05308764out:
8765 return err;
8766}
8767
Ken Chene9515c32008-12-15 22:04:15 -08008768static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
8769 struct seq_file *m)
8770{
8771 struct cpuacct *ca = cgroup_ca(cgroup);
8772 u64 percpu;
8773 int i;
8774
8775 for_each_present_cpu(i) {
8776 percpu = cpuacct_cpuusage_read(ca, i);
8777 seq_printf(m, "%llu ", (unsigned long long) percpu);
8778 }
8779 seq_printf(m, "\n");
8780 return 0;
8781}
8782
Bharata B Raoef12fef2009-03-31 10:02:22 +05308783static const char *cpuacct_stat_desc[] = {
8784 [CPUACCT_STAT_USER] = "user",
8785 [CPUACCT_STAT_SYSTEM] = "system",
8786};
8787
8788static int cpuacct_stats_show(struct cgroup *cgrp, struct cftype *cft,
8789 struct cgroup_map_cb *cb)
8790{
8791 struct cpuacct *ca = cgroup_ca(cgrp);
8792 int i;
8793
8794 for (i = 0; i < CPUACCT_STAT_NSTATS; i++) {
8795 s64 val = percpu_counter_read(&ca->cpustat[i]);
8796 val = cputime64_to_clock_t(val);
8797 cb->fill(cb, cpuacct_stat_desc[i], val);
8798 }
8799 return 0;
8800}
8801
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008802static struct cftype files[] = {
8803 {
8804 .name = "usage",
Paul Menagef4c753b2008-04-29 00:59:56 -07008805 .read_u64 = cpuusage_read,
8806 .write_u64 = cpuusage_write,
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008807 },
Ken Chene9515c32008-12-15 22:04:15 -08008808 {
8809 .name = "usage_percpu",
8810 .read_seq_string = cpuacct_percpu_seq_read,
8811 },
Bharata B Raoef12fef2009-03-31 10:02:22 +05308812 {
8813 .name = "stat",
8814 .read_map = cpuacct_stats_show,
8815 },
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008816};
8817
Dhaval Giani32cd7562008-02-29 10:02:43 +05308818static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008819{
Dhaval Giani32cd7562008-02-29 10:02:43 +05308820 return cgroup_add_files(cgrp, ss, files, ARRAY_SIZE(files));
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008821}
8822
8823/*
8824 * charge this task's execution time to its accounting group.
8825 *
8826 * called with rq->lock held.
8827 */
8828static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
8829{
8830 struct cpuacct *ca;
Bharata B Rao934352f2008-11-10 20:41:13 +05308831 int cpu;
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008832
Li Zefanc40c6f82009-02-26 15:40:15 +08008833 if (unlikely(!cpuacct_subsys.active))
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008834 return;
8835
Bharata B Rao934352f2008-11-10 20:41:13 +05308836 cpu = task_cpu(tsk);
Bharata B Raoa18b83b2009-03-23 10:02:53 +05308837
8838 rcu_read_lock();
8839
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008840 ca = task_ca(tsk);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008841
Bharata B Rao934352f2008-11-10 20:41:13 +05308842 for (; ca; ca = ca->parent) {
Rusty Russellb36128c2009-02-20 16:29:08 +09008843 u64 *cpuusage = per_cpu_ptr(ca->cpuusage, cpu);
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008844 *cpuusage += cputime;
8845 }
Bharata B Raoa18b83b2009-03-23 10:02:53 +05308846
8847 rcu_read_unlock();
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008848}
8849
Bharata B Raoef12fef2009-03-31 10:02:22 +05308850/*
Anton Blanchardfa535a72010-02-02 14:46:13 -08008851 * When CONFIG_VIRT_CPU_ACCOUNTING is enabled one jiffy can be very large
8852 * in cputime_t units. As a result, cpuacct_update_stats calls
8853 * percpu_counter_add with values large enough to always overflow the
8854 * per cpu batch limit causing bad SMP scalability.
8855 *
8856 * To fix this we scale percpu_counter_batch by cputime_one_jiffy so we
8857 * batch the same amount of time with CONFIG_VIRT_CPU_ACCOUNTING disabled
8858 * and enabled. We cap it at INT_MAX which is the largest allowed batch value.
8859 */
8860#ifdef CONFIG_SMP
8861#define CPUACCT_BATCH \
8862 min_t(long, percpu_counter_batch * cputime_one_jiffy, INT_MAX)
8863#else
8864#define CPUACCT_BATCH 0
8865#endif
8866
8867/*
Bharata B Raoef12fef2009-03-31 10:02:22 +05308868 * Charge the system/user time to the task's accounting group.
8869 */
8870static void cpuacct_update_stats(struct task_struct *tsk,
8871 enum cpuacct_stat_index idx, cputime_t val)
8872{
8873 struct cpuacct *ca;
Anton Blanchardfa535a72010-02-02 14:46:13 -08008874 int batch = CPUACCT_BATCH;
Bharata B Raoef12fef2009-03-31 10:02:22 +05308875
8876 if (unlikely(!cpuacct_subsys.active))
8877 return;
8878
8879 rcu_read_lock();
8880 ca = task_ca(tsk);
8881
8882 do {
Anton Blanchardfa535a72010-02-02 14:46:13 -08008883 __percpu_counter_add(&ca->cpustat[idx], val, batch);
Bharata B Raoef12fef2009-03-31 10:02:22 +05308884 ca = ca->parent;
8885 } while (ca);
8886 rcu_read_unlock();
8887}
8888
Srivatsa Vaddagirid842de82007-12-02 20:04:49 +01008889struct cgroup_subsys cpuacct_subsys = {
8890 .name = "cpuacct",
8891 .create = cpuacct_create,
8892 .destroy = cpuacct_destroy,
8893 .populate = cpuacct_populate,
8894 .subsys_id = cpuacct_subsys_id,
8895};
8896#endif /* CONFIG_CGROUP_CPUACCT */
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008897
8898#ifndef CONFIG_SMP
8899
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008900void synchronize_sched_expedited(void)
8901{
Paul E. McKenneyfc390cd2010-05-06 11:42:52 -07008902 barrier();
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008903}
8904EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
8905
8906#else /* #ifndef CONFIG_SMP */
8907
Paul E. McKenneycc631fb2010-05-06 18:49:21 +02008908static atomic_t synchronize_sched_expedited_count = ATOMIC_INIT(0);
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008909
Paul E. McKenneycc631fb2010-05-06 18:49:21 +02008910static int synchronize_sched_expedited_cpu_stop(void *data)
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008911{
Tejun Heo969c7922010-05-06 18:49:21 +02008912 /*
8913 * There must be a full memory barrier on each affected CPU
8914 * between the time that try_stop_cpus() is called and the
8915 * time that it returns.
8916 *
8917 * In the current initial implementation of cpu_stop, the
8918 * above condition is already met when the control reaches
8919 * this point and the following smp_mb() is not strictly
8920 * necessary. Do smp_mb() anyway for documentation and
8921 * robustness against future implementation changes.
8922 */
Paul E. McKenneycc631fb2010-05-06 18:49:21 +02008923 smp_mb(); /* See above comment block. */
Tejun Heo969c7922010-05-06 18:49:21 +02008924 return 0;
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008925}
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008926
8927/*
8928 * Wait for an rcu-sched grace period to elapse, but use "big hammer"
8929 * approach to force grace period to end quickly. This consumes
8930 * significant time on all CPUs, and is thus not recommended for
8931 * any sort of common-case code.
8932 *
8933 * Note that it is illegal to call this function while holding any
8934 * lock that is acquired by a CPU-hotplug notifier. Failing to
8935 * observe this restriction will result in deadlock.
8936 */
8937void synchronize_sched_expedited(void)
8938{
Tejun Heo969c7922010-05-06 18:49:21 +02008939 int snap, trycount = 0;
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008940
8941 smp_mb(); /* ensure prior mod happens before capturing snap. */
Tejun Heo969c7922010-05-06 18:49:21 +02008942 snap = atomic_read(&synchronize_sched_expedited_count) + 1;
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008943 get_online_cpus();
Tejun Heo969c7922010-05-06 18:49:21 +02008944 while (try_stop_cpus(cpu_online_mask,
8945 synchronize_sched_expedited_cpu_stop,
Tejun Heo94458d52010-05-06 18:49:21 +02008946 NULL) == -EAGAIN) {
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008947 put_online_cpus();
8948 if (trycount++ < 10)
8949 udelay(trycount * num_online_cpus());
8950 else {
8951 synchronize_sched();
8952 return;
8953 }
Tejun Heo969c7922010-05-06 18:49:21 +02008954 if (atomic_read(&synchronize_sched_expedited_count) - snap > 0) {
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008955 smp_mb(); /* ensure test happens before caller kfree */
8956 return;
8957 }
8958 get_online_cpus();
8959 }
Tejun Heo969c7922010-05-06 18:49:21 +02008960 atomic_inc(&synchronize_sched_expedited_count);
Paul E. McKenneycc631fb2010-05-06 18:49:21 +02008961 smp_mb__after_atomic_inc(); /* ensure post-GP actions seen after GP. */
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008962 put_online_cpus();
Paul E. McKenney03b042b2009-06-25 09:08:16 -07008963}
8964EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
8965
8966#endif /* #else #ifndef CONFIG_SMP */