Prev: [tip:sched/urgent] sched: set_cpus_allowed_ptr(): Don't use rq->migration_thread after unlock
Next: [tip:perf/core] perf, x86: implement ARCH_PERFMON_EVENTSEL bit masks
From: tip-bot for Peter Zijlstra on 2 Apr 2010 15:20
Commit-ID: 65cc8e4859ff29a9ddc989c88557d6059834c2a2
Gitweb: http://git.kernel.org/tip/65cc8e4859ff29a9ddc989c88557d6059834c2a2
Author: Peter Zijlstra <a.p.zijlstra(a)chello.nl>
AuthorDate: Thu, 25 Mar 2010 21:05:16 +0100
Committer: Ingo Molnar <mingo(a)elte.hu>
CommitDate: Fri, 2 Apr 2010 20:12:04 +0200

sched: Optimize task_rq_lock()

Now that we hold the rq->lock over set_task_cpu() again, we can do
away with most of the TASK_WAKING checks and reduce them again to
set_cpus_allowed_ptr().

Removes some conditionals from scheduling hot-paths.

Signed-off-by: Peter Zijlstra <a.p.zijlstra(a)chello.nl>
Cc: Oleg Nesterov <oleg(a)redhat.com>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo(a)elte.hu>
---
kernel/sched.c | 23 +++++++++++++++--------
1 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/kernel/sched.c b/kernel/sched.c
index dcd1773..51d336e 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -914,8 +914,8 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
#endif /* __ARCH_WANT_UNLOCKED_CTXSW */

/*
- * Check whether the task is waking, we use this to synchronize against
- * ttwu() so that task_cpu() reports a stable number.
+ * Check whether the task is waking, we use this to synchronize ->cpus_allowed
+ * against ttwu().
*/
static inline int task_is_waking(struct task_struct *p)
{
@@ -932,11 +932,9 @@ static inline struct rq *__task_rq_lock(struct task_struct *p)
struct rq *rq;

for (;;) {
- while (task_is_waking(p))
- cpu_relax();
rq = task_rq(p);
raw_spin_lock(&rq->lock);
- if (likely(rq == task_rq(p) && !task_is_waking(p)))
+ if (likely(rq == task_rq(p)))
return rq;
raw_spin_unlock(&rq->lock);
}
@@ -953,12 +951,10 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
struct rq *rq;

for (;;) {
- while (task_is_waking(p))
- cpu_relax();
local_irq_save(*flags);
rq = task_rq(p);
raw_spin_lock(&rq->lock);
- if (likely(rq == task_rq(p) && !task_is_waking(p)))
+ if (likely(rq == task_rq(p)))
return rq;
raw_spin_unlock_irqrestore(&rq->lock, *flags);
}
@@ -5262,7 +5258,18 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
struct rq *rq;
int ret = 0;

+ /*
+ * Serialize against TASK_WAKING so that ttwu() and wunt() can
+ * drop the rq->lock and still rely on ->cpus_allowed.
+ */
+again:
+ while (task_is_waking(p))
+ cpu_relax();
rq = task_rq_lock(p, &flags);
+ if (task_is_waking(p)) {
+ task_rq_unlock(rq, &flags);
+ goto again;
+ }

if (!cpumask_intersects(new_mask, cpu_active_mask)) {
ret = -EINVAL;
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo(a)vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
 | 
Pages: 1
Prev: [tip:sched/urgent] sched: set_cpus_allowed_ptr(): Don't use rq->migration_thread after unlock
Next: [tip:perf/core] perf, x86: implement ARCH_PERFMON_EVENTSEL bit masks