Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

torvalds · torvalds · commit dd0508093b79 · 2013-12-17T12:35:54.000-08:00
Pull scheduler fixes from Ingo Molnar:
 "Three fixes for scheduler crashes, each triggers in relatively rare,
  hardware environment dependent situations"

* 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched/fair: Rework sched_fair time accounting
  math64: Add mul_u64_u32_shr()
  sched: Remove PREEMPT_NEED_RESCHED from generic code
  sched: Initialize power_orig for overlapping groups
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
@@ -26,6 +26,7 @@ config X86
 	select HAVE_AOUT if X86_32
 	select HAVE_UNSTABLE_SCHED_CLOCK
 	select ARCH_SUPPORTS_NUMA_BALANCING
+	select ARCH_SUPPORTS_INT128 if X86_64
 	select ARCH_WANTS_PROT_NUMA_PROT_NONE
 	select HAVE_IDE
 	select HAVE_OPROFILE
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
@@ -7,6 +7,12 @@
 
 DECLARE_PER_CPU(int, __preempt_count);
 
+/*
+ * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
+ * that a decrement hitting 0 means we can and should reschedule.
+ */
+#define PREEMPT_ENABLED	(0 + PREEMPT_NEED_RESCHED)
+
 /*
  * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
  * that think a non-zero value indicates we cannot preempt.
@@ -74,6 +80,11 @@ static __always_inline void __preempt_count_sub(int val)
 	__this_cpu_add_4(__preempt_count, -val);
 }
 
+/*
+ * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
+ * a decrement which hits zero means we have no preempt_count and should
+ * reschedule.
+ */
 static __always_inline bool __preempt_count_dec_and_test(void)
 {
 	GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), "e");
diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h
@@ -3,25 +3,18 @@
 
 #include <linux/thread_info.h>
 
-/*
- * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
- * that think a non-zero value indicates we cannot preempt.
- */
+#define PREEMPT_ENABLED	(0)
+
 static __always_inline int preempt_count(void)
 {
-	return current_thread_info()->preempt_count & ~PREEMPT_NEED_RESCHED;
+	return current_thread_info()->preempt_count;
 }
 
 static __always_inline int *preempt_count_ptr(void)
 {
 	return &current_thread_info()->preempt_count;
 }
 
-/*
- * We now loose PREEMPT_NEED_RESCHED and cause an extra reschedule; however the
- * alternative is loosing a reschedule. Better schedule too often -- also this
- * should be a very rare operation.
- */
 static __always_inline void preempt_count_set(int pc)
 {
 	*preempt_count_ptr() = pc;
@@ -41,28 +34,17 @@ static __always_inline void preempt_count_set(int pc)
 	task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \
 } while (0)
 
-/*
- * We fold the NEED_RESCHED bit into the preempt count such that
- * preempt_enable() can decrement and test for needing to reschedule with a
- * single instruction.
- *
- * We invert the actual bit, so that when the decrement hits 0 we know we both
- * need to resched (the bit is cleared) and can resched (no preempt count).
- */
-
 static __always_inline void set_preempt_need_resched(void)
 {
-	*preempt_count_ptr() &= ~PREEMPT_NEED_RESCHED;
 }
 
 static __always_inline void clear_preempt_need_resched(void)
 {
-	*preempt_count_ptr() |= PREEMPT_NEED_RESCHED;
 }
 
 static __always_inline bool test_preempt_need_resched(void)
 {
-	return !(*preempt_count_ptr() & PREEMPT_NEED_RESCHED);
+	return false;
 }
 
 /*
@@ -81,15 +63,20 @@ static __always_inline void __preempt_count_sub(int val)
 
 static __always_inline bool __preempt_count_dec_and_test(void)
 {
-	return !--*preempt_count_ptr();
+	/*
+	 * Because of load-store architectures cannot do per-cpu atomic
+	 * operations; we cannot use PREEMPT_NEED_RESCHED because it might get
+	 * lost.
+	 */
+	return !--*preempt_count_ptr() && tif_need_resched();
 }
 
 /*
  * Returns true when we need to resched and can (barring IRQ state).
  */
 static __always_inline bool should_resched(void)
 {
-	return unlikely(!*preempt_count_ptr());
+	return unlikely(!preempt_count() && tif_need_resched());
 }
 
 #ifdef CONFIG_PREEMPT
diff --git a/include/linux/math64.h b/include/linux/math64.h
@@ -133,4 +133,34 @@ __iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder)
 	return ret;
 }
 
+#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__)
+
+#ifndef mul_u64_u32_shr
+static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
+{
+	return (u64)(((unsigned __int128)a * mul) >> shift);
+}
+#endif /* mul_u64_u32_shr */
+
+#else
+
+#ifndef mul_u64_u32_shr
+static inline u64 mul_u64_u32_shr(u64 a, u32 mul, unsigned int shift)
+{
+	u32 ah, al;
+	u64 ret;
+
+	al = a;
+	ah = a >> 32;
+
+	ret = ((u64)al * mul) >> shift;
+	if (ah)
+		ret += ((u64)ah * mul) << (32 - shift);
+
+	return ret;
+}
+#endif /* mul_u64_u32_shr */
+
+#endif
+
 #endif /* _LINUX_MATH64_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
@@ -440,8 +440,6 @@ struct task_cputime {
 		.sum_exec_runtime = 0,				\
 	}
 
-#define PREEMPT_ENABLED		(PREEMPT_NEED_RESCHED)
-
 #ifdef CONFIG_PREEMPT_COUNT
 #define PREEMPT_DISABLED	(1 + PREEMPT_ENABLED)
 #else
@@ -932,7 +930,8 @@ struct pipe_inode_info;
 struct uts_namespace;
 
 struct load_weight {
-	unsigned long weight, inv_weight;
+	unsigned long weight;
+	u32 inv_weight;
 };
 
 struct sched_avg {
diff --git a/init/Kconfig b/init/Kconfig
@@ -809,6 +809,12 @@ config GENERIC_SCHED_CLOCK
 config ARCH_SUPPORTS_NUMA_BALANCING
 	bool
 
+#
+# For architectures that know their GCC __int128 support is sound
+#
+config ARCH_SUPPORTS_INT128
+	bool
+
 # For architectures that (ab)use NUMA to represent different memory regions
 # all cpu-local but of different latencies, such as SuperH.
 #
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
@@ -5112,6 +5112,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu)
 		 * die on a /0 trap.
 		 */
 		sg->sgp->power = SCHED_POWER_SCALE * cpumask_weight(sg_span);
+		sg->sgp->power_orig = sg->sgp->power;
 
 		/*
 		 * Make sure the first group of this domain contains the
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c

Original file line number	Diff line number	Diff line change
`@@ -3,25 +3,18 @@`
`3`	`3`
`4`	`4`	`#include <linux/thread_info.h>`
`5`	`5`
`6`		`-/*`
`7`		`- * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users`
`8`		`- * that think a non-zero value indicates we cannot preempt.`
`9`		`- */`
	`6`	`+#define PREEMPT_ENABLED (0)`
	`7`	`+`
`10`	`8`	`static __always_inline int preempt_count(void)`
`11`	`9`	`{`
`12`		`- return current_thread_info()->preempt_count & ~PREEMPT_NEED_RESCHED;`
	`10`	`+ return current_thread_info()->preempt_count;`
`13`	`11`	`}`
`14`	`12`
`15`	`13`	`static __always_inline int *preempt_count_ptr(void)`
`16`	`14`	`{`
`17`	`15`	`return &current_thread_info()->preempt_count;`
`18`	`16`	`}`
`19`	`17`
`20`		`-/*`
`21`		`- * We now loose PREEMPT_NEED_RESCHED and cause an extra reschedule; however the`
`22`		`- * alternative is loosing a reschedule. Better schedule too often -- also this`
`23`		`- * should be a very rare operation.`
`24`		`- */`
`25`	`18`	`static __always_inline void preempt_count_set(int pc)`
`26`	`19`	`{`
`27`	`20`	`*preempt_count_ptr() = pc;`
`@@ -41,28 +34,17 @@ static __always_inline void preempt_count_set(int pc)`
`41`	`34`	`task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \`
`42`	`35`	`} while (0)`
`43`	`36`
`44`		`-/*`
`45`		`- * We fold the NEED_RESCHED bit into the preempt count such that`
`46`		`- * preempt_enable() can decrement and test for needing to reschedule with a`
`47`		`- * single instruction.`
`48`		`- *`
`49`		`- * We invert the actual bit, so that when the decrement hits 0 we know we both`
`50`		`- * need to resched (the bit is cleared) and can resched (no preempt count).`
`51`		`- */`
`52`		`-`
`53`	`37`	`static __always_inline void set_preempt_need_resched(void)`
`54`	`38`	`{`
`55`		`- *preempt_count_ptr() &= ~PREEMPT_NEED_RESCHED;`
`56`	`39`	`}`
`57`	`40`
`58`	`41`	`static __always_inline void clear_preempt_need_resched(void)`
`59`	`42`	`{`
`60`		`- *preempt_count_ptr() \|= PREEMPT_NEED_RESCHED;`
`61`	`43`	`}`
`62`	`44`
`63`	`45`	`static __always_inline bool test_preempt_need_resched(void)`
`64`	`46`	`{`
`65`		`- return !(*preempt_count_ptr() & PREEMPT_NEED_RESCHED);`
	`47`	`+ return false;`
`66`	`48`	`}`
`67`	`49`
`68`	`50`	`/*`
`@@ -81,15 +63,20 @@ static __always_inline void __preempt_count_sub(int val)`
`81`	`63`
`82`	`64`	`static __always_inline bool __preempt_count_dec_and_test(void)`
`83`	`65`	`{`
`84`		`- return !--*preempt_count_ptr();`
	`66`	`+ /*`
	`67`	`+ * Because of load-store architectures cannot do per-cpu atomic`
	`68`	`+ * operations; we cannot use PREEMPT_NEED_RESCHED because it might get`
	`69`	`+ * lost.`
	`70`	`+ */`
	`71`	`+ return !--*preempt_count_ptr() && tif_need_resched();`
`85`	`72`	`}`
`86`	`73`
`87`	`74`	`/*`
`88`	`75`	`* Returns true when we need to resched and can (barring IRQ state).`
`89`	`76`	`*/`
`90`	`77`	`static __always_inline bool should_resched(void)`
`91`	`78`	`{`
`92`		`- return unlikely(!*preempt_count_ptr());`
	`79`	`+ return unlikely(!preempt_count() && tif_need_resched());`
`93`	`80`	`}`
`94`	`81`
`95`	`82`	`#ifdef CONFIG_PREEMPT`