@@ -114,13 +114,13 @@ static inline void __preempt_count_sub(int val)
static inline bool __preempt_count_dec_and_test(void)
{
- return !--S390_lowcore.preempt_count && tif_need_resched();
+ return !--S390_lowcore.preempt_count && tif_need_resched(RESCHED_eager);
}
static inline bool should_resched(int preempt_offset)
{
return unlikely(preempt_count() == preempt_offset &&
- tif_need_resched());
+ tif_need_resched(RESCHED_eager));
}
#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
@@ -108,7 +108,7 @@ static const struct dmi_system_id processor_power_dmi_table[] = {
*/
static void __cpuidle acpi_safe_halt(void)
{
- if (!tif_need_resched()) {
+ if (!need_resched()) {
raw_safe_halt();
raw_local_irq_disable();
}
@@ -66,7 +66,7 @@ static __always_inline bool __preempt_count_dec_and_test(void)
* operations; we cannot use PREEMPT_NEED_RESCHED because it might get
* lost.
*/
- return !--*preempt_count_ptr() && tif_need_resched();
+ return !--*preempt_count_ptr() && tif_need_resched(RESCHED_eager);
}
/*
@@ -75,7 +75,7 @@ static __always_inline bool __preempt_count_dec_and_test(void)
static __always_inline bool should_resched(int preempt_offset)
{
return unlikely(preempt_count() == preempt_offset &&
- tif_need_resched());
+ tif_need_resched(RESCHED_eager));
}
#ifdef CONFIG_PREEMPTION
@@ -301,7 +301,7 @@ do { \
} while (0)
#define preempt_fold_need_resched() \
do { \
- if (tif_need_resched()) \
+ if (tif_need_resched(RESCHED_eager)) \
set_preempt_need_resched(); \
} while (0)
@@ -2172,9 +2172,11 @@ static inline int rwlock_needbreak(rwlock_t *lock)
static __always_inline bool need_resched(void)
{
- return unlikely(tif_need_resched());
+ return unlikely(tif_need_resched(RESCHED_eager) ||
+ tif_need_resched(RESCHED_lazy));
}
+
/*
* Wrappers for p->thread_info->cpu access. No-op on UP.
*/
@@ -63,7 +63,7 @@ static __always_inline bool __must_check current_set_polling_and_test(void)
*/
smp_mb__after_atomic();
- return unlikely(tif_need_resched());
+ return unlikely(need_resched());
}
static __always_inline bool __must_check current_clr_polling_and_test(void)
@@ -76,7 +76,7 @@ static __always_inline bool __must_check current_clr_polling_and_test(void)
*/
smp_mb__after_atomic();
- return unlikely(tif_need_resched());
+ return unlikely(need_resched());
}
#else
@@ -85,11 +85,11 @@ static inline void __current_clr_polling(void) { }
static inline bool __must_check current_set_polling_and_test(void)
{
- return unlikely(tif_need_resched());
+ return unlikely(need_resched());
}
static inline bool __must_check current_clr_polling_and_test(void)
{
- return unlikely(tif_need_resched());
+ return unlikely(need_resched());
}
#endif
@@ -200,17 +200,17 @@ static __always_inline unsigned long read_ti_thread_flags(struct thread_info *ti
#ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
-static __always_inline bool tif_need_resched(void)
+static __always_inline bool tif_need_resched(resched_t r)
{
- return arch_test_bit(TIF_NEED_RESCHED,
+ return arch_test_bit(tif_resched(r),
(unsigned long *)(¤t_thread_info()->flags));
}
#else
-static __always_inline bool tif_need_resched(void)
+static __always_inline bool tif_need_resched(resched_t r)
{
- return test_bit(TIF_NEED_RESCHED,
+ return test_bit(tif_resched(r),
(unsigned long *)(¤t_thread_info()->flags));
}
@@ -57,7 +57,7 @@ static noinline int __cpuidle cpu_idle_poll(void)
ct_cpuidle_enter();
raw_local_irq_enable();
- while (!tif_need_resched() &&
+ while (!need_resched() &&
(cpu_idle_force_poll || tick_check_broadcast_expired()))
cpu_relax();
raw_local_irq_disable();
@@ -2720,7 +2720,7 @@ unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
trace_flags |= TRACE_FLAG_BH_OFF;
- if (tif_need_resched())
+ if (tif_need_resched(RESCHED_eager))
trace_flags |= TRACE_FLAG_NEED_RESCHED;
if (test_preempt_need_resched())
trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
tif_need_resched() now takes a parameter specifying the resched type: RESCHED_lazy for when we allow the running task to run to completion before eventually scheduling at a userspace boundary and, RESCHED_eager for the next safe preemption point. need_resched(), which is used by non-core code now checks for presence of either of the need-resched bits. Also given that need_resched() (and tif_need_resched() to a lesser extent), is used extensively in the kernel so it is worth noting the common uses and how they will change: - idle: we always want to schedule out of idle whenever there is any work. So the appropriate check is for both the conditions. (Currently we use need_resched() most places and the interfaces defined in sched/idle.h use tif_need_resched().) However, as discussed in later commits it is critical that when scheduling out of idle, we always reschedule with RESCHED_eager (which maps to TIF_NEED_RESCHED.) This suggests that idle code everywhere should instead just do: while (!tif_need_resched(RESCHED_eager) { ... } or similar. That is true, but we have a lot of idle code and it does not seem to make sense to expose scheduler implementation details all over. - uses in conjunction with preempt_count(): we only ever want to fold or make preemption decisions based on TIF_NEED_RESCHED, not TIF_NEED_RESCHED_LAZY. So, related logic needs to use tif_need_resched(RESCHED_eager). - code that relinquishes resources temporarily (locks, irq, etc) checks for should_resched() and would preempt if TIF_NEED_RESCHED were set due to the (preempt_count() == offset) check. The hand-rolled versions, typically check for need_resched() which is a wider check. In either case the final arbiter is preempt_schedule() which checks via preemptible() does the more narrow check. Would it make sense to schedule out for both the need-resched flags? Originally-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com> --- arch/s390/include/asm/preempt.h | 4 ++-- drivers/acpi/processor_idle.c | 2 +- include/asm-generic/preempt.h | 4 ++-- include/linux/preempt.h | 2 +- include/linux/sched.h | 4 +++- include/linux/sched/idle.h | 8 ++++---- include/linux/thread_info.h | 8 ++++---- kernel/sched/idle.c | 2 +- kernel/trace/trace.c | 2 +- 9 files changed, 19 insertions(+), 17 deletions(-)