nohz: Remove tick_nohz_idle_enter_norcu() / tick_nohz_idle_exit_norcu()

Those two APIs were provided to optimize the calls of
tick_nohz_idle_enter() and rcu_idle_enter() into a single
irq disabled section. This way no interrupt happening in-between would
needlessly process any RCU job.

Now we are talking about an optimization for which benefits
have yet to be measured. Let's start simple and completely decouple
idle rcu and dyntick idle logics to simplify.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 47e34c091276f4ee4c272d8bc696f2a66931a191..e8e8fe505df140f11be486792c10801d971f343b 100644 (file)
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -183,7 +183,8 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                leds_event(led_idle_start);
                while (!need_resched()) {
 #ifdef CONFIG_HOTPLUG_CPU
@@ -213,7 +214,8 @@ void cpu_idle(void)
                        }
                }
                leds_event(led_idle_end);
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c
index 34c8c703bb161e3585d3bdc581413549d8dfbc3e..ea33957503243c44ce8eaed1b7d114fec608f792 100644 (file)
--- a/arch/avr32/kernel/process.c
+++ b/arch/avr32/kernel/process.c
@@ -34,10 +34,12 @@ void cpu_idle(void)
 {
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched())
                        cpu_idle_sleep();
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c
index 57e07498a0e76db1045028b801ccd718ed860e0b..8dd0416673cb56f631e2b6f45db7af749982730d 100644 (file)
--- a/arch/blackfin/kernel/process.c
+++ b/arch/blackfin/kernel/process.c
@@ -88,10 +88,12 @@ void cpu_idle(void)
 #endif
                if (!idle)
                        idle = default_idle;
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched())
                        idle();
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c
index 13d59f34b94e151ad5af802b3c88d352384f7b2a..7dcb5bfffb75527a6875e7c41ed701d973d44c4c 100644 (file)
--- a/arch/microblaze/kernel/process.c
+++ b/arch/microblaze/kernel/process.c
@@ -103,10 +103,12 @@ void cpu_idle(void)
                if (!idle)
                        idle = default_idle;
 
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched())
                        idle();
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
 
                preempt_enable_no_resched();
                schedule();

diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c
index 17fb3a270160d3cd2b8903aac8a80bbcca6ce7de..7955409051c479e1075fa424dd02f993731fbd53 100644 (file)
--- a/arch/mips/kernel/process.c
+++ b/arch/mips/kernel/process.c
@@ -56,7 +56,8 @@ void __noreturn cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched() && cpu_online(cpu)) {
 #ifdef CONFIG_MIPS_MT_SMTC
                        extern void smtc_idle_loop_hook(void);
@@ -77,7 +78,8 @@ void __noreturn cpu_idle(void)
                     system_state == SYSTEM_BOOTING))
                        play_dead();
 #endif
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/openrisc/kernel/idle.c b/arch/openrisc/kernel/idle.c
index 2e82cd0fa5e1939840f3b7d1508b68a761247863..e5fc788778306965c13615f90a0bccff84626402 100644 (file)
--- a/arch/openrisc/kernel/idle.c
+++ b/arch/openrisc/kernel/idle.c
@@ -51,7 +51,8 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
 
                while (!need_resched()) {
                        check_pgt_cache();
@@ -69,7 +70,8 @@ void cpu_idle(void)
                        set_thread_flag(TIF_POLLING_NRFLAG);
                }
 
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c
index 3cd73d1fc427b3a618e26fa301e348dbcffcc064..9c3cd490b1bd978bbde68327f103b73bfd1d0e6a 100644 (file)
--- a/arch/powerpc/kernel/idle.c
+++ b/arch/powerpc/kernel/idle.c
@@ -62,10 +62,10 @@ void cpu_idle(void)
 
        set_thread_flag(TIF_POLLING_NRFLAG);
        while (1) {
-               if (idle_uses_rcu)
-                       tick_nohz_idle_enter();
-               else
-                       tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               if (!idle_uses_rcu)
+                       rcu_idle_enter();
+
                while (!need_resched() && !cpu_should_die()) {
                        ppc64_runlatch_off();
 
@@ -102,10 +102,9 @@ void cpu_idle(void)
 
                HMT_medium();
                ppc64_runlatch_on();
-               if (idle_uses_rcu)
-                       tick_nohz_idle_exit();
-               else
-                       tick_nohz_idle_exit_norcu();
+               if (!idle_uses_rcu)
+                       rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                if (cpu_should_die())
                        cpu_die();

diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c
index d69d3d185e89289be10968fcb6517f47bcedbe5c..8fc62586a973a8017179b90df32203aa36dea60f 100644 (file)
--- a/arch/powerpc/platforms/iseries/setup.c
+++ b/arch/powerpc/platforms/iseries/setup.c
@@ -563,7 +563,8 @@ static void yield_shared_processor(void)
 static void iseries_shared_idle(void)
 {
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched() && !hvlpevent_is_pending()) {
                        local_irq_disable();
                        ppc64_runlatch_off();
@@ -577,7 +578,8 @@ static void iseries_shared_idle(void)
                }
 
                ppc64_runlatch_on();
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
 
                if (hvlpevent_is_pending())
                        process_iSeries_events();
@@ -593,7 +595,8 @@ static void iseries_dedicated_idle(void)
        set_thread_flag(TIF_POLLING_NRFLAG);
 
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                if (!need_resched()) {
                        while (!need_resched()) {
                                ppc64_runlatch_off();
@@ -610,7 +613,8 @@ static void iseries_dedicated_idle(void)
                }
 
                ppc64_runlatch_on();
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 6fa987367ae670ec14634f725751a7ef5cb1b4c9..3201ae447990c50052d7536d079e39286a77120d 100644 (file)
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -91,10 +91,12 @@ static void default_idle(void)
 void cpu_idle(void)
 {
        for (;;) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched())
                        default_idle();
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c
index ad58e7535a7c479ba3236584f0c65bcacd019219..406508d4ce742d1cffb375dfdd1bee911b6adf76 100644 (file)
--- a/arch/sh/kernel/idle.c
+++ b/arch/sh/kernel/idle.c
@@ -89,7 +89,8 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
 
                while (!need_resched()) {
                        check_pgt_cache();
@@ -111,7 +112,8 @@ void cpu_idle(void)
                        start_critical_timings();
                }
 
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c
index 4a0e7d79cb926ee2a302ce78a2cfca247afbc0f4..39d8b05201a23bd83eb6866d05477f3b34664957 100644 (file)
--- a/arch/sparc/kernel/process_64.c
+++ b/arch/sparc/kernel/process_64.c
@@ -95,12 +95,14 @@ void cpu_idle(void)
        set_thread_flag(TIF_POLLING_NRFLAG);
 
        while(1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
 
                while (!need_resched() && !cpu_is_offline(cpu))
                        sparc64_yield(cpu);
 
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
 
                preempt_enable_no_resched();
 

diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c
index 53ac89595ab153b1172dcdd3582677bf90c661c1..4c1ac6e5347ac687bdbb4de54be8a736485d9416 100644 (file)
--- a/arch/tile/kernel/process.c
+++ b/arch/tile/kernel/process.c
@@ -85,7 +85,8 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched()) {
                        if (cpu_is_offline(cpu))
                                BUG();  /* no HOTPLUG_CPU */
@@ -105,7 +106,8 @@ void cpu_idle(void)
                                local_irq_enable();
                        current_thread_info()->status |= TS_POLLING;
                }
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c
index 55d2cf455f630b7f70bfd422bce51e707e7d87ab..69f24905abdc4350e368a84e94e9ed5dc9da3d3c 100644 (file)
--- a/arch/um/kernel/process.c
+++ b/arch/um/kernel/process.c
@@ -246,10 +246,12 @@ void default_idle(void)
                if (need_resched())
                        schedule();
 
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                nsecs = disable_timer();
                idle_sleep(nsecs);
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
        }
 }
 

diff --git a/arch/unicore32/kernel/process.c b/arch/unicore32/kernel/process.c
index 095ff5a57928fb0f7e7e0b6c64bc8d73f6e41d24..52edc2b628732552c1e8cc6578acb5fe2a8a0427 100644 (file)
--- a/arch/unicore32/kernel/process.c
+++ b/arch/unicore32/kernel/process.c
@@ -55,7 +55,8 @@ void cpu_idle(void)
 {
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched()) {
                        local_irq_disable();
                        stop_critical_timings();
@@ -63,7 +64,8 @@ void cpu_idle(void)
                        local_irq_enable();
                        start_critical_timings();
                }
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index f94da3920c36bc1501b395b24ef3cac4ace20d78..485204f58cdaaf7e8306198375a7dce17450b025 100644 (file)
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -99,7 +99,8 @@ void cpu_idle(void)
 
        /* endless idle loop with no priority at all */
        while (1) {
-               tick_nohz_idle_enter_norcu();
+               tick_nohz_idle_enter();
+               rcu_idle_enter();
                while (!need_resched()) {
 
                        check_pgt_cache();
@@ -116,7 +117,8 @@ void cpu_idle(void)
                                pm_idle();
                        start_critical_timings();
                }
-               tick_nohz_idle_exit_norcu();
+               rcu_idle_exit();
+               tick_nohz_idle_exit();
                preempt_enable_no_resched();
                schedule();
                preempt_disable();

diff --git a/include/linux/tick.h b/include/linux/tick.h
index 327434a057572548f089443c8108422419923402..ab8be90b5cc957c9fad62cab8c86796a0645d77a 100644 (file)
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -122,45 +122,8 @@ static inline int tick_oneshot_mode_active(void) { return 0; }
 #endif /* !CONFIG_GENERIC_CLOCKEVENTS */
 
 # ifdef CONFIG_NO_HZ
-extern void __tick_nohz_idle_enter(void);
-static inline void tick_nohz_idle_enter(void)
-{
-       local_irq_disable();
-       __tick_nohz_idle_enter();
-       local_irq_enable();
-}
+extern void tick_nohz_idle_enter(void);
 extern void tick_nohz_idle_exit(void);
-
-/*
- * Call this pair of function if the arch doesn't make any use
- * of RCU in-between. You won't need to call rcu_idle_enter() and
- * rcu_idle_exit().
- * Otherwise you need to call tick_nohz_idle_enter() and tick_nohz_idle_exit()
- * and explicitly tell RCU about the window around the place the CPU enters low
- * power mode where no RCU use is made. This is done by calling rcu_idle_enter()
- * after the last use of RCU before the CPU is put to sleep and by calling
- * rcu_idle_exit() before the first use of RCU after the CPU woke up.
- */
-static inline void tick_nohz_idle_enter_norcu(void)
-{
-       /*
-        * Also call rcu_idle_enter() in the irq disabled section even
-        * if it disables irq itself.
-        * Just an optimization that prevents from an interrupt happening
-        * between it and __tick_nohz_idle_enter() to lose time to help
-        * completing a grace period while we could be in extended grace
-        * period already.
-        */
-       local_irq_disable();
-       __tick_nohz_idle_enter();
-       rcu_idle_enter();
-       local_irq_enable();
-}
-static inline void tick_nohz_idle_exit_norcu(void)
-{
-       rcu_idle_exit();
-       tick_nohz_idle_exit();
-}
 extern void tick_nohz_irq_exit(void);
 extern ktime_t tick_nohz_get_sleep_length(void);
 extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time);
@@ -168,14 +131,6 @@ extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
 # else
 static inline void tick_nohz_idle_enter(void) { }
 static inline void tick_nohz_idle_exit(void) { }
-static inline void tick_nohz_idle_enter_norcu(void)
-{
-       rcu_idle_enter();
-}
-static inline void tick_nohz_idle_exit_norcu(void)
-{
-       rcu_idle_exit();
-}
 
 static inline ktime_t tick_nohz_get_sleep_length(void)
 {

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index c76aefe764b0f19e502da8feaf20bbe7acad726f..0ec8b832ab6bfe88819a48be3bab229e1a4e4ad5 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -454,21 +454,20 @@ out:
  * When the next event is more than a tick into the future, stop the idle tick
  * Called when we start the idle loop.
  *
- * If no use of RCU is made in the idle loop between
- * tick_nohz_idle_enter() and tick_nohz_idle_exit() calls, then
- * tick_nohz_idle_enter_norcu() should be called instead and the arch
- * doesn't need to call rcu_idle_enter() and rcu_idle_exit() explicitly.
- *
- * Otherwise the arch is responsible of calling:
+ * The arch is responsible of calling:
  *
  * - rcu_idle_enter() after its last use of RCU before the CPU is put
  *  to sleep.
  * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
  */
-void __tick_nohz_idle_enter(void)
+void tick_nohz_idle_enter(void)
 {
        struct tick_sched *ts;
 
+       WARN_ON_ONCE(irqs_disabled());
+
+       local_irq_disable();
+
        ts = &__get_cpu_var(tick_cpu_sched);
        /*
         * set ts->inidle unconditionally. even if the system did not
@@ -477,6 +476,8 @@ void __tick_nohz_idle_enter(void)
         */
        ts->inidle = 1;
        tick_nohz_stop_sched_tick(ts);
+
+       local_irq_enable();
 }
 
 /**