rcu: Keep invoking callbacks if CPU otherwise idle

The rcu_do_batch() function that invokes callbacks for TREE_RCU and
TREE_PREEMPT_RCU normally throttles callback invocation to avoid degrading
scheduling latency.  However, as long as the CPU would otherwise be idle,
there is no downside to continuing to invoke any callbacks that have passed
through their grace periods.  In fact, processing such callbacks in a
timely manner has the benefit of increasing the probability that the
CPU can enter the power-saving dyntick-idle mode.

Therefore, this commit allows callback invocation to continue beyond the
preset limit as long as the scheduler does not have some other task to
run and as long as context is that of the idle task or the relevant
RCU kthread.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index c0ed3765ec39095421493c1668376c8c84ab4ac0..4ec4b14cfba6620ad4d69436c26fb9acf583ac6c 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1403,7 +1403,10 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
                debug_rcu_head_unqueue(list);
                __rcu_reclaim(rsp->name, list);
                list = next;
-               if (++count >= bl)
+               /* Stop only if limit reached and CPU has something to do. */
+               if (++count >= bl &&
+                   (need_resched() ||
+                    (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
                        break;
        }
 

diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 9bcfbc9d16c6456990a83869c295d264dc183434..fddff92d66760870682655f2bfccda602a3e1905 100644 (file)
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -455,6 +455,7 @@ static void __init __rcu_init_preempt(void);
 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
 static void invoke_rcu_callbacks_kthread(void);
+static bool rcu_is_callbacks_kthread(void);
 #ifdef CONFIG_RCU_BOOST
 static void rcu_preempt_do_callbacks(void);
 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,

diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index dbcea6b93aea6277a7306193c2aa36fc28e62740..adb6e666c6f456d59f888b1b8ee3cefecd19bc47 100644 (file)
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -1336,6 +1336,15 @@ static void invoke_rcu_callbacks_kthread(void)
        local_irq_restore(flags);
 }
 
+/*
+ * Is the current CPU running the RCU-callbacks kthread?
+ * Caller must have preemption disabled.
+ */
+static bool rcu_is_callbacks_kthread(void)
+{
+       return __get_cpu_var(rcu_cpu_kthread_task) == current;
+}
+
 /*
  * Set the affinity of the boost kthread.  The CPU-hotplug locks are
  * held, so no one should be messing with the existence of the boost
@@ -1780,6 +1789,11 @@ static void invoke_rcu_callbacks_kthread(void)
        WARN_ON_ONCE(1);
 }
 
+static bool rcu_is_callbacks_kthread(void)
+{
+       return false;
+}
+
 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
 {
 }