rcu: Augment rcu_batch_end tracing for idle and callback state

The current rcu_batch_end event trace records only the name of the RCU
flavor and the total number of callbacks that remain queued on the
current CPU.  This is insufficient for testing and tuning the new
dyntick-idle RCU_FAST_NO_HZ code, so this commit adds idle state along
with whether or not any of the callbacks that were ready to invoke
at the beginning of rcu_do_batch() are still queued.

Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index c75418c3ccb81070ba095910f8a55421c29da1c2..d2d88bed891b1481ad740cd5f4c1a1bfd4074f01 100644 (file)
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -461,27 +461,46 @@ TRACE_EVENT(rcu_invoke_kfree_callback,
 
 /*
  * Tracepoint for exiting rcu_do_batch after RCU callbacks have been
- * invoked.  The first argument is the name of the RCU flavor and
- * the second argument is number of callbacks actually invoked.
+ * invoked.  The first argument is the name of the RCU flavor,
+ * the second argument is number of callbacks actually invoked,
+ * the third argument (cb) is whether or not any of the callbacks that
+ * were ready to invoke at the beginning of this batch are still
+ * queued, the fourth argument (nr) is the return value of need_resched(),
+ * the fifth argument (iit) is 1 if the current task is the idle task,
+ * and the sixth argument (risk) is the return value from
+ * rcu_is_callbacks_kthread().
  */
 TRACE_EVENT(rcu_batch_end,
 
-       TP_PROTO(char *rcuname, int callbacks_invoked),
+       TP_PROTO(char *rcuname, int callbacks_invoked,
+                bool cb, bool nr, bool iit, bool risk),
 
-       TP_ARGS(rcuname, callbacks_invoked),
+       TP_ARGS(rcuname, callbacks_invoked, cb, nr, iit, risk),
 
        TP_STRUCT__entry(
                __field(char *, rcuname)
                __field(int, callbacks_invoked)
+               __field(bool, cb)
+               __field(bool, nr)
+               __field(bool, iit)
+               __field(bool, risk)
        ),
 
        TP_fast_assign(
                __entry->rcuname = rcuname;
                __entry->callbacks_invoked = callbacks_invoked;
-       ),
-
-       TP_printk("%s CBs-invoked=%d",
-                 __entry->rcuname, __entry->callbacks_invoked)
+               __entry->cb = cb;
+               __entry->nr = nr;
+               __entry->iit = iit;
+               __entry->risk = risk;
+       ),
+
+       TP_printk("%s CBs-invoked=%d idle=%c%c%c%c",
+                 __entry->rcuname, __entry->callbacks_invoked,
+                 __entry->cb ? 'C' : '.',
+                 __entry->nr ? 'S' : '.',
+                 __entry->iit ? 'I' : '.',
+                 __entry->risk ? 'R' : '.')
 );
 
 /*
@@ -524,7 +543,8 @@ TRACE_EVENT(rcu_torture_read,
 #define trace_rcu_batch_start(rcuname, qlen, blimit) do { } while (0)
 #define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0)
 #define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0)
-#define trace_rcu_batch_end(rcuname, callbacks_invoked) do { } while (0)
+#define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \
+       do { } while (0)
 #define trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
 
 #endif /* #else #ifdef CONFIG_RCU_TRACE */

diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c
index e5bd94954fa3f78c8d85f38128b64d791642994f..977296dca0a41100b1d549a5ee2fc8a3bd9e22c3 100644 (file)
--- a/kernel/rcutiny.c
+++ b/kernel/rcutiny.c
@@ -259,7 +259,11 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
        /* If no RCU callbacks ready to invoke, just return. */
        if (&rcp->rcucblist == rcp->donetail) {
                RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, -1));
-               RCU_TRACE(trace_rcu_batch_end(rcp->name, 0));
+               RCU_TRACE(trace_rcu_batch_end(rcp->name, 0,
+                                             ACCESS_ONCE(rcp->rcucblist),
+                                             need_resched(),
+                                             is_idle_task(current),
+                                             rcu_is_callbacks_kthread()));
                return;
        }
 
@@ -288,7 +292,9 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
                RCU_TRACE(cb_count++);
        }
        RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count));
-       RCU_TRACE(trace_rcu_batch_end(rcp->name, cb_count));
+       RCU_TRACE(trace_rcu_batch_end(rcp->name, cb_count, 0, need_resched(),
+                                     is_idle_task(current),
+                                     rcu_is_callbacks_kthread()));
 }
 
 static void rcu_process_callbacks(struct softirq_action *unused)

diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 2b0484a5dc285bcb68773843a934cd0cc05212c8..dfa97cbb39106ef7e16753e56e3ebe551c034a57 100644 (file)
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -885,6 +885,19 @@ static void invoke_rcu_callbacks(void)
        wake_up(&rcu_kthread_wq);
 }
 
+#ifdef CONFIG_RCU_TRACE
+
+/*
+ * Is the current CPU running the RCU-callbacks kthread?
+ * Caller must have preemption disabled.
+ */
+static bool rcu_is_callbacks_kthread(void)
+{
+       return rcu_kthread_task == current;
+}
+
+#endif /* #ifdef CONFIG_RCU_TRACE */
+
 /*
  * This kthread invokes RCU callbacks whose grace periods have
  * elapsed.  It is awakened as needed, and takes the place of the
@@ -938,6 +951,18 @@ void invoke_rcu_callbacks(void)
        raise_softirq(RCU_SOFTIRQ);
 }
 
+#ifdef CONFIG_RCU_TRACE
+
+/*
+ * There is no callback kthread, so this thread is never it.
+ */
+static bool rcu_is_callbacks_kthread(void)
+{
+       return false;
+}
+
+#endif /* #ifdef CONFIG_RCU_TRACE */
+
 void rcu_init(void)
 {
        open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 2b2e1a996a65ceea8d9b38f152cf783b9c271a1e..6c4a6722abfd2fc7297da3e93e6789b94edb6de0 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -1373,7 +1373,9 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
        /* If no callbacks are ready, just return.*/
        if (!cpu_has_callbacks_ready_to_invoke(rdp)) {
                trace_rcu_batch_start(rsp->name, 0, 0);
-               trace_rcu_batch_end(rsp->name, 0);
+               trace_rcu_batch_end(rsp->name, 0, !!ACCESS_ONCE(rdp->nxtlist),
+                                   need_resched(), is_idle_task(current),
+                                   rcu_is_callbacks_kthread());
                return;
        }
 
@@ -1409,7 +1411,9 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp)
        }
 
        local_irq_save(flags);
-       trace_rcu_batch_end(rsp->name, count);
+       trace_rcu_batch_end(rsp->name, count, !!list, need_resched(),
+                           is_idle_task(current),
+                           rcu_is_callbacks_kthread());
 
        /* Update count, and requeue any remaining callbacks. */
        rdp->qlen -= count;