rcu: Add leaf-node macros

This commit adds rcu_first_leaf_node() that returns a pointer to
the first leaf rcu_node structure in the specified RCU flavor and an
rcu_is_leaf_node() that returns true iff the specified rcu_node structure
is a leaf.  This commit also uses these macros where appropriate.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Nicholas Piggin <npiggin@gmail.com>

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 7a693e31184a6448a58c5a7c52a878b72291aefd..5b5bb9ee2e204ac8fd09aa713e9b3cb3b445d319 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -270,6 +270,12 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt)
        }
 }
 
+/* Returns first leaf rcu_node of the specified RCU flavor. */
+#define rcu_first_leaf_node(rsp) ((rsp)->level[rcu_num_lvls - 1])
+
+/* Is this rcu_node a leaf? */
+#define rcu_is_leaf_node(rnp) ((rnp)->level == rcu_num_lvls - 1)
+
 /*
  * Do a full breadth-first scan of the rcu_node structures for the
  * specified rcu_state structure.
@@ -284,8 +290,7 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt)
  * rcu_node tree with but one rcu_node structure, this loop is a no-op.
  */
 #define rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) \
-       for ((rnp) = &(rsp)->node[0]; \
-            (rnp) < (rsp)->level[rcu_num_lvls - 1]; (rnp)++)
+       for ((rnp) = &(rsp)->node[0]; !rcu_is_leaf_node(rsp, rnp); (rnp)++)
 
 /*
  * Scan the leaves of the rcu_node hierarchy for the specified rcu_state
@@ -294,7 +299,7 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt)
  * It is still a leaf node, even if it is also the root node.
  */
 #define rcu_for_each_leaf_node(rsp, rnp) \
-       for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \
+       for ((rnp) = rcu_first_leaf_node(rsp); \
             (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)
 
 /*

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index c4db0e20b035d25fadb024bee9971150ed633b1d..b22d2e1ca5c0e7ac278569fd0c97e1e1f035b3c4 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2398,7 +2398,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp,
                        return;
                }
                WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
-               WARN_ON_ONCE(rnp->level != rcu_num_lvls - 1 &&
+               WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
                             rcu_preempt_blocked_readers_cgp(rnp));
                rnp->qsmask &= ~mask;
                trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum,
@@ -4056,7 +4056,7 @@ static void __init rcu_init_one(struct rcu_state *rsp)
 
        init_swait_queue_head(&rsp->gp_wq);
        init_swait_queue_head(&rsp->expedited_wq);
-       rnp = rsp->level[rcu_num_lvls - 1];
+       rnp = rcu_first_leaf_node(rsp);
        for_each_possible_cpu(i) {
                while (i > rnp->grphi)
                        rnp++;

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index d37b9bb3f481ec6ce523315bc2b15046f3e3a147..b999032e94668e3453414ffa0d24d1794512cb52 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -182,7 +182,7 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
 
        raw_lockdep_assert_held_rcu_node(rnp);
        WARN_ON_ONCE(rdp->mynode != rnp);
-       WARN_ON_ONCE(rnp->level != rcu_num_lvls - 1);
+       WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
 
        /*
         * Decide where to queue the newly blocked task.  In theory,
@@ -533,7 +533,7 @@ void rcu_read_unlock_special(struct task_struct *t)
                rnp = t->rcu_blocked_node;
                raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
                WARN_ON_ONCE(rnp != t->rcu_blocked_node);
-               WARN_ON_ONCE(rnp->level != rcu_num_lvls - 1);
+               WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
                empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
                empty_exp = sync_rcu_preempt_exp_done(rnp);
                smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */