net/mlx5e: Protect neigh hash encap list with spinlock and rcu

Rcu-ify mlx5e_neigh_hash_entry->encap_list by changing operations on encap
list to their rcu counterparts and extending encap structure with rcu_head
to free the encap instances after rcu grace period. Use rcu read lock when
traversing encap list. Implement helper mlx5e_get_next_valid_encap()
function that is used by mlx5e_tc_update_neigh_used_value() to safely
iterate over valid entries of nhe->encap_list.

Signed-off-by: Vlad Buslov <vladbu@mellanox.com>
Reviewed-by: Jianbo Liu <jianbol@mellanox.com>
Reviewed-by: Roi Dayan <roid@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 218772d5c062f4e61e016248a630c0eb2e62c124..f26edf4581529810d5bca36889c0ed891eec2e06 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -1064,6 +1064,7 @@ static int mlx5e_rep_neigh_entry_create(struct mlx5e_priv *priv,
        (*nhe)->priv = priv;
        memcpy(&(*nhe)->m_neigh, &e->m_neigh, sizeof(e->m_neigh));
        INIT_WORK(&(*nhe)->neigh_update_work, mlx5e_rep_neigh_update);
+       spin_lock_init(&(*nhe)->encap_list_lock);
        INIT_LIST_HEAD(&(*nhe)->encap_list);
        refcount_set(&(*nhe)->refcnt, 1);
 
@@ -1103,7 +1104,10 @@ int mlx5e_rep_encap_entry_attach(struct mlx5e_priv *priv,
        }
 
        e->nhe = nhe;
-       list_add(&e->encap_list, &nhe->encap_list);
+       spin_lock(&nhe->encap_list_lock);
+       list_add_rcu(&e->encap_list, &nhe->encap_list);
+       spin_unlock(&nhe->encap_list_lock);
+
        mutex_unlock(&rpriv->neigh_update.encap_lock);
 
        return 0;
@@ -1119,7 +1123,9 @@ void mlx5e_rep_encap_entry_detach(struct mlx5e_priv *priv,
        if (!e->nhe)
                return;
 
-       list_del(&e->encap_list);
+       spin_lock(&e->nhe->encap_list_lock);
+       list_del_rcu(&e->encap_list);
+       spin_unlock(&e->nhe->encap_list_lock);
 
        mlx5e_rep_neigh_entry_release(e->nhe);
        e->nhe = NULL;

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index 8fa27832bd81ea8ed60e0a25c908b65f8b5304ec..a0ae5069d8c3bf7063528572b2951e9e6935e93a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -119,6 +119,8 @@ struct mlx5e_neigh_hash_entry {
         */
        struct list_head neigh_list;
 
+       /* protects encap list */
+       spinlock_t encap_list_lock;
        /* encap list sharing the same neigh */
        struct list_head encap_list;
 
@@ -173,6 +175,7 @@ struct mlx5e_encap_entry {
        refcount_t refcnt;
        struct completion res_ready;
        int compl_result;
+       struct rcu_head rcu;
 };
 
 struct mlx5e_rep_sq {

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index 3917834b48ff2350bc2c6ca806d9a894663e1f2b..a4d11274be30523987b1687da365e32e20532d40 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -1412,11 +1412,56 @@ static struct mlx5_fc *mlx5e_tc_get_counter(struct mlx5e_tc_flow *flow)
                return flow->nic_attr->counter;
 }
 
+static struct mlx5e_encap_entry *
+mlx5e_get_next_valid_encap(struct mlx5e_neigh_hash_entry *nhe,
+                          struct mlx5e_encap_entry *e)
+{
+       struct mlx5e_encap_entry *next = NULL;
+
+retry:
+       rcu_read_lock();
+
+       /* find encap with non-zero reference counter value */
+       for (next = e ?
+                    list_next_or_null_rcu(&nhe->encap_list,
+                                          &e->encap_list,
+                                          struct mlx5e_encap_entry,
+                                          encap_list) :
+                    list_first_or_null_rcu(&nhe->encap_list,
+                                           struct mlx5e_encap_entry,
+                                           encap_list);
+            next;
+            next = list_next_or_null_rcu(&nhe->encap_list,
+                                         &next->encap_list,
+                                         struct mlx5e_encap_entry,
+                                         encap_list))
+               if (mlx5e_encap_take(next))
+                       break;
+
+       rcu_read_unlock();
+
+       /* release starting encap */
+       if (e)
+               mlx5e_encap_put(netdev_priv(e->out_dev), e);
+       if (!next)
+               return next;
+
+       /* wait for encap to be fully initialized */
+       wait_for_completion(&next->res_ready);
+       /* continue searching if encap entry is not in valid state after completion */
+       if (!(next->flags & MLX5_ENCAP_ENTRY_VALID)) {
+               e = next;
+               goto retry;
+       }
+
+       return next;
+}
+
 void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
 {
        struct mlx5e_neigh *m_neigh = &nhe->m_neigh;
+       struct mlx5e_encap_entry *e = NULL;
        struct mlx5e_tc_flow *flow;
-       struct mlx5e_encap_entry *e;
        struct mlx5_fc *counter;
        struct neigh_table *tbl;
        bool neigh_used = false;
@@ -1432,13 +1477,12 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
        else
                return;
 
-       list_for_each_entry(e, &nhe->encap_list, encap_list) {
+       /* mlx5e_get_next_valid_encap() releases previous encap before returning
+        * next one.
+        */
+       while ((e = mlx5e_get_next_valid_encap(nhe, e)) != NULL) {
                struct encap_flow_item *efi, *tmp;
 
-               if (!(e->flags & MLX5_ENCAP_ENTRY_VALID) ||
-                   !mlx5e_encap_take(e))
-                       continue;
-
                list_for_each_entry_safe(efi, tmp, &e->flows, list) {
                        flow = container_of(efi, struct mlx5e_tc_flow,
                                            encaps[efi->index]);
@@ -1458,9 +1502,11 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
                        mlx5e_flow_put(netdev_priv(e->out_dev), flow);
                }
 
-               mlx5e_encap_put(netdev_priv(e->out_dev), e);
-               if (neigh_used)
+               if (neigh_used) {
+                       /* release current encap before breaking the loop */
+                       mlx5e_encap_put(netdev_priv(e->out_dev), e);
                        break;
+               }
        }
 
        if (neigh_used) {
@@ -1490,7 +1536,7 @@ static void mlx5e_encap_dealloc(struct mlx5e_priv *priv, struct mlx5e_encap_entr
        }
 
        kfree(e->encap_header);
-       kfree(e);
+       kfree_rcu(e, rcu);
 }
 
 void mlx5e_encap_put(struct mlx5e_priv *priv, struct mlx5e_encap_entry *e)