From c01d0efa5136c3eaec3ae26d1e0c8c25d3a69c65 Mon Sep 17 00:00:00 2001 From: Pieter Jansen van Vuuren Date: Mon, 8 Oct 2018 18:57:34 -0700 Subject: [PATCH] nfp: flower: use rhashtable for flow caching Make use of relativistic hash tables for tracking flows instead of fixed sized hash tables. Signed-off-by: Pieter Jansen van Vuuren Reviewed-by: Jakub Kicinski Signed-off-by: David S. Miller --- drivers/net/ethernet/netronome/nfp/bpf/main.c | 5 -- .../net/ethernet/netronome/nfp/flower/main.h | 8 +- .../ethernet/netronome/nfp/flower/metadata.c | 73 ++++++++++++++++--- .../ethernet/netronome/nfp/flower/offload.c | 12 ++- drivers/net/ethernet/netronome/nfp/nfp_app.c | 5 ++ drivers/net/ethernet/netronome/nfp/nfp_app.h | 1 + 6 files changed, 82 insertions(+), 22 deletions(-) diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c index d9d37aa860e0..28af263d4577 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c @@ -509,11 +509,6 @@ err_free_bpf: return err; } -static void nfp_check_rhashtable_empty(void *ptr, void *arg) -{ - WARN_ON_ONCE(1); -} - static void nfp_bpf_clean(struct nfp_app *app) { struct nfp_app_bpf *bpf = app->priv; diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h index 81d941ab895c..0e8e2ed4bdb4 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.h +++ b/drivers/net/ethernet/netronome/nfp/flower/main.h @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -53,7 +54,6 @@ struct nfp_app; #define NFP_FL_STATS_ENTRY_RS BIT(20) #define NFP_FL_STATS_ELEM_RS 4 #define NFP_FL_REPEATED_HASH_MAX BIT(17) -#define NFP_FLOWER_HASH_BITS 19 #define NFP_FLOWER_MASK_ENTRY_RS 256 #define NFP_FLOWER_MASK_ELEMENT_RS 1 #define NFP_FLOWER_MASK_HASH_BITS 10 @@ -171,7 +171,7 @@ struct nfp_flower_priv { struct nfp_fl_stats_id stats_ids; struct nfp_fl_mask_id mask_ids; DECLARE_HASHTABLE(mask_table, NFP_FLOWER_MASK_HASH_BITS); - DECLARE_HASHTABLE(flow_table, NFP_FLOWER_HASH_BITS); + struct rhashtable flow_table; struct work_struct cmsg_work; struct sk_buff_head cmsg_skbs_high; struct sk_buff_head cmsg_skbs_low; @@ -227,7 +227,7 @@ struct nfp_fl_stats { struct nfp_fl_payload { struct nfp_fl_rule_metadata meta; unsigned long tc_flower_cookie; - struct hlist_node link; + struct rhash_head fl_node; struct rcu_head rcu; spinlock_t lock; /* lock stats */ struct nfp_fl_stats stats; @@ -239,6 +239,8 @@ struct nfp_fl_payload { bool ingress_offload; }; +extern const struct rhashtable_params nfp_flower_table_params; + struct nfp_fl_stats_frame { __be32 stats_con_id; __be32 pkt_count; diff --git a/drivers/net/ethernet/netronome/nfp/flower/metadata.c b/drivers/net/ethernet/netronome/nfp/flower/metadata.c index c098730544b7..2427c994c91d 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/metadata.c +++ b/drivers/net/ethernet/netronome/nfp/flower/metadata.c @@ -48,6 +48,12 @@ struct nfp_mask_id_table { u8 mask_id; }; +struct nfp_fl_flow_table_cmp_arg { + struct net_device *netdev; + unsigned long cookie; + __be32 host_ctx; +}; + static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) { struct nfp_flower_priv *priv = app->priv; @@ -102,18 +108,15 @@ struct nfp_fl_payload * nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, struct net_device *netdev, __be32 host_ctx) { + struct nfp_fl_flow_table_cmp_arg flower_cmp_arg; struct nfp_flower_priv *priv = app->priv; - struct nfp_fl_payload *flower_entry; - hash_for_each_possible_rcu(priv->flow_table, flower_entry, link, - tc_flower_cookie) - if (flower_entry->tc_flower_cookie == tc_flower_cookie && - (!netdev || flower_entry->ingress_dev == netdev) && - (host_ctx == NFP_FL_STATS_CTX_DONT_CARE || - flower_entry->meta.host_ctx_id == host_ctx)) - return flower_entry; + flower_cmp_arg.netdev = netdev; + flower_cmp_arg.cookie = tc_flower_cookie; + flower_cmp_arg.host_ctx = host_ctx; - return NULL; + return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg, + nfp_flower_table_params); } static void @@ -389,12 +392,56 @@ int nfp_modify_flow_metadata(struct nfp_app *app, return nfp_release_stats_entry(app, temp_ctx_id); } +static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, + const void *obj) +{ + const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key; + const struct nfp_fl_payload *flow_entry = obj; + + if ((!cmp_arg->netdev || flow_entry->ingress_dev == cmp_arg->netdev) && + (cmp_arg->host_ctx == NFP_FL_STATS_CTX_DONT_CARE || + flow_entry->meta.host_ctx_id == cmp_arg->host_ctx)) + return flow_entry->tc_flower_cookie != cmp_arg->cookie; + + return 1; +} + +static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed) +{ + const struct nfp_fl_payload *flower_entry = data; + + return jhash2((u32 *)&flower_entry->tc_flower_cookie, + sizeof(flower_entry->tc_flower_cookie) / sizeof(u32), + seed); +} + +static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed) +{ + const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data; + + return jhash2((u32 *)&cmp_arg->cookie, + sizeof(cmp_arg->cookie) / sizeof(u32), seed); +} + +const struct rhashtable_params nfp_flower_table_params = { + .head_offset = offsetof(struct nfp_fl_payload, fl_node), + .hashfn = nfp_fl_key_hashfn, + .obj_cmpfn = nfp_fl_obj_cmpfn, + .obj_hashfn = nfp_fl_obj_hashfn, + .automatic_shrinking = true, +}; + int nfp_flower_metadata_init(struct nfp_app *app) { struct nfp_flower_priv *priv = app->priv; + int err; hash_init(priv->mask_table); - hash_init(priv->flow_table); + + err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params); + if (err) + return err; + get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); /* Init ring buffer and unallocated mask_ids. */ @@ -402,7 +449,7 @@ int nfp_flower_metadata_init(struct nfp_app *app) kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); if (!priv->mask_ids.mask_id_free_list.buf) - return -ENOMEM; + goto err_free_flow_table; priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; @@ -428,6 +475,8 @@ err_free_last_used: kfree(priv->mask_ids.last_used); err_free_mask_id: kfree(priv->mask_ids.mask_id_free_list.buf); +err_free_flow_table: + rhashtable_destroy(&priv->flow_table); return -ENOMEM; } @@ -438,6 +487,8 @@ void nfp_flower_metadata_cleanup(struct nfp_app *app) if (!priv) return; + rhashtable_free_and_destroy(&priv->flow_table, + nfp_check_rhashtable_empty, NULL); kfree(priv->mask_ids.mask_id_free_list.buf); kfree(priv->mask_ids.last_used); vfree(priv->stats_ids.free_list.buf); diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c index bd19624f10cf..cd9f13695bc7 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/offload.c +++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c @@ -513,9 +513,12 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, if (err) goto err_destroy_flow; - INIT_HLIST_NODE(&flow_pay->link); flow_pay->tc_flower_cookie = flow->cookie; - hash_add_rcu(priv->flow_table, &flow_pay->link, flow->cookie); + err = rhashtable_insert_fast(&priv->flow_table, &flow_pay->fl_node, + nfp_flower_table_params); + if (err) + goto err_destroy_flow; + port->tc_offload_cnt++; /* Deallocate flow payload when flower rule has been destroyed. */ @@ -550,6 +553,7 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, struct tc_cls_flower_offload *flow, bool egress) { struct nfp_port *port = nfp_port_from_netdev(netdev); + struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *nfp_flow; struct net_device *ingr_dev; int err; @@ -573,11 +577,13 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, goto err_free_flow; err_free_flow: - hash_del_rcu(&nfp_flow->link); port->tc_offload_cnt--; kfree(nfp_flow->action_data); kfree(nfp_flow->mask_data); kfree(nfp_flow->unmasked_data); + WARN_ON_ONCE(rhashtable_remove_fast(&priv->flow_table, + &nfp_flow->fl_node, + nfp_flower_table_params)); kfree_rcu(nfp_flow, rcu); return err; } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.c b/drivers/net/ethernet/netronome/nfp/nfp_app.c index 8607d09ab732..01116822ddf7 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_app.c @@ -60,6 +60,11 @@ static const struct nfp_app_type *apps[] = { #endif }; +void nfp_check_rhashtable_empty(void *ptr, void *arg) +{ + WARN_ON_ONCE(1); +} + struct nfp_app *nfp_app_from_netdev(struct net_device *netdev) { if (nfp_netdev_is_nfp_net(netdev)) { diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h b/drivers/net/ethernet/netronome/nfp/nfp_app.h index c896eb8f87a1..19f82f24c6ad 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h @@ -196,6 +196,7 @@ struct nfp_app { void *priv; }; +void nfp_check_rhashtable_empty(void *ptr, void *arg); bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb); bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb); -- 2.30.2