net_sched: use idr to allocate u32 filter handles

Instead of calling u32_lookup_ht() in a loop to find
a unused handle, just switch to idr API to allocate
new handles. u32 filters are special as the handle
could contain a hash table id and a key id, so we
need two IDR to allocate each of them.

Cc: Chris Mi <chrism@mellanox.com>
Cc: Jamal Hadi Salim <jhs@mojatatu.com>
Signed-off-by: Cong Wang <xiyou.wangcong@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c
index 10b8d851fc6be96c761bea127776b84d21b4fc05..094d224411a93d575cc1e164e34e33b783adae2b 100644 (file)
--- a/net/sched/cls_u32.c
+++ b/net/sched/cls_u32.c
@@ -46,6 +46,7 @@
 #include <net/act_api.h>
 #include <net/pkt_cls.h>
 #include <linux/netdevice.h>
+#include <linux/idr.h>
 
 struct tc_u_knode {
        struct tc_u_knode __rcu *next;
@@ -82,6 +83,7 @@ struct tc_u_hnode {
        struct tc_u_common      *tp_c;
        int                     refcnt;
        unsigned int            divisor;
+       struct idr              handle_idr;
        struct rcu_head         rcu;
        /* The 'ht' field MUST be the last field in structure to allow for
         * more entries allocated at end of structure.
@@ -93,7 +95,7 @@ struct tc_u_common {
        struct tc_u_hnode __rcu *hlist;
        struct Qdisc            *q;
        int                     refcnt;
-       u32                     hgenerator;
+       struct idr              handle_idr;
        struct hlist_node       hnode;
        struct rcu_head         rcu;
 };
@@ -311,19 +313,19 @@ static void *u32_get(struct tcf_proto *tp, u32 handle)
        return u32_lookup_key(ht, handle);
 }
 
-static u32 gen_new_htid(struct tc_u_common *tp_c)
+static u32 gen_new_htid(struct tc_u_common *tp_c, struct tc_u_hnode *ptr)
 {
-       int i = 0x800;
+       unsigned long idr_index;
+       int err;
 
-       /* hgenerator only used inside rtnl lock it is safe to increment
+       /* This is only used inside rtnl lock it is safe to increment
         * without read _copy_ update semantics
         */
-       do {
-               if (++tp_c->hgenerator == 0x7FF)
-                       tp_c->hgenerator = 1;
-       } while (--i > 0 && u32_lookup_ht(tp_c, (tp_c->hgenerator|0x800)<<20));
-
-       return i > 0 ? (tp_c->hgenerator|0x800)<<20 : 0;
+       err = idr_alloc_ext(&tp_c->handle_idr, ptr, &idr_index,
+                           1, 0x7FF, GFP_KERNEL);
+       if (err)
+               return 0;
+       return (u32)(idr_index | 0x800) << 20;
 }
 
 static struct hlist_head *tc_u_common_hash;
@@ -366,8 +368,9 @@ static int u32_init(struct tcf_proto *tp)
                return -ENOBUFS;
 
        root_ht->refcnt++;
-       root_ht->handle = tp_c ? gen_new_htid(tp_c) : 0x80000000;
+       root_ht->handle = tp_c ? gen_new_htid(tp_c, root_ht) : 0x80000000;
        root_ht->prio = tp->prio;
+       idr_init(&root_ht->handle_idr);
 
        if (tp_c == NULL) {
                tp_c = kzalloc(sizeof(*tp_c), GFP_KERNEL);
@@ -377,6 +380,7 @@ static int u32_init(struct tcf_proto *tp)
                }
                tp_c->q = tp->q;
                INIT_HLIST_NODE(&tp_c->hnode);
+               idr_init(&tp_c->handle_idr);
 
                h = tc_u_hash(tp);
                hlist_add_head(&tp_c->hnode, &tc_u_common_hash[h]);
@@ -565,6 +569,7 @@ static void u32_clear_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht)
                                         rtnl_dereference(n->next));
                        tcf_unbind_filter(tp, &n->res);
                        u32_remove_hw_knode(tp, n->handle);
+                       idr_remove_ext(&ht->handle_idr, n->handle);
                        call_rcu(&n->rcu, u32_delete_key_freepf_rcu);
                }
        }
@@ -586,6 +591,8 @@ static int u32_destroy_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht)
             hn = &phn->next, phn = rtnl_dereference(*hn)) {
                if (phn == ht) {
                        u32_clear_hw_hnode(tp, ht);
+                       idr_destroy(&ht->handle_idr);
+                       idr_remove_ext(&tp_c->handle_idr, ht->handle);
                        RCU_INIT_POINTER(*hn, ht->next);
                        kfree_rcu(ht, rcu);
                        return 0;
@@ -633,6 +640,7 @@ static void u32_destroy(struct tcf_proto *tp)
                        kfree_rcu(ht, rcu);
                }
 
+               idr_destroy(&tp_c->handle_idr);
                kfree(tp_c);
        }
 
@@ -701,27 +709,21 @@ ret:
        return ret;
 }
 
-#define NR_U32_NODE (1<<12)
-static u32 gen_new_kid(struct tc_u_hnode *ht, u32 handle)
+static u32 gen_new_kid(struct tc_u_hnode *ht, u32 htid)
 {
-       struct tc_u_knode *n;
-       unsigned long i;
-       unsigned long *bitmap = kzalloc(BITS_TO_LONGS(NR_U32_NODE) * sizeof(unsigned long),
-                                       GFP_KERNEL);
-       if (!bitmap)
-               return handle | 0xFFF;
-
-       for (n = rtnl_dereference(ht->ht[TC_U32_HASH(handle)]);
-            n;
-            n = rtnl_dereference(n->next))
-               set_bit(TC_U32_NODE(n->handle), bitmap);
-
-       i = find_next_zero_bit(bitmap, NR_U32_NODE, 0x800);
-       if (i >= NR_U32_NODE)
-               i = find_next_zero_bit(bitmap, NR_U32_NODE, 1);
+       unsigned long idr_index;
+       u32 start = htid | 0x800;
+       u32 max = htid | 0xFFF;
+       u32 min = htid;
+
+       if (idr_alloc_ext(&ht->handle_idr, NULL, &idr_index,
+                         start, max + 1, GFP_KERNEL)) {
+               if (idr_alloc_ext(&ht->handle_idr, NULL, &idr_index,
+                                 min + 1, max + 1, GFP_KERNEL))
+                       return max;
+       }
 
-       kfree(bitmap);
-       return handle | (i >= NR_U32_NODE ? 0xFFF : i);
+       return (u32)idr_index;
 }
 
 static const struct nla_policy u32_policy[TCA_U32_MAX + 1] = {
@@ -806,6 +808,7 @@ static void u32_replace_knode(struct tcf_proto *tp, struct tc_u_common *tp_c,
                if (pins->handle == n->handle)
                        break;
 
+       idr_replace_ext(&ht->handle_idr, n, n->handle);
        RCU_INIT_POINTER(n->next, pins->next);
        rcu_assign_pointer(*ins, n);
 }
@@ -937,22 +940,33 @@ static int u32_change(struct net *net, struct sk_buff *in_skb,
                        return -EINVAL;
                if (TC_U32_KEY(handle))
                        return -EINVAL;
-               if (handle == 0) {
-                       handle = gen_new_htid(tp->data);
-                       if (handle == 0)
-                               return -ENOMEM;
-               }
                ht = kzalloc(sizeof(*ht) + divisor*sizeof(void *), GFP_KERNEL);
                if (ht == NULL)
                        return -ENOBUFS;
+               if (handle == 0) {
+                       handle = gen_new_htid(tp->data, ht);
+                       if (handle == 0) {
+                               kfree(ht);
+                               return -ENOMEM;
+                       }
+               } else {
+                       err = idr_alloc_ext(&tp_c->handle_idr, ht, NULL,
+                                           handle, handle + 1, GFP_KERNEL);
+                       if (err) {
+                               kfree(ht);
+                               return err;
+                       }
+               }
                ht->tp_c = tp_c;
                ht->refcnt = 1;
                ht->divisor = divisor;
                ht->handle = handle;
                ht->prio = tp->prio;
+               idr_init(&ht->handle_idr);
 
                err = u32_replace_hw_hnode(tp, ht, flags);
                if (err) {
+                       idr_remove_ext(&tp_c->handle_idr, handle);
                        kfree(ht);
                        return err;
                }
@@ -986,24 +1000,33 @@ static int u32_change(struct net *net, struct sk_buff *in_skb,
                if (TC_U32_HTID(handle) && TC_U32_HTID(handle^htid))
                        return -EINVAL;
                handle = htid | TC_U32_NODE(handle);
+               err = idr_alloc_ext(&ht->handle_idr, NULL, NULL,
+                                   handle, handle + 1,
+                                   GFP_KERNEL);
+               if (err)
+                       return err;
        } else
                handle = gen_new_kid(ht, htid);
 
-       if (tb[TCA_U32_SEL] == NULL)
-               return -EINVAL;
+       if (tb[TCA_U32_SEL] == NULL) {
+               err = -EINVAL;
+               goto erridr;
+       }
 
        s = nla_data(tb[TCA_U32_SEL]);
 
        n = kzalloc(sizeof(*n) + s->nkeys*sizeof(struct tc_u32_key), GFP_KERNEL);
-       if (n == NULL)
-               return -ENOBUFS;
+       if (n == NULL) {
+               err = -ENOBUFS;
+               goto erridr;
+       }
 
 #ifdef CONFIG_CLS_U32_PERF
        size = sizeof(struct tc_u32_pcnt) + s->nkeys * sizeof(u64);
        n->pf = __alloc_percpu(size, __alignof__(struct tc_u32_pcnt));
        if (!n->pf) {
-               kfree(n);
-               return -ENOBUFS;
+               err = -ENOBUFS;
+               goto errfree;
        }
 #endif
 
@@ -1066,9 +1089,12 @@ errhw:
 errout:
        tcf_exts_destroy(&n->exts);
 #ifdef CONFIG_CLS_U32_PERF
+errfree:
        free_percpu(n->pf);
 #endif
        kfree(n);
+erridr:
+       idr_remove_ext(&ht->handle_idr, handle);
        return err;
 }