net: Embed hh_cache inside of struct neighbour.

Now that there is a one-to-one correspondance between neighbour
and hh_cache entries, we no longer need:

1) dynamic allocation
2) attachment to dst->hh
3) refcounting

Initialization of the hh_cache entry is indicated by hh_len
being non-zero, and such initialization is always done with
the neighbour's lock held as a writer.

Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 75382378a1ba2ec74d25dbcf08098230b6a1733e..5ccc0cb8352b1e95a5f25aa7197ee96eb29ddd6f 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -252,14 +252,7 @@ struct netdev_hw_addr_list {
        netdev_hw_addr_list_for_each(ha, &(dev)->mc)
 
 struct hh_cache {
-       atomic_t        hh_refcnt;      /* number of users                   */
-/*
- * We want hh_output, hh_len, hh_lock and hh_data be a in a separate
- * cache line on SMP.
- * They are mostly read, but hh_refcnt may be changed quite frequently,
- * incurring cache line ping pongs.
- */
-       u16             hh_len ____cacheline_aligned_in_smp;
+       u16             hh_len;
        u16             __pad;
        int             (*hh_output)(struct sk_buff *skb);
        seqlock_t       hh_lock;
@@ -273,12 +266,6 @@ struct hh_cache {
        unsigned long   hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
 };
 
-static inline void hh_cache_put(struct hh_cache *hh)
-{
-       if (atomic_dec_and_test(&hh->hh_refcnt))
-               kfree(hh);
-}
-
 /* Reserve HH_DATA_MOD byte aligned hard_header_len, but at least that much.
  * Alternative is:
  *   dev->hard_header_len ? (dev->hard_header_len +

diff --git a/include/net/dst.h b/include/net/dst.h
index e12ddfb9eb1652626b9d8cc1d0c0f62f177895d2..0dd7ccbc0dd59a27e2459963322dc73fb8ee0e30 100644 (file)
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -38,7 +38,6 @@ struct dst_entry {
        unsigned long           expires;
        struct dst_entry        *path;
        struct neighbour        *neighbour;
-       struct hh_cache         *hh;
 #ifdef CONFIG_XFRM
        struct xfrm_state       *xfrm;
 #else
@@ -47,6 +46,14 @@ struct dst_entry {
        int                     (*input)(struct sk_buff*);
        int                     (*output)(struct sk_buff*);
 
+       int                     flags;
+#define DST_HOST               0x0001
+#define DST_NOXFRM             0x0002
+#define DST_NOPOLICY           0x0004
+#define DST_NOHASH             0x0008
+#define DST_NOCACHE            0x0010
+#define DST_NOCOUNT            0x0020
+
        short                   error;
        short                   obsolete;
        unsigned short          header_len;     /* more space at head required */
@@ -62,7 +69,7 @@ struct dst_entry {
         * (L1_CACHE_SIZE would be too much)
         */
 #ifdef CONFIG_64BIT
-       long                    __pad_to_align_refcnt[1];
+       long                    __pad_to_align_refcnt[2];
 #endif
        /*
         * __refcnt wants to be on a different cache line from
@@ -71,13 +78,6 @@ struct dst_entry {
        atomic_t                __refcnt;       /* client references    */
        int                     __use;
        unsigned long           lastuse;
-       int                     flags;
-#define DST_HOST               0x0001
-#define DST_NOXFRM             0x0002
-#define DST_NOPOLICY           0x0004
-#define DST_NOHASH             0x0008
-#define DST_NOCACHE            0x0010
-#define DST_NOCOUNT            0x0020
        union {
                struct dst_entry        *next;
                struct rtable __rcu     *rt_next;

diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 6fe8c2cd5acb4602f36d4312d8c7a68af505f98d..bd8f9f09ab5c648038cb9f3f82239bbd76c81793 100644 (file)
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -108,7 +108,7 @@ struct neighbour {
        __u8                    dead;
        seqlock_t               ha_lock;
        unsigned char           ha[ALIGN(MAX_ADDR_LEN, sizeof(unsigned long))];
-       struct hh_cache         *hh;
+       struct hh_cache         hh;
        int                     (*output)(struct sk_buff *skb);
        const struct neigh_ops  *ops;
        struct rcu_head         rcu;

diff --git a/net/bridge/br_netfilter.c b/net/bridge/br_netfilter.c
index 56149ec36d7fd5d8a411b3996f5e7985a3792c77..75ee421917c783d34376ea54c4d2e9d9a422307f 100644 (file)
--- a/net/bridge/br_netfilter.c
+++ b/net/bridge/br_netfilter.c
@@ -343,14 +343,16 @@ static int br_nf_pre_routing_finish_ipv6(struct sk_buff *skb)
 static int br_nf_pre_routing_finish_bridge(struct sk_buff *skb)
 {
        struct nf_bridge_info *nf_bridge = skb->nf_bridge;
+       struct neighbour *neigh;
        struct dst_entry *dst;
 
        skb->dev = bridge_parent(skb->dev);
        if (!skb->dev)
                goto free_skb;
        dst = skb_dst(skb);
-       if (dst->hh) {
-               neigh_hh_bridge(dst->hh, skb);
+       neigh = dst->neighbour;
+       if (neigh->hh.hh_len) {
+               neigh_hh_bridge(&neigh->hh, skb);
                skb->dev = nf_bridge->physindev;
                return br_handle_frame_finish(skb);
        } else if (dst->neighbour) {

diff --git a/net/core/dst.c b/net/core/dst.c
index 6135f3671692689b773c2289d940ecabb29c6a26..4aacc14936a052960bbd56921e0a69339aad2cab 100644 (file)
--- a/net/core/dst.c
+++ b/net/core/dst.c
@@ -172,7 +172,6 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
        dst->expires = 0UL;
        dst->path = dst;
        dst->neighbour = NULL;
-       dst->hh = NULL;
 #ifdef CONFIG_XFRM
        dst->xfrm = NULL;
 #endif
@@ -226,19 +225,13 @@ struct dst_entry *dst_destroy(struct dst_entry * dst)
 {
        struct dst_entry *child;
        struct neighbour *neigh;
-       struct hh_cache *hh;
 
        smp_rmb();
 
 again:
        neigh = dst->neighbour;
-       hh = dst->hh;
        child = dst->child;
 
-       dst->hh = NULL;
-       if (hh)
-               hh_cache_put(hh);
-
        if (neigh) {
                dst->neighbour = NULL;
                neigh_release(neigh);

diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index f879bb552994b9985be31cfa5cd6bf5dabb28d68..77a399f2ad034d5c0f91078b6b10e3f7c74137db 100644 (file)
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -297,6 +297,7 @@ static struct neighbour *neigh_alloc(struct neigh_table *tbl)
        n->updated        = n->used = now;
        n->nud_state      = NUD_NONE;
        n->output         = neigh_blackhole;
+       seqlock_init(&n->hh.hh_lock);
        n->parms          = neigh_parms_clone(&tbl->parms);
        setup_timer(&n->timer, neigh_timer_handler, (unsigned long)n);
 
@@ -702,14 +703,11 @@ void neigh_destroy(struct neighbour *neigh)
        if (neigh_del_timer(neigh))
                printk(KERN_WARNING "Impossible event.\n");
 
-       hh = neigh->hh;
-       if (hh) {
-               neigh->hh = NULL;
-
+       hh = &neigh->hh;
+       if (hh->hh_len) {
                write_seqlock_bh(&hh->hh_lock);
                hh->hh_output = neigh_blackhole;
                write_sequnlock_bh(&hh->hh_lock);
-               hh_cache_put(hh);
        }
 
        skb_queue_purge(&neigh->arp_queue);
@@ -737,8 +735,8 @@ static void neigh_suspect(struct neighbour *neigh)
 
        neigh->output = neigh->ops->output;
 
-       hh = neigh->hh;
-       if (hh)
+       hh = &neigh->hh;
+       if (hh->hh_len)
                hh->hh_output = neigh->ops->output;
 }
 
@@ -755,8 +753,8 @@ static void neigh_connect(struct neighbour *neigh)
 
        neigh->output = neigh->ops->connected_output;
 
-       hh = neigh->hh;
-       if (hh)
+       hh = &neigh->hh;
+       if (hh->hh_len)
                hh->hh_output = neigh->ops->hh_output;
 }
 
@@ -1017,7 +1015,7 @@ out_unlock_bh:
 }
 EXPORT_SYMBOL(__neigh_event_send);
 
-static void neigh_update_hhs(const struct neighbour *neigh)
+static void neigh_update_hhs(struct neighbour *neigh)
 {
        struct hh_cache *hh;
        void (*update)(struct hh_cache*, const struct net_device*, const unsigned char *)
@@ -1027,8 +1025,8 @@ static void neigh_update_hhs(const struct neighbour *neigh)
                update = neigh->dev->header_ops->cache_update;
 
        if (update) {
-               hh = neigh->hh;
-               if (hh) {
+               hh = &neigh->hh;
+               if (hh->hh_len) {
                        write_seqlock_bh(&hh->hh_lock);
                        update(hh, neigh->dev, neigh->ha);
                        write_sequnlock_bh(&hh->hh_lock);
@@ -1214,62 +1212,29 @@ struct neighbour *neigh_event_ns(struct neigh_table *tbl,
 }
 EXPORT_SYMBOL(neigh_event_ns);
 
-static inline bool neigh_hh_lookup(struct neighbour *n, struct dst_entry *dst)
-{
-       struct hh_cache *hh;
-
-       smp_rmb(); /* paired with smp_wmb() in neigh_hh_init() */
-       hh = n->hh;
-       if (hh) {
-               atomic_inc(&hh->hh_refcnt);
-               if (unlikely(cmpxchg(&dst->hh, NULL, hh) != NULL))
-                       hh_cache_put(hh);
-               return true;
-       }
-       return false;
-}
-
 /* called with read_lock_bh(&n->lock); */
-static void neigh_hh_init(struct neighbour *n, struct dst_entry *dst,
-                         __be16 protocol)
+static void neigh_hh_init(struct neighbour *n, struct dst_entry *dst)
 {
-       struct hh_cache *hh;
        struct net_device *dev = dst->dev;
-
-       if (likely(neigh_hh_lookup(n, dst)))
-               return;
-
-       /* slow path */
-       hh = kzalloc(sizeof(*hh), GFP_ATOMIC);
-       if (!hh)
-               return;
-
-       seqlock_init(&hh->hh_lock);
-       atomic_set(&hh->hh_refcnt, 2);
-
-       if (dev->header_ops->cache(n, hh, protocol)) {
-               kfree(hh);
-               return;
-       }
+       __be16 prot = dst->ops->protocol;
+       struct hh_cache *hh = &n->hh;
 
        write_lock_bh(&n->lock);
 
-       /* must check if another thread already did the insert */
-       if (neigh_hh_lookup(n, dst)) {
-               kfree(hh);
+       /* Only one thread can come in here and initialize the
+        * hh_cache entry.
+        */
+       if (hh->hh_len)
+               goto end;
+
+       if (dev->header_ops->cache(n, hh, prot))
                goto end;
-       }
 
        if (n->nud_state & NUD_CONNECTED)
                hh->hh_output = n->ops->hh_output;
        else
                hh->hh_output = n->ops->output;
 
-       smp_wmb(); /* paired with smp_rmb() in neigh_hh_lookup() */
-       n->hh       = hh;
-
-       if (unlikely(cmpxchg(&dst->hh, NULL, hh) != NULL))
-               hh_cache_put(hh);
 end:
        write_unlock_bh(&n->lock);
 }
@@ -1312,10 +1277,8 @@ int neigh_resolve_output(struct sk_buff *skb)
                struct net_device *dev = neigh->dev;
                unsigned int seq;
 
-               if (dev->header_ops->cache &&
-                   !dst->hh &&
-                   !(dst->flags & DST_NOCACHE))
-                       neigh_hh_init(neigh, dst, dst->ops->protocol);
+               if (dev->header_ops->cache && !neigh->hh.hh_len)
+                       neigh_hh_init(neigh, dst);
 
                do {
                        seq = read_seqbegin(&neigh->ha_lock);

diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 54119d5aae8fad442fa22f1d8ce7c94cf19781ac..a621b96aed157c57dc92112f849b1ed910c837f9 100644 (file)
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -182,6 +182,7 @@ static inline int ip_finish_output2(struct sk_buff *skb)
        struct rtable *rt = (struct rtable *)dst;
        struct net_device *dev = dst->dev;
        unsigned int hh_len = LL_RESERVED_SPACE(dev);
+       struct neighbour *neigh;
 
        if (rt->rt_type == RTN_MULTICAST) {
                IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
@@ -203,11 +204,14 @@ static inline int ip_finish_output2(struct sk_buff *skb)
                skb = skb2;
        }
 
-       if (dst->hh)
-               return neigh_hh_output(dst->hh, skb);
-       else if (dst->neighbour)
-               return dst->neighbour->output(skb);
-
+       neigh = dst->neighbour;
+       if (neigh) {
+               struct hh_cache *hh = &neigh->hh;
+               if (hh->hh_len)
+                       return neigh_hh_output(hh, skb);
+               else
+                       return dst->neighbour->output(skb);
+       }
        if (net_ratelimit())
                printk(KERN_DEBUG "ip_finish_output2: No header cache and no neighbour!\n");
        kfree_skb(skb);

diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index c6388e825ed309d7d8d5f239bc18dc6971711ac2..a52bb74d261257c1ed60a8c527b83e05b1c989cb 100644 (file)
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -426,9 +426,10 @@ static int rt_cache_seq_show(struct seq_file *seq, void *v)
                        (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
                              dst_metric(&r->dst, RTAX_RTTVAR)),
                        r->rt_key_tos,
-                       r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
-                       r->dst.hh ? (r->dst.hh->hh_output ==
-                                      dev_queue_xmit) : 0,
+                       -1,
+                       (r->dst.neighbour ?
+                        (r->dst.neighbour->hh.hh_output ==
+                         dev_queue_xmit) : 0),
                        r->rt_spec_dst, &len);
 
                seq_printf(seq, "%*s\n", 127 - len, "");

diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 9d4b165837d6abff686c729967aec4534d435feb..f0f144cac0bdef4bcf581db89150b9a8fb0ccfae 100644 (file)
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -100,6 +100,7 @@ static int ip6_finish_output2(struct sk_buff *skb)
 {
        struct dst_entry *dst = skb_dst(skb);
        struct net_device *dev = dst->dev;
+       struct neighbour *neigh;
 
        skb->protocol = htons(ETH_P_IPV6);
        skb->dev = dev;
@@ -134,11 +135,14 @@ static int ip6_finish_output2(struct sk_buff *skb)
                                skb->len);
        }
 
-       if (dst->hh)
-               return neigh_hh_output(dst->hh, skb);
-       else if (dst->neighbour)
-               return dst->neighbour->output(skb);
-
+       neigh = dst->neighbour;
+       if (neigh) {
+               struct hh_cache *hh = &neigh->hh;
+               if (hh->hh_len)
+                       return neigh_hh_output(hh, skb);
+               else
+                       return dst->neighbour->output(skb);
+       }
        IP6_INC_STATS_BH(dev_net(dst->dev),
                         ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
        kfree_skb(skb);