Btrfs: reduce lock contention on extent buffer locks

The extent buffers have a refs_lock which we use to make coordinate freeing
the extent buffer with operations on the radix tree.  On tree roots and
other extent buffers that very cache hot, this can be highly contended.

These are also the extent buffers that are basically pinned in memory.
This commit adds code to cmpxchg our way through the ref modifications,
and as long as the result of the reference change is still pinned in
ram, we skip the expensive spinlock.

Signed-off-by: Chris Mason <chris.mason@fusionio.com>

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 3b9fb478b0d1aff2189c713c7e54cfd77a58f885..907ed0025dd4076dcc0af0f6ca283967e44b85ce 100644 (file)
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -4184,6 +4184,7 @@ static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
 
 static void check_buffer_tree_ref(struct extent_buffer *eb)
 {
+       int refs;
        /* the ref bit is tricky.  We have to make sure it is set
         * if we have the buffer dirty.   Otherwise the
         * code to free a buffer can end up dropping a dirty
@@ -4204,6 +4205,10 @@ static void check_buffer_tree_ref(struct extent_buffer *eb)
         * So bump the ref count first, then set the bit.  If someone
         * beat us to it, drop the ref we added.
         */
+       refs = atomic_read(&eb->refs);
+       if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
+               return;
+
        spin_lock(&eb->refs_lock);
        if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
                atomic_inc(&eb->refs);
@@ -4405,9 +4410,20 @@ static int release_extent_buffer(struct extent_buffer *eb, gfp_t mask)
 
 void free_extent_buffer(struct extent_buffer *eb)
 {
+       int refs;
+       int old;
        if (!eb)
                return;
 
+       while (1) {
+               refs = atomic_read(&eb->refs);
+               if (refs <= 3)
+                       break;
+               old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
+               if (old == refs)
+                       return;
+       }
+
        spin_lock(&eb->refs_lock);
        if (atomic_read(&eb->refs) == 2 &&
            test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))