sbitmap: fix sbitmap_for_each_set()

We need to ignore bits in the cleared mask when iterating over all set
bits.

Fixes: ea86ea2cdced ("sbitmap: ammortize cost of clearing bits")
Reported-by: Jens Axboe@kernel.dk>
Signed-off-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h
index 92806a2dbab730c517adb9b3d141a80166956023..03f50fcedc79b8965b27e3b0da3f9d4c03bfa6a8 100644 (file)
--- a/include/linux/sbitmap.h
+++ b/include/linux/sbitmap.h
@@ -265,12 +265,14 @@ static inline void __sbitmap_for_each_set(struct sbitmap *sb,
        nr = SB_NR_TO_BIT(sb, start);
 
        while (scanned < sb->depth) {
-               struct sbitmap_word *word = &sb->map[index];
-               unsigned int depth = min_t(unsigned int, word->depth - nr,
+               unsigned long word;
+               unsigned int depth = min_t(unsigned int,
+                                          sb->map[index].depth - nr,
                                           sb->depth - scanned);
 
                scanned += depth;
-               if (!word->word)
+               word = sb->map[index].word & ~sb->map[index].cleared;
+               if (!word)
                        goto next;
 
                /*
@@ -280,7 +282,7 @@ static inline void __sbitmap_for_each_set(struct sbitmap *sb,
                 */
                depth += nr;
                while (1) {
-                       nr = find_next_bit(&word->word, depth, nr);
+                       nr = find_next_bit(&word, depth, nr);
                        if (nr >= depth)
                                break;
                        if (!fn(sb, (index << sb->shift) + nr, data))