block: allow bio_for_each_segment_all() to iterate over multi-page bvec

This patch introduces one extra iterator variable to bio_for_each_segment_all(),
then we can allow bio_for_each_segment_all() to iterate over multi-page bvec.

Given it is just one mechannical & simple change on all bio_for_each_segment_all()
users, this patch does tree-wide change in one single patch, so that we can
avoid to use a temporary helper for this conversion.

Reviewed-by: Omar Sandoval <osandov@fb.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bio.c b/block/bio.c
index 4db1008309edd9c6ca8113afb553aad969de5b85..968b12fea5645d92bc9bcc02c1fb7813791d1c4c 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -1072,8 +1072,9 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
 {
        int i;
        struct bio_vec *bvec;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                ssize_t ret;
 
                ret = copy_page_from_iter(bvec->bv_page,
@@ -1103,8 +1104,9 @@ static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
 {
        int i;
        struct bio_vec *bvec;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                ssize_t ret;
 
                ret = copy_page_to_iter(bvec->bv_page,
@@ -1126,8 +1128,9 @@ void bio_free_pages(struct bio *bio)
 {
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i)
+       bio_for_each_segment_all(bvec, bio, i, iter_all)
                __free_page(bvec->bv_page);
 }
 EXPORT_SYMBOL(bio_free_pages);
@@ -1295,6 +1298,7 @@ struct bio *bio_map_user_iov(struct request_queue *q,
        struct bio *bio;
        int ret;
        struct bio_vec *bvec;
+       struct bvec_iter_all iter_all;
 
        if (!iov_iter_count(iter))
                return ERR_PTR(-EINVAL);
@@ -1368,7 +1372,7 @@ struct bio *bio_map_user_iov(struct request_queue *q,
        return bio;
 
  out_unmap:
-       bio_for_each_segment_all(bvec, bio, j) {
+       bio_for_each_segment_all(bvec, bio, j, iter_all) {
                put_page(bvec->bv_page);
        }
        bio_put(bio);
@@ -1379,11 +1383,12 @@ static void __bio_unmap_user(struct bio *bio)
 {
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
        /*
         * make sure we dirty pages we wrote to
         */
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                if (bio_data_dir(bio) == READ)
                        set_page_dirty_lock(bvec->bv_page);
 
@@ -1475,8 +1480,9 @@ static void bio_copy_kern_endio_read(struct bio *bio)
        char *p = bio->bi_private;
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
                p += bvec->bv_len;
        }
@@ -1585,8 +1591,9 @@ void bio_set_pages_dirty(struct bio *bio)
 {
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                if (!PageCompound(bvec->bv_page))
                        set_page_dirty_lock(bvec->bv_page);
        }
@@ -1596,8 +1603,9 @@ static void bio_release_pages(struct bio *bio)
 {
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i)
+       bio_for_each_segment_all(bvec, bio, i, iter_all)
                put_page(bvec->bv_page);
 }
 
@@ -1644,8 +1652,9 @@ void bio_check_pages_dirty(struct bio *bio)
        struct bio_vec *bvec;
        unsigned long flags;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
                        goto defer;
        }

diff --git a/block/bounce.c b/block/bounce.c
index ffb9e9ecfa7ee78f1428390a30d0667a8f778cde..add085e28b1d1d98204cc9f856815ee30759b728 100644 (file)
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -165,11 +165,12 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool)
        struct bio_vec *bvec, orig_vec;
        int i;
        struct bvec_iter orig_iter = bio_orig->bi_iter;
+       struct bvec_iter_all iter_all;
 
        /*
         * free up bounce indirect pages used
         */
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                orig_vec = bio_iter_iovec(bio_orig, orig_iter);
                if (bvec->bv_page != orig_vec.bv_page) {
                        dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
@@ -294,6 +295,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
        bool bounce = false;
        int sectors = 0;
        bool passthrough = bio_is_passthrough(*bio_orig);
+       struct bvec_iter_all iter_all;
 
        bio_for_each_segment(from, *bio_orig, iter) {
                if (i++ < BIO_MAX_PAGES)
@@ -313,7 +315,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
        bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL :
                        &bounce_bio_set);
 
-       bio_for_each_segment_all(to, bio, i) {
+       bio_for_each_segment_all(to, bio, i, iter_all) {
                struct page *page = to->bv_page;
 
                if (page_to_pfn(page) <= q->limits.bounce_pfn)

diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 23cb1dc7296bf2701fa93659bfeef3b8b66621ca..64def336f0532a91b5da500c77c3c8de38b7ee37 100644 (file)
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -432,8 +432,9 @@ static void do_btree_node_write(struct btree *b)
                int j;
                struct bio_vec *bv;
                void *base = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1));
+               struct bvec_iter_all iter_all;
 
-               bio_for_each_segment_all(bv, b->bio, j)
+               bio_for_each_segment_all(bv, b->bio, j, iter_all)
                        memcpy(page_address(bv->bv_page),
                               base + j * PAGE_SIZE, PAGE_SIZE);
 

diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 47d4e0d30bf08b3c5179c9196100b129f8b0437c..9a29037f56158f4664c28fa90ef3a733edaf533a 100644 (file)
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -1447,8 +1447,9 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
 {
        unsigned int i;
        struct bio_vec *bv;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bv, clone, i) {
+       bio_for_each_segment_all(bv, clone, i, iter_all) {
                BUG_ON(!bv->bv_page);
                mempool_free(bv->bv_page, &cc->page_pool);
        }

diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 7e63ccc4ae7b41c5f00191c8b91f9cbc62b502df..88c61d3090b0521f1d3cb6f47192beffef90c8a6 100644 (file)
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -2112,13 +2112,14 @@ static void process_checks(struct r1bio *r1_bio)
                struct page **spages = get_resync_pages(sbio)->pages;
                struct bio_vec *bi;
                int page_len[RESYNC_PAGES] = { 0 };
+               struct bvec_iter_all iter_all;
 
                if (sbio->bi_end_io != end_sync_read)
                        continue;
                /* Now we can 'fixup' the error value */
                sbio->bi_status = 0;
 
-               bio_for_each_segment_all(bi, sbio, j)
+               bio_for_each_segment_all(bi, sbio, j, iter_all)
                        page_len[j] = bi->bv_len;
 
                if (!status) {

diff --git a/drivers/staging/erofs/data.c b/drivers/staging/erofs/data.c
index 5a55f0bfdfbb268b09ca94014379fbb0d417005e..4871ba7b7d9ac7a826aec3258f52d25f76b402e6 100644 (file)
--- a/drivers/staging/erofs/data.c
+++ b/drivers/staging/erofs/data.c
@@ -20,8 +20,9 @@ static inline void read_endio(struct bio *bio)
        int i;
        struct bio_vec *bvec;
        const blk_status_t err = bio->bi_status;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
 
                /* page is already locked */

diff --git a/drivers/staging/erofs/unzip_vle.c b/drivers/staging/erofs/unzip_vle.c
index 4ac1099a39c6c6e0028b09916eb0238f9abcb4d9..c057c5616b1ddd0feccc069449c29ef34878e3d9 100644 (file)
--- a/drivers/staging/erofs/unzip_vle.c
+++ b/drivers/staging/erofs/unzip_vle.c
@@ -830,8 +830,9 @@ static inline void z_erofs_vle_read_endio(struct bio *bio)
 #ifdef EROFS_FS_HAS_MANAGED_CACHE
        struct address_space *mc = NULL;
 #endif
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
                bool cachemngd = false;
 

diff --git a/fs/block_dev.c b/fs/block_dev.c
index 58a4c1217fa87d60376771a2c315b8b02d0bff1c..7758adee6efecb3dfc27997e020ab00b67a59850 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -211,6 +211,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
        ssize_t ret;
        blk_qc_t qc;
        int i;
+       struct bvec_iter_all iter_all;
 
        if ((pos | iov_iter_alignment(iter)) &
            (bdev_logical_block_size(bdev) - 1))
@@ -260,7 +261,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
        }
        __set_current_state(TASK_RUNNING);
 
-       bio_for_each_segment_all(bvec, &bio, i) {
+       bio_for_each_segment_all(bvec, &bio, i, iter_all) {
                if (should_dirty && !PageCompound(bvec->bv_page))
                        set_page_dirty_lock(bvec->bv_page);
                put_page(bvec->bv_page);
@@ -329,8 +330,9 @@ static void blkdev_bio_end_io(struct bio *bio)
        } else {
                struct bio_vec *bvec;
                int i;
+               struct bvec_iter_all iter_all;
 
-               bio_for_each_segment_all(bvec, bio, i)
+               bio_for_each_segment_all(bvec, bio, i, iter_all)
                        put_page(bvec->bv_page);
                bio_put(bio);
        }

diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
index 548057630b69e9735d75eb1426869c85c0e521e9..6896ea60c843229b412bf705028172d652cad1ca 100644 (file)
--- a/fs/btrfs/compression.c
+++ b/fs/btrfs/compression.c
@@ -162,13 +162,14 @@ csum_failed:
        } else {
                int i;
                struct bio_vec *bvec;
+               struct bvec_iter_all iter_all;
 
                /*
                 * we have verified the checksum already, set page
                 * checked so the end_io handlers know about it
                 */
                ASSERT(!bio_flagged(bio, BIO_CLONED));
-               bio_for_each_segment_all(bvec, cb->orig_bio, i)
+               bio_for_each_segment_all(bvec, cb->orig_bio, i, iter_all)
                        SetPageChecked(bvec->bv_page);
 
                bio_endio(cb->orig_bio);

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 6a2a2a9517058b429b286557fd0a2b1a512c833d..ca1b7da6dd1b927e54fe84ef12c04ec56197875f 100644 (file)
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -832,9 +832,10 @@ static blk_status_t btree_csum_one_bio(struct bio *bio)
        struct bio_vec *bvec;
        struct btrfs_root *root;
        int i, ret = 0;
+       struct bvec_iter_all iter_all;
 
        ASSERT(!bio_flagged(bio, BIO_CLONED));
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                root = BTRFS_I(bvec->bv_page->mapping->host)->root;
                ret = csum_dirty_buffer(root->fs_info, bvec->bv_page);
                if (ret)

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 986ef49b02696bc2027144e15f3586785677a7cb..4ed58c9a94a9916959e304d28f77e9002b64e8df 100644 (file)
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2422,9 +2422,10 @@ static void end_bio_extent_writepage(struct bio *bio)
        u64 start;
        u64 end;
        int i;
+       struct bvec_iter_all iter_all;
 
        ASSERT(!bio_flagged(bio, BIO_CLONED));
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
                struct inode *inode = page->mapping->host;
                struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@@ -2493,9 +2494,10 @@ static void end_bio_extent_readpage(struct bio *bio)
        int mirror;
        int ret;
        int i;
+       struct bvec_iter_all iter_all;
 
        ASSERT(!bio_flagged(bio, BIO_CLONED));
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
                struct inode *inode = page->mapping->host;
                struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
@@ -3635,9 +3637,10 @@ static void end_bio_extent_buffer_writepage(struct bio *bio)
        struct bio_vec *bvec;
        struct extent_buffer *eb;
        int i, done;
+       struct bvec_iter_all iter_all;
 
        ASSERT(!bio_flagged(bio, BIO_CLONED));
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
 
                eb = (struct extent_buffer *)page->private;

diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 5c349667c761b5fed05a15d565c69eff015709e8..7ade5769f6915acfa1acfbb492db1d926c4df594 100644 (file)
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -7777,6 +7777,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
        struct bio_vec *bvec;
        struct extent_io_tree *io_tree, *failure_tree;
        int i;
+       struct bvec_iter_all iter_all;
 
        if (bio->bi_status)
                goto end;
@@ -7788,7 +7789,7 @@ static void btrfs_retry_endio_nocsum(struct bio *bio)
 
        done->uptodate = 1;
        ASSERT(!bio_flagged(bio, BIO_CLONED));
-       bio_for_each_segment_all(bvec, bio, i)
+       bio_for_each_segment_all(bvec, bio, i, iter_all)
                clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
                                 io_tree, done->start, bvec->bv_page,
                                 btrfs_ino(BTRFS_I(inode)), 0);
@@ -7867,6 +7868,7 @@ static void btrfs_retry_endio(struct bio *bio)
        int uptodate;
        int ret;
        int i;
+       struct bvec_iter_all iter_all;
 
        if (bio->bi_status)
                goto end;
@@ -7880,7 +7882,7 @@ static void btrfs_retry_endio(struct bio *bio)
        failure_tree = &BTRFS_I(inode)->io_failure_tree;
 
        ASSERT(!bio_flagged(bio, BIO_CLONED));
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
                                             bvec->bv_offset, done->start,
                                             bvec->bv_len);

diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index e74455eb42f9b09bdc6031098143fac7ec8650d0..1869ba8e5981c948c435bf26e3c2f9d8016770b0 100644 (file)
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -1443,10 +1443,11 @@ static void set_bio_pages_uptodate(struct bio *bio)
 {
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
        ASSERT(!bio_flagged(bio, BIO_CLONED));
 
-       bio_for_each_segment_all(bvec, bio, i)
+       bio_for_each_segment_all(bvec, bio, i, iter_all)
                SetPageUptodate(bvec->bv_page);
 }
 

diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c
index 0959044c5ceecb37168743784df8d6d269f6301b..5759bcd018cd3402686266afbd0a8d37fb2817f8 100644 (file)
--- a/fs/crypto/bio.c
+++ b/fs/crypto/bio.c
@@ -30,8 +30,9 @@ static void __fscrypt_decrypt_bio(struct bio *bio, bool done)
 {
        struct bio_vec *bv;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bv, bio, i) {
+       bio_for_each_segment_all(bv, bio, i, iter_all) {
                struct page *page = bv->bv_page;
                int ret = fscrypt_decrypt_page(page->mapping->host, page,
                                PAGE_SIZE, 0, page->index);

diff --git a/fs/direct-io.c b/fs/direct-io.c
index ec2fb6fe6d37c628d5335b7a0971b464824b0623..9bb015bc4a83676be822ca5bf46462ed8dafb047 100644 (file)
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -551,7 +551,9 @@ static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)
        if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) {
                bio_check_pages_dirty(bio);     /* transfers ownership */
        } else {
-               bio_for_each_segment_all(bvec, bio, i) {
+               struct bvec_iter_all iter_all;
+
+               bio_for_each_segment_all(bvec, bio, i, iter_all) {
                        struct page *page = bvec->bv_page;
 
                        if (dio->op == REQ_OP_READ && !PageCompound(page) &&

diff --git a/fs/exofs/ore.c b/fs/exofs/ore.c
index 5331a15a61f198fd8e897b582e3e48b9c9add993..24a8e34882e956ec4427a5f1e358a13c8b93f0c3 100644 (file)
--- a/fs/exofs/ore.c
+++ b/fs/exofs/ore.c
@@ -420,8 +420,9 @@ static void _clear_bio(struct bio *bio)
 {
        struct bio_vec *bv;
        unsigned i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bv, bio, i) {
+       bio_for_each_segment_all(bv, bio, i, iter_all) {
                unsigned this_count = bv->bv_len;
 
                if (likely(PAGE_SIZE == this_count))

diff --git a/fs/exofs/ore_raid.c b/fs/exofs/ore_raid.c
index 199590f362030d2c4dad33968b2e0a12ef90299a..e83bab54b03e26624b4305433253d4a6ef35d0c4 100644 (file)
--- a/fs/exofs/ore_raid.c
+++ b/fs/exofs/ore_raid.c
@@ -468,11 +468,12 @@ static void _mark_read4write_pages_uptodate(struct ore_io_state *ios, int ret)
        /* loop on all devices all pages */
        for (d = 0; d < ios->numdevs; d++) {
                struct bio *bio = ios->per_dev[d].bio;
+               struct bvec_iter_all iter_all;
 
                if (!bio)
                        continue;
 
-               bio_for_each_segment_all(bv, bio, i) {
+               bio_for_each_segment_all(bv, bio, i, iter_all) {
                        struct page *page = bv->bv_page;
 
                        SetPageUptodate(page);

diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c
index 2aa62d58d8dd87e095bcb61f84aa78ef755463a1..cff4c4aa7a9c2d300c9ac2bc8f50c3981726d4d5 100644 (file)
--- a/fs/ext4/page-io.c
+++ b/fs/ext4/page-io.c
@@ -63,8 +63,9 @@ static void ext4_finish_bio(struct bio *bio)
 {
        int i;
        struct bio_vec *bvec;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
 #ifdef CONFIG_EXT4_FS_ENCRYPTION
                struct page *data_page = NULL;

diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
index 6aa282ee455a929aa2b5927e39d0f20c7bd33acd..e53639784892895044fef09a4d5c5b30f8461203 100644 (file)
--- a/fs/ext4/readpage.c
+++ b/fs/ext4/readpage.c
@@ -72,6 +72,7 @@ static void mpage_end_io(struct bio *bio)
 {
        struct bio_vec *bv;
        int i;
+       struct bvec_iter_all iter_all;
 
        if (ext4_bio_encrypted(bio)) {
                if (bio->bi_status) {
@@ -81,7 +82,7 @@ static void mpage_end_io(struct bio *bio)
                        return;
                }
        }
-       bio_for_each_segment_all(bv, bio, i) {
+       bio_for_each_segment_all(bv, bio, i, iter_all) {
                struct page *page = bv->bv_page;
 
                if (!bio->bi_status) {

diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index f91d8630c9a2d768d365e443811c37a4a1023a1e..da060b77f64d905c828c1ee3bd1ea069fc6c816b 100644 (file)
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -87,8 +87,9 @@ static void __read_end_io(struct bio *bio)
        struct page *page;
        struct bio_vec *bv;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bv, bio, i) {
+       bio_for_each_segment_all(bv, bio, i, iter_all) {
                page = bv->bv_page;
 
                /* PG_error was set if any post_read step failed */
@@ -164,13 +165,14 @@ static void f2fs_write_end_io(struct bio *bio)
        struct f2fs_sb_info *sbi = bio->bi_private;
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
        if (time_to_inject(sbi, FAULT_WRITE_IO)) {
                f2fs_show_injection_info(FAULT_WRITE_IO);
                bio->bi_status = BLK_STS_IOERR;
        }
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
                enum count_type type = WB_DATA_TYPE(page);
 
@@ -347,6 +349,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
        struct bio_vec *bvec;
        struct page *target;
        int i;
+       struct bvec_iter_all iter_all;
 
        if (!io->bio)
                return false;
@@ -354,7 +357,7 @@ static bool __has_merged_page(struct f2fs_bio_info *io, struct inode *inode,
        if (!inode && !page && !ino)
                return true;
 
-       bio_for_each_segment_all(bvec, io->bio, i) {
+       bio_for_each_segment_all(bvec, io->bio, i, iter_all) {
 
                if (bvec->bv_page->mapping)
                        target = bvec->bv_page;

diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index 94dcab655bc0211ddffd934886850e3c808903c3..15deefeaafd0d68f912c4b4733aa5dee936da43d 100644 (file)
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -170,7 +170,8 @@ u64 gfs2_log_bmap(struct gfs2_sbd *sdp)
  * that is pinned in the pagecache.
  */
 
-static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct bio_vec *bvec,
+static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp,
+                                 struct bio_vec *bvec,
                                  blk_status_t error)
 {
        struct buffer_head *bh, *next;
@@ -208,6 +209,7 @@ static void gfs2_end_log_write(struct bio *bio)
        struct bio_vec *bvec;
        struct page *page;
        int i;
+       struct bvec_iter_all iter_all;
 
        if (bio->bi_status) {
                fs_err(sdp, "Error %d writing to journal, jid=%u\n",
@@ -215,7 +217,7 @@ static void gfs2_end_log_write(struct bio *bio)
                wake_up(&sdp->sd_logd_waitq);
        }
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                page = bvec->bv_page;
                if (page_has_buffers(page))
                        gfs2_end_log_write_bh(sdp, bvec, bio->bi_status);
@@ -388,8 +390,9 @@ static void gfs2_end_log_read(struct bio *bio)
        struct page *page;
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                page = bvec->bv_page;
                if (bio->bi_status) {
                        int err = blk_status_to_errno(bio->bi_status);

diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c
index be9c0bf697fe6ec4d8d0659900d3de9555ba0a4a..3201342404a7785b5dc16429d7b2abe37e30b905 100644 (file)
--- a/fs/gfs2/meta_io.c
+++ b/fs/gfs2/meta_io.c
@@ -190,8 +190,9 @@ static void gfs2_meta_read_endio(struct bio *bio)
 {
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i) {
+       bio_for_each_segment_all(bvec, bio, i, iter_all) {
                struct page *page = bvec->bv_page;
                struct buffer_head *bh = page_buffers(page);
                unsigned int len = bvec->bv_len;

diff --git a/fs/iomap.c b/fs/iomap.c
index a3088fae567ba3092735667e4f9ac2595a1814b1..af736acd9006196eeb49db6bd9290a209c393b2e 100644 (file)
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -267,8 +267,9 @@ iomap_read_end_io(struct bio *bio)
        int error = blk_status_to_errno(bio->bi_status);
        struct bio_vec *bvec;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bvec, bio, i)
+       bio_for_each_segment_all(bvec, bio, i, iter_all)
                iomap_read_page_end_io(bvec, error);
        bio_put(bio);
 }
@@ -1559,8 +1560,9 @@ static void iomap_dio_bio_end_io(struct bio *bio)
        } else {
                struct bio_vec *bvec;
                int i;
+               struct bvec_iter_all iter_all;
 
-               bio_for_each_segment_all(bvec, bio, i)
+               bio_for_each_segment_all(bvec, bio, i, iter_all)
                        put_page(bvec->bv_page);
                bio_put(bio);
        }

diff --git a/fs/mpage.c b/fs/mpage.c
index c820dc9bebab6143a759c829ed478cb90931048a..3f19da75178be4eba8d68f2537a10bbc5e63f175 100644 (file)
--- a/fs/mpage.c
+++ b/fs/mpage.c
@@ -48,8 +48,9 @@ static void mpage_end_io(struct bio *bio)
 {
        struct bio_vec *bv;
        int i;
+       struct bvec_iter_all iter_all;
 
-       bio_for_each_segment_all(bv, bio, i) {
+       bio_for_each_segment_all(bv, bio, i, iter_all) {
                struct page *page = bv->bv_page;
                page_endio(page, bio_op(bio),
                           blk_status_to_errno(bio->bi_status));

diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 338b9d9984e04a62d790ae72638017c9ac3329d5..1f1829e506e8d5904c2e60d81eeaa1a17e31bc00 100644 (file)
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -62,7 +62,7 @@ xfs_find_daxdev_for_inode(
 static void
 xfs_finish_page_writeback(
        struct inode            *inode,
-       struct bio_vec          *bvec,
+       struct bio_vec  *bvec,
        int                     error)
 {
        struct iomap_page       *iop = to_iomap_page(bvec->bv_page);
@@ -98,6 +98,7 @@ xfs_destroy_ioend(
        for (bio = &ioend->io_inline_bio; bio; bio = next) {
                struct bio_vec  *bvec;
                int             i;
+               struct bvec_iter_all iter_all;
 
                /*
                 * For the last bio, bi_private points to the ioend, so we
@@ -109,7 +110,7 @@ xfs_destroy_ioend(
                        next = bio->bi_private;
 
                /* walk each page on bio, ending page IO on them */
-               bio_for_each_segment_all(bvec, bio, i)
+               bio_for_each_segment_all(bvec, bio, i, iter_all)
                        xfs_finish_page_writeback(inode, bvec, error);
                bio_put(bio);
        }

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 7ef8a7505c0a221f2d9b05fa92dc6a6fb0895f1c..089370eb84d97bd1b471d5789bc0010d6adbe911 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -128,12 +128,19 @@ static inline bool bio_full(struct bio *bio)
        return bio->bi_vcnt >= bio->bi_max_vecs;
 }
 
+#define mp_bvec_for_each_segment(bv, bvl, i, iter_all)                 \
+       for (bv = bvec_init_iter_all(&iter_all);                        \
+               (iter_all.done < (bvl)->bv_len) &&                      \
+               (mp_bvec_next_segment((bvl), &iter_all), 1);            \
+               iter_all.done += bv->bv_len, i += 1)
+
 /*
  * drivers should _never_ use the all version - the bio may have been split
  * before it got to the driver and the driver won't own all of it
  */
-#define bio_for_each_segment_all(bvl, bio, i)                          \
-       for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++)
+#define bio_for_each_segment_all(bvl, bio, i, iter_all)                \
+       for (i = 0, iter_all.idx = 0; iter_all.idx < (bio)->bi_vcnt; iter_all.idx++)    \
+               mp_bvec_for_each_segment(bvl, &((bio)->bi_io_vec[iter_all.idx]), i, iter_all)
 
 static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
                                    unsigned bytes)

diff --git a/include/linux/bvec.h b/include/linux/bvec.h
index 21f76bad7be2263b34aba49ec385d6724f67698e..30a57b68d0175c091aae8ddfb1cf43b448ee49b1 100644 (file)
--- a/include/linux/bvec.h
+++ b/include/linux/bvec.h
@@ -45,6 +45,12 @@ struct bvec_iter {
                                                   current bvec */
 };
 
+struct bvec_iter_all {
+       struct bio_vec  bv;
+       int             idx;
+       unsigned        done;
+};
+
 /*
  * various member access, note that bio_data should of course not be used
  * on highmem page vectors
@@ -131,6 +137,30 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
        .bi_bvec_done   = 0,                                            \
 }
 
+static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
+{
+       iter_all->bv.bv_page = NULL;
+       iter_all->done = 0;
+
+       return &iter_all->bv;
+}
+
+static inline void mp_bvec_next_segment(const struct bio_vec *bvec,
+                                       struct bvec_iter_all *iter_all)
+{
+       struct bio_vec *bv = &iter_all->bv;
+
+       if (bv->bv_page) {
+               bv->bv_page = nth_page(bv->bv_page, 1);
+               bv->bv_offset = 0;
+       } else {
+               bv->bv_page = bvec->bv_page;
+               bv->bv_offset = bvec->bv_offset;
+       }
+       bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
+                          bvec->bv_len - iter_all->done);
+}
+
 /*
  * Get the last single-page segment from the multi-page bvec and store it
  * in @seg