@@ -440,11 +440,7 @@ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev,
return bio;
}
put_cpu();
- bio = bio_alloc_bioset(bdev, nr_vecs, opf, gfp, bs);
- if (!bio)
- return NULL;
- bio_set_flag(bio, BIO_PERCPU_CACHE);
- return bio;
+ return NULL;
}
/**
@@ -488,11 +484,24 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
gfp_t saved_gfp = gfp_mask;
struct bio *bio;
void *p;
+ bool use_alloc_cache;
/* should not use nobvec bioset for nr_vecs > 0 */
if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0))
return NULL;
+ use_alloc_cache = (bs->cache && (opf & REQ_ALLOC_CACHE) &&
+ nr_vecs <= BIO_INLINE_VECS);
+ if (use_alloc_cache) {
+ bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf, gfp_mask, bs);
+ if (bio)
+ return bio;
+ /*
+ * No cached bio available, mark bio returned below to
+ * particpate in per-cpu alloc cache.
+ */
+ }
+
/*
* submit_bio_noacct() converts recursion to iteration; this means if
* we're running beneath it, any bios we allocate and submit will not be
@@ -546,6 +555,8 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
bio_init(bio, bdev, NULL, 0, opf);
}
+ if (use_alloc_cache)
+ bio_set_flag(bio, BIO_PERCPU_CACHE);
bio->bi_pool = bs;
return bio;
@@ -795,10 +806,7 @@ struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src,
{
struct bio *bio;
- if (bs->cache && bio_src->bi_opf & REQ_POLLED)
- bio = bio_alloc_percpu_cache(bdev, 0, bio_src->bi_opf, gfp, bs);
- else
- bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs);
+ bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs);
if (!bio)
return NULL;
@@ -1792,10 +1800,9 @@ EXPORT_SYMBOL(bioset_init_from_src);
struct bio *bio_alloc_kiocb(struct kiocb *kiocb, struct block_device *bdev,
unsigned short nr_vecs, unsigned int opf, struct bio_set *bs)
{
- if (!(kiocb->ki_flags & IOCB_ALLOC_CACHE) || nr_vecs > BIO_INLINE_VECS)
- return bio_alloc_bioset(bdev, nr_vecs, opf, GFP_KERNEL, bs);
-
- return bio_alloc_percpu_cache(bdev, nr_vecs, opf, GFP_KERNEL, bs);
+ if (kiocb->ki_flags & IOCB_ALLOC_CACHE)
+ opf |= REQ_ALLOC_CACHE;
+ return bio_alloc_bioset(bdev, nr_vecs, opf, GFP_KERNEL, bs);
}
EXPORT_SYMBOL_GPL(bio_alloc_kiocb);
@@ -783,6 +783,8 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
{
bio->bi_opf |= REQ_POLLED;
+ if (kiocb->ki_flags & IOCB_ALLOC_CACHE)
+ bio->bi_opf |= REQ_ALLOC_CACHE;
if (!is_sync_kiocb(kiocb))
bio->bi_opf |= REQ_NOWAIT;
}
@@ -791,7 +793,7 @@ static inline void bio_clear_polled(struct bio *bio)
{
/* can't support alloc cache if we turn off polling */
bio_clear_flag(bio, BIO_PERCPU_CACHE);
- bio->bi_opf &= ~REQ_POLLED;
+ bio->bi_opf &= ~(REQ_POLLED | REQ_ALLOC_CACHE);
}
struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev,
@@ -327,7 +327,7 @@ enum {
BIO_TRACKED, /* set if bio goes through the rq_qos path */
BIO_REMAPPED,
BIO_ZONE_WRITE_LOCKED, /* Owns a zoned device zone write lock */
- BIO_PERCPU_CACHE, /* can participate in per-cpu alloc cache */
+ BIO_PERCPU_CACHE, /* participates in per-cpu alloc cache */
BIO_FLAG_LAST
};
@@ -414,6 +414,7 @@ enum req_flag_bits {
__REQ_NOUNMAP, /* do not free blocks when zeroing */
__REQ_POLLED, /* caller polls for completion using bio_poll */
+ __REQ_ALLOC_CACHE, /* allocate IO from cache if available */
/* for driver use */
__REQ_DRV,
@@ -439,6 +440,7 @@ enum req_flag_bits {
#define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP)
#define REQ_POLLED (1ULL << __REQ_POLLED)
+#define REQ_ALLOC_CACHE (1ULL << __REQ_ALLOC_CACHE)
#define REQ_DRV (1ULL << __REQ_DRV)
#define REQ_SWAP (1ULL << __REQ_SWAP)
Add REQ_ALLOC_CACHE and set it in %opf passed to bio_alloc_bioset to inform bio_alloc_bioset (and any stacked block drivers) that bio should be allocated from respective bioset's per-cpu alloc cache if possible. This decouples access control to the alloc cache (via REQ_ALLOC_CACHE) from actual participation in a specific alloc cache (BIO_PERCPU_CACHE). Otherwise an upper layer's bioset may not have an alloc cache, in which case the bio issued to underlying device(s) wouldn't reflect that allocating from an alloc cache warranted (if possible). Signed-off-by: Mike Snitzer <snitzer@kernel.org> --- block/bio.c | 33 ++++++++++++++++++++------------- include/linux/bio.h | 4 +++- include/linux/blk_types.h | 4 +++- 3 files changed, 26 insertions(+), 15 deletions(-)