diff mbox series

[10/12] block: RCU free polled bios

Message ID 20210426134821.2191160-11-hch@lst.de (mailing list archive)
State New
Headers show
Series [01/12] direct-io: remove blk_poll support | expand

Commit Message

Christoph Hellwig April 26, 2021, 1:48 p.m. UTC
Free the actual memory used for the bio using RCU for polled bios.
This will allow to store all the polling information in the bio and
thus simplify passing the cookie and allow for polling stacked devices.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 block/bio.c               | 28 +++++++++++++++++++---------
 include/linux/blk_types.h |  6 +++++-
 2 files changed, 24 insertions(+), 10 deletions(-)
diff mbox series

Patch

diff --git a/block/bio.c b/block/bio.c
index 303298996afe..7296abe293de 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -215,25 +215,35 @@  void bio_uninit(struct bio *bio)
 }
 EXPORT_SYMBOL(bio_uninit);
 
+static inline void bio_mempool_free(struct bio *bio)
+{
+	struct bio_set *bs = bio->bi_pool;
+	void *p = bio;
+
+	p -= bs->front_pad;
+	mempool_free(p, &bs->bio_pool);
+}
+
+static void bio_free_rcu(struct rcu_head *head)
+{
+	bio_mempool_free(container_of(head, struct bio, bi_rcu_free));
+}
+
 static void bio_free(struct bio *bio)
 {
 	struct bio_set *bs = bio->bi_pool;
-	void *p;
 
 	bio_uninit(bio);
 
 	if (bs) {
 		bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
-
-		/*
-		 * If we have front padding, adjust the bio pointer before freeing
-		 */
-		p = bio;
-		p -= bs->front_pad;
-
-		mempool_free(p, &bs->bio_pool);
+		if (bio->bi_opf & REQ_POLLED)
+			call_rcu(&bio->bi_rcu_free, bio_free_rcu);
+		else
+			bio_mempool_free(bio);
 	} else {
 		/* Bio was allocated by bio_kmalloc() */
+		WARN_ON_ONCE(bio->bi_opf & REQ_POLLED);
 		kfree(bio);
 	}
 }
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index ac60432752e3..183a76bf24b7 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -10,6 +10,7 @@ 
 #include <linux/bvec.h>
 #include <linux/device.h>
 #include <linux/ktime.h>
+#include <linux/rcupdate.h>
 
 struct bio_set;
 struct bio;
@@ -233,7 +234,10 @@  struct bio {
 	blk_status_t		bi_status;
 	atomic_t		__bi_remaining;
 
-	struct bvec_iter	bi_iter;
+	union {
+		struct bvec_iter	bi_iter;
+		struct rcu_head		bi_rcu_free;
+	};
 
 	bio_end_io_t		*bi_end_io;