@@ -273,6 +273,7 @@ struct blkdev_dio {
};
size_t size;
atomic_t ref;
+ blk_qc_t qc;
bool multi_bio : 1;
bool should_dirty : 1;
bool is_sync : 1;
@@ -281,6 +282,21 @@ struct blkdev_dio {
static struct bio_set blkdev_dio_pool;
+static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
+{
+ struct blkdev_dio *dio = READ_ONCE(kiocb->private);
+
+ /* dio can be NULL here, if the IO hasn't been submitted yet */
+ if (dio) {
+ struct block_device *bdev;
+
+ bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
+ return blk_poll(bdev_get_queue(bdev), READ_ONCE(dio->qc), wait);
+ }
+
+ return 0;
+}
+
static void blkdev_bio_end_io(struct bio *bio)
{
struct blkdev_dio *dio = bio->bi_private;
@@ -335,7 +351,6 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;
bool is_read = (iov_iter_rw(iter) == READ), is_sync;
loff_t pos = iocb->ki_pos;
- blk_qc_t qc = BLK_QC_T_NONE;
int ret = 0;
if ((pos | iov_iter_alignment(iter)) &
@@ -355,6 +370,9 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
dio->size = 0;
dio->multi_bio = false;
dio->should_dirty = is_read && iter_is_iovec(iter);
+ dio->qc = BLK_QC_T_NONE;
+
+ WRITE_ONCE(iocb->private, dio);
/*
* Don't plug for HIPRI/polled IO, as those should go straight
@@ -395,7 +413,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
if (iocb->ki_flags & IOCB_HIPRI)
bio->bi_opf |= REQ_HIPRI;
- qc = submit_bio(bio);
+ dio->qc = submit_bio(bio);
break;
}
@@ -423,7 +441,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
break;
if (!(iocb->ki_flags & IOCB_HIPRI) ||
- !blk_poll(bdev_get_queue(bdev), qc, true))
+ !blk_poll(bdev_get_queue(bdev), dio->qc, true))
io_schedule();
}
__set_current_state(TASK_RUNNING);
@@ -2061,6 +2079,7 @@ const struct file_operations def_blk_fops = {
.llseek = block_llseek,
.read_iter = blkdev_read_iter,
.write_iter = blkdev_write_iter,
+ .iopoll = blkdev_iopoll,
.mmap = generic_file_mmap,
.fsync = blkdev_fsync,
.unlocked_ioctl = block_ioctl,
Signed-off-by: Jens Axboe <axboe@kernel.dk> --- fs/block_dev.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-)