@@ -74,8 +74,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
return generic_file_read_iter(iocb, to);
}
- ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL,
- is_sync_kiocb(iocb));
+ ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0);
inode_unlock_shared(inode);
file_accessed(iocb->ki_filp);
@@ -371,6 +370,7 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
handle_t *handle;
struct inode *inode = file_inode(iocb->ki_filp);
bool extend = false, overwrite = false, unaligned_aio = false;
+ unsigned int dio_flags = 0;
if (iocb->ki_flags & IOCB_NOWAIT) {
if (!inode_trylock(inode))
@@ -404,6 +404,7 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) &&
!is_sync_kiocb(iocb) && ext4_unaligned_aio(inode, from, offset)) {
unaligned_aio = true;
+ dio_flags |= IOMAP_DIO_SYNCHRONOUS;
inode_dio_wait(inode);
}
@@ -432,11 +433,12 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
}
extend = true;
+ dio_flags |= IOMAP_DIO_SYNCHRONOUS;
ext4_journal_stop(handle);
}
ret = iomap_dio_rw(iocb, from, &ext4_iomap_ops, &ext4_dio_write_ops,
- is_sync_kiocb(iocb) || unaligned_aio || extend);
+ dio_flags);
if (extend)
ret = ext4_handle_inode_extension(inode, offset, ret, count);
@@ -771,8 +771,7 @@ static ssize_t gfs2_file_direct_read(struct kiocb *iocb, struct iov_iter *to)
if (ret)
goto out_uninit;
- ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL,
- is_sync_kiocb(iocb));
+ ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, 0);
gfs2_glock_dq(&gh);
out_uninit:
@@ -807,8 +806,7 @@ static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
if (offset + len > i_size_read(&ip->i_inode))
goto out;
- ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL,
- is_sync_kiocb(iocb));
+ ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, 0);
out:
gfs2_glock_dq(&gh);
@@ -400,7 +400,7 @@ iomap_dio_actor(struct inode *inode, loff_t pos, loff_t length,
ssize_t
iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
- bool wait_for_completion)
+ unsigned int dio_flags)
{
struct address_space *mapping = iocb->ki_filp->f_mapping;
struct inode *inode = file_inode(iocb->ki_filp);
@@ -410,14 +410,15 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
unsigned int flags = IOMAP_DIRECT;
struct blk_plug plug;
struct iomap_dio *dio;
+ bool wait_for_completion = false;
lockdep_assert_held(&inode->i_rwsem);
if (!count)
return 0;
- if (WARN_ON(is_sync_kiocb(iocb) && !wait_for_completion))
- return -EIO;
+ if (is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_SYNCHRONOUS))
+ wait_for_completion = true;
dio = kmalloc(sizeof(*dio), GFP_KERNEL);
if (!dio)
@@ -193,8 +193,7 @@ xfs_file_dio_aio_read(
} else {
xfs_ilock(ip, XFS_IOLOCK_SHARED);
}
- ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL,
- is_sync_kiocb(iocb));
+ ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, 0);
xfs_iunlock(ip, XFS_IOLOCK_SHARED);
return ret;
@@ -493,6 +492,7 @@ xfs_file_dio_aio_write(
int iolock;
size_t count = iov_iter_count(from);
struct xfs_buftarg *target = xfs_inode_buftarg(ip);
+ unsigned int dio_flags = 0;
/* DIO must be aligned to device logical sector size */
if ((iocb->ki_pos | count) & target->bt_logical_sectormask)
@@ -538,27 +538,24 @@ xfs_file_dio_aio_write(
count = iov_iter_count(from);
/*
- * If we are doing unaligned IO, we can't allow any other overlapping IO
- * in-flight at the same time or we risk data corruption. Wait for all
- * other IO to drain before we submit. If the IO is aligned, demote the
- * iolock if we had to take the exclusive lock in
+ * If we are doing unaligned I/O, we can't allow any other overlapping
+ * I/O in-flight at the same time or we risk data corruption. Wait for
+ * all other I/O to drain before we submit and execute the I/O
+ * synchronously to prevent subsequent overlapping I/O. If the I/O is
+ * aligned, demote the iolock if we had to take the exclusive lock in
* xfs_file_aio_write_checks() for other reasons.
*/
if (unaligned_io) {
inode_dio_wait(inode);
+ dio_flags = IOMAP_DIO_SYNCHRONOUS;
} else if (iolock == XFS_IOLOCK_EXCL) {
xfs_ilock_demote(ip, XFS_IOLOCK_EXCL);
iolock = XFS_IOLOCK_SHARED;
}
trace_xfs_file_direct_write(ip, count, iocb->ki_pos);
- /*
- * If unaligned, this is the only IO in-flight. Wait on it before we
- * release the iolock to prevent subsequent overlapping IO.
- */
ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
- &xfs_dio_write_ops,
- is_sync_kiocb(iocb) || unaligned_io);
+ &xfs_dio_write_ops, dio_flags);
out:
xfs_iunlock(ip, iolock);
@@ -244,10 +244,11 @@ int iomap_writepages(struct address_space *mapping,
const struct iomap_writeback_ops *ops);
/*
- * Flags for direct I/O ->end_io:
+ * Flags for iomap_dio_complete and ->end_io:
*/
#define IOMAP_DIO_UNWRITTEN (1 << 0) /* covers unwritten extent(s) */
#define IOMAP_DIO_COW (1 << 1) /* covers COW extent(s) */
+#define IOMAP_DIO_SYNCHRONOUS (1 << 2) /* no async completion */
struct iomap_dio_ops {
int (*end_io)(struct kiocb *iocb, ssize_t size, int error,
@@ -256,7 +257,7 @@ struct iomap_dio_ops {
ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
- bool wait_for_completion);
+ unsigned int dio_flags);
int iomap_dio_iopoll(struct kiocb *kiocb, bool spin);
#ifdef CONFIG_SWAP
Replace the wait_for_completion flag in struct iomap_dio with a new IOMAP_DIO_SYNCHRONOUS flag for dio->flags, and allow passing the initial flags to iomap_dio_rw. Also take the check for synchronous iocbs into iomap_dio_rw instead of duplicating it in all the callers. Signed-off-by: Christoph Hellwig <hch@lst.de> --- fs/ext4/file.c | 8 +++++--- fs/gfs2/file.c | 6 ++---- fs/iomap/direct-io.c | 7 ++++--- fs/xfs/xfs_file.c | 21 +++++++++------------ include/linux/iomap.h | 5 +++-- 5 files changed, 23 insertions(+), 24 deletions(-)