diff mbox

[v7,4/4] block: ignore flush requests when storage is clean

Message ID 1468590449-25465-5-git-send-email-den@openvz.org (mailing list archive)
State New, archived
Headers show

Commit Message

Denis V. Lunev July 15, 2016, 1:47 p.m. UTC
From: Evgeny Yakovlev <eyakovlev@virtuozzo.com>

Some guests (win2008 server for example) do a lot of unnecessary
flushing when underlying media has not changed. This adds additional
overhead on host when calling fsync/fdatasync.

This change introduces a write generation scheme in BlockDriverState.
Current write generation is checked against last flushed generation to
avoid unnessesary flushes.

The problem with excessive flushing was found by a performance test
which does parallel directory tree creation (from 2 processes).
Results improved from 0.424 loops/sec to 0.432 loops/sec.
Each loop creates 10^3 directories with 10 files in each.

This affected some blkdebug testcases that were expecting error logs from
failure-injected flushes which are now skipped entirely
(tests 026 071 089).

This also affects the performance of block jobs and thus BLOCK_JOB_READY
events for driver-mirror and active block-commit commands now arrives
faster, before QMP send successfully returns to caller (tests 141 144).

Signed-off-by: Evgeny Yakovlev <eyakovlev@virtuozzo.com>
Signed-off-by: Denis V. Lunev <den@openvz.org>
Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
CC: Kevin Wolf <kwolf@redhat.com>
CC: Max Reitz <mreitz@redhat.com>
CC: Stefan Hajnoczi <stefanha@redhat.com>
CC: Fam Zheng <famz@redhat.com>
CC: John Snow <jsnow@redhat.com>
---
 block.c                            |  3 +++
 block/io.c                         | 21 ++++++++++++++++
 include/block/block_int.h          |  5 ++++
 tests/qemu-iotests/026.out.nocache | 50 --------------------------------------
 tests/qemu-iotests/071.out         |  8 ------
 tests/qemu-iotests/089.out         |  2 --
 tests/qemu-iotests/141.out         |  4 +--
 tests/qemu-iotests/144.out         |  2 +-
 8 files changed, 32 insertions(+), 63 deletions(-)

Comments

John Snow July 18, 2016, 7:10 p.m. UTC | #1
On 07/15/2016 09:47 AM, Denis V. Lunev wrote:
> From: Evgeny Yakovlev <eyakovlev@virtuozzo.com>
> 
> Some guests (win2008 server for example) do a lot of unnecessary
> flushing when underlying media has not changed. This adds additional
> overhead on host when calling fsync/fdatasync.
> 
> This change introduces a write generation scheme in BlockDriverState.
> Current write generation is checked against last flushed generation to
> avoid unnessesary flushes.
> 
> The problem with excessive flushing was found by a performance test
> which does parallel directory tree creation (from 2 processes).
> Results improved from 0.424 loops/sec to 0.432 loops/sec.
> Each loop creates 10^3 directories with 10 files in each.
> 
> This affected some blkdebug testcases that were expecting error logs from
> failure-injected flushes which are now skipped entirely
> (tests 026 071 089).
> 
> This also affects the performance of block jobs and thus BLOCK_JOB_READY
> events for driver-mirror and active block-commit commands now arrives
> faster, before QMP send successfully returns to caller (tests 141 144).
> 
> Signed-off-by: Evgeny Yakovlev <eyakovlev@virtuozzo.com>
> Signed-off-by: Denis V. Lunev <den@openvz.org>
> Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
> CC: Kevin Wolf <kwolf@redhat.com>
> CC: Max Reitz <mreitz@redhat.com>
> CC: Stefan Hajnoczi <stefanha@redhat.com>
> CC: Fam Zheng <famz@redhat.com>
> CC: John Snow <jsnow@redhat.com>
> ---
>  block.c                            |  3 +++
>  block/io.c                         | 21 ++++++++++++++++
>  include/block/block_int.h          |  5 ++++
>  tests/qemu-iotests/026.out.nocache | 50 --------------------------------------
>  tests/qemu-iotests/071.out         |  8 ------
>  tests/qemu-iotests/089.out         |  2 --
>  tests/qemu-iotests/141.out         |  4 +--
>  tests/qemu-iotests/144.out         |  2 +-
>  8 files changed, 32 insertions(+), 63 deletions(-)
> 
> diff --git a/block.c b/block.c
> index 823ff1d..060f88e 100644
> --- a/block.c
> +++ b/block.c
> @@ -234,6 +234,8 @@ BlockDriverState *bdrv_new(void)
>      bs->refcnt = 1;
>      bs->aio_context = qemu_get_aio_context();
>  
> +    qemu_co_queue_init(&bs->flush_queue);
> +
>      QTAILQ_INSERT_TAIL(&all_bdrv_states, bs, bs_list);
>  
>      return bs;
> @@ -2472,6 +2474,7 @@ int bdrv_truncate(BlockDriverState *bs, int64_t offset)
>          ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
>          bdrv_dirty_bitmap_truncate(bs);
>          bdrv_parent_cb_resize(bs);
> +        ++bs->write_gen;
>      }
>      return ret;
>  }
> diff --git a/block/io.c b/block/io.c
> index 7086908..f181ff7 100644
> --- a/block/io.c
> +++ b/block/io.c
> @@ -1303,6 +1303,7 @@ static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
>      }
>      bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
>  
> +    ++bs->write_gen;
>      bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
>  
>      if (bs->wr_highest_offset < offset + bytes) {
> @@ -2235,6 +2236,15 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>  
>      tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
>  
> +    int current_gen = bs->write_gen;
> +
> +    /* Wait until any previous flushes are completed */
> +    while (bs->flush_started_gen != bs->flushed_gen) {
> +        qemu_co_queue_wait(&bs->flush_queue);
> +    }
> +
> +    bs->flush_started_gen = current_gen;
> +
>      /* Write back all layers by calling one driver function */
>      if (bs->drv->bdrv_co_flush) {
>          ret = bs->drv->bdrv_co_flush(bs);
> @@ -2255,6 +2265,11 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>          goto flush_parent;
>      }
>  
> +    /* Check if we really need to flush anything */
> +    if (bs->flushed_gen == current_gen) {
> +        goto flush_parent;
> +    }
> +
>      BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
>      if (bs->drv->bdrv_co_flush_to_disk) {
>          ret = bs->drv->bdrv_co_flush_to_disk(bs);
> @@ -2285,6 +2300,7 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>           */
>          ret = 0;
>      }
> +
>      if (ret < 0) {
>          goto out;
>      }
> @@ -2295,6 +2311,10 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>  flush_parent:
>      ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
>  out:
> +    /* Notify any pending flushes that we have completed */
> +    bs->flushed_gen = current_gen;
> +    qemu_co_queue_restart_all(&bs->flush_queue);
> +
>      tracked_request_end(&req);
>      return ret;
>  }
> @@ -2420,6 +2440,7 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
>      }
>      ret = 0;
>  out:
> +    ++bs->write_gen;
>      bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
>                     req.bytes >> BDRV_SECTOR_BITS);
>      tracked_request_end(&req);
> diff --git a/include/block/block_int.h b/include/block/block_int.h
> index 47b9aac..396bd2b 100644
> --- a/include/block/block_int.h
> +++ b/include/block/block_int.h
> @@ -439,6 +439,11 @@ struct BlockDriverState {
>      int copy_on_read; /* if nonzero, copy read backing sectors into image.
>                           note this is a reference count */
>  
> +    CoQueue flush_queue;            /* Serializing flush queue */
> +    unsigned int write_gen;         /* Current data generation */
> +    unsigned int flush_started_gen; /* Generation for which flush has started */
> +    unsigned int flushed_gen;       /* Flushed write generation */
> +
>      BlockDriver *drv; /* NULL means no media */
>      void *opaque;
>  
> diff --git a/tests/qemu-iotests/026.out.nocache b/tests/qemu-iotests/026.out.nocache
> index 9c2c8a9..672d77c 100644
> --- a/tests/qemu-iotests/026.out.nocache
> +++ b/tests/qemu-iotests/026.out.nocache
> @@ -14,7 +14,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_update; errno: 5; imm: off; once: off; write 
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  
> @@ -23,7 +22,6 @@ This means waste of disk space, but no harm to data.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_update; errno: 5; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  
> @@ -42,7 +40,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_update; errno: 28; imm: off; once: off; write 
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  
> @@ -51,7 +48,6 @@ This means waste of disk space, but no harm to data.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_update; errno: 28; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  
> @@ -78,11 +74,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>  Event: l2_load; errno: 5; imm: off; once: off; write 
>  wrote 131072/131072 bytes at offset 0
>  128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  read failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
> @@ -90,11 +82,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>  Event: l2_load; errno: 5; imm: off; once: off; write -b
>  wrote 131072/131072 bytes at offset 0
>  128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  read failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
> @@ -118,11 +106,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>  Event: l2_load; errno: 28; imm: off; once: off; write 
>  wrote 131072/131072 bytes at offset 0
>  128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> -Failed to flush the L2 table cache: No space left on device
> -Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
> -Failed to flush the L2 table cache: No space left on device
> -Failed to flush the refcount block cache: No space left on device
>  read failed: No space left on device
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
> @@ -130,11 +114,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>  Event: l2_load; errno: 28; imm: off; once: off; write -b
>  wrote 131072/131072 bytes at offset 0
>  128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> -Failed to flush the L2 table cache: No space left on device
> -Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
> -Failed to flush the L2 table cache: No space left on device
> -Failed to flush the refcount block cache: No space left on device
>  read failed: No space left on device
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
> @@ -314,14 +294,12 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_load; errno: 5; imm: off; once: off; write 
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_load; errno: 5; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
> @@ -338,14 +316,12 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_load; errno: 28; imm: off; once: off; write 
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_load; errno: 28; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
> @@ -362,14 +338,12 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_update_part; errno: 5; imm: off; once: off; write 
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_update_part; errno: 5; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
> @@ -386,14 +360,12 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_update_part; errno: 28; imm: off; once: off; write 
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_update_part; errno: 28; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
> @@ -410,14 +382,12 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc; errno: 5; imm: off; once: off; write 
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc; errno: 5; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
> @@ -434,14 +404,12 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc; errno: 28; imm: off; once: off; write 
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc; errno: 28; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
> @@ -458,15 +426,11 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: cluster_alloc; errno: 5; imm: off; once: off; write 
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: cluster_alloc; errno: 5; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
> @@ -482,15 +446,11 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: cluster_alloc; errno: 28; imm: off; once: off; write 
> -Failed to flush the L2 table cache: No space left on device
> -Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: cluster_alloc; errno: 28; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: No space left on device
> -Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
>  
> @@ -561,7 +521,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write 
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  
> @@ -570,7 +529,6 @@ This means waste of disk space, but no harm to data.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  
> @@ -589,7 +547,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write 
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  
> @@ -598,7 +555,6 @@ This means waste of disk space, but no harm to data.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write -b
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  
> @@ -643,8 +599,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
> @@ -655,8 +609,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
> -Failed to flush the L2 table cache: No space left on device
> -Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
> @@ -667,7 +619,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_grow_write_table; errno: 5; imm: off; once: off
> -Failed to flush the L2 table cache: Input/output error
>  Failed to flush the refcount block cache: Input/output error
>  write failed: Input/output error
>  No errors were found on the image.
> @@ -679,7 +630,6 @@ No errors were found on the image.
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
>  
>  Event: l1_grow_write_table; errno: 28; imm: off; once: off
> -Failed to flush the L2 table cache: No space left on device
>  Failed to flush the refcount block cache: No space left on device
>  write failed: No space left on device
>  No errors were found on the image.
> diff --git a/tests/qemu-iotests/071.out b/tests/qemu-iotests/071.out
> index 2b40ead..8c6851e 100644
> --- a/tests/qemu-iotests/071.out
> +++ b/tests/qemu-iotests/071.out
> @@ -30,14 +30,10 @@ blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
>  
>  === Testing blkdebug through filename ===
>  
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  read failed: Input/output error
>  
>  === Testing blkdebug through file blockref ===
>  
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  read failed: Input/output error
>  
>  === Testing blkdebug on existing block device ===
> @@ -51,8 +47,6 @@ read failed: Input/output error
>  {"return": ""}
>  {"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
> -QEMU_PROG: Failed to flush the L2 table cache: Input/output error
> -QEMU_PROG: Failed to flush the refcount block cache: Input/output error
>  
>  
>  === Testing blkverify on existing block device ===
> @@ -92,7 +86,5 @@ read failed: Input/output error
>  {"return": ""}
>  {"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
> -QEMU_PROG: Failed to flush the L2 table cache: Input/output error
> -QEMU_PROG: Failed to flush the refcount block cache: Input/output error
>  
>  *** done
> diff --git a/tests/qemu-iotests/089.out b/tests/qemu-iotests/089.out
> index 5b541a3..18f5fdd 100644
> --- a/tests/qemu-iotests/089.out
> +++ b/tests/qemu-iotests/089.out
> @@ -24,8 +24,6 @@ read 512/512 bytes at offset 0
>  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
>  wrote 512/512 bytes at offset 229376
>  512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
> -Failed to flush the L2 table cache: Input/output error
> -Failed to flush the refcount block cache: Input/output error
>  read failed: Input/output error
>  
>  === Testing qemu-img info output ===
> diff --git a/tests/qemu-iotests/141.out b/tests/qemu-iotests/141.out
> index adceac1..eaf1e60 100644
> --- a/tests/qemu-iotests/141.out
> +++ b/tests/qemu-iotests/141.out
> @@ -18,8 +18,8 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.
>  
>  {"return": {}}
>  Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
> -{"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
> +{"return": {}}
>  {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: mirror"}}
>  {"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
> @@ -28,8 +28,8 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.
>  === Testing active block-commit ===
>  
>  {"return": {}}
> -{"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
> +{"return": {}}
>  {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}}
>  {"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
> diff --git a/tests/qemu-iotests/144.out b/tests/qemu-iotests/144.out
> index 410d741..387855c 100644
> --- a/tests/qemu-iotests/144.out
> +++ b/tests/qemu-iotests/144.out
> @@ -12,9 +12,9 @@ Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/
>  
>  === Performing block-commit on active layer ===
>  
> -{"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
>  {"return": {}}
> +{"return": {}}
>  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
>  
>  === Performing Live Snapshot 2 ===
> 

Patches 1-3: OK.
Patch 4 still breaks test 026...

I will try to get this merged, but please run the entire test suite on
patch 4 before re-sending.

--js
Denis V. Lunev July 18, 2016, 7:17 p.m. UTC | #2
On 07/18/2016 10:10 PM, John Snow wrote:
>
> On 07/15/2016 09:47 AM, Denis V. Lunev wrote:
>> From: Evgeny Yakovlev <eyakovlev@virtuozzo.com>
>>
>> Some guests (win2008 server for example) do a lot of unnecessary
>> flushing when underlying media has not changed. This adds additional
>> overhead on host when calling fsync/fdatasync.
>>
>> This change introduces a write generation scheme in BlockDriverState.
>> Current write generation is checked against last flushed generation to
>> avoid unnessesary flushes.
>>
>> The problem with excessive flushing was found by a performance test
>> which does parallel directory tree creation (from 2 processes).
>> Results improved from 0.424 loops/sec to 0.432 loops/sec.
>> Each loop creates 10^3 directories with 10 files in each.
>>
>> This affected some blkdebug testcases that were expecting error logs from
>> failure-injected flushes which are now skipped entirely
>> (tests 026 071 089).
>>
>> This also affects the performance of block jobs and thus BLOCK_JOB_READY
>> events for driver-mirror and active block-commit commands now arrives
>> faster, before QMP send successfully returns to caller (tests 141 144).
>>
>> Signed-off-by: Evgeny Yakovlev <eyakovlev@virtuozzo.com>
>> Signed-off-by: Denis V. Lunev <den@openvz.org>
>> Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
>> CC: Kevin Wolf <kwolf@redhat.com>
>> CC: Max Reitz <mreitz@redhat.com>
>> CC: Stefan Hajnoczi <stefanha@redhat.com>
>> CC: Fam Zheng <famz@redhat.com>
>> CC: John Snow <jsnow@redhat.com>
>> ---
>>   block.c                            |  3 +++
>>   block/io.c                         | 21 ++++++++++++++++
>>   include/block/block_int.h          |  5 ++++
>>   tests/qemu-iotests/026.out.nocache | 50 --------------------------------------
>>   tests/qemu-iotests/071.out         |  8 ------
>>   tests/qemu-iotests/089.out         |  2 --
>>   tests/qemu-iotests/141.out         |  4 +--
>>   tests/qemu-iotests/144.out         |  2 +-
>>   8 files changed, 32 insertions(+), 63 deletions(-)
>>
>> diff --git a/block.c b/block.c
>> index 823ff1d..060f88e 100644
>> --- a/block.c
>> +++ b/block.c
>> @@ -234,6 +234,8 @@ BlockDriverState *bdrv_new(void)
>>       bs->refcnt = 1;
>>       bs->aio_context = qemu_get_aio_context();
>>   
>> +    qemu_co_queue_init(&bs->flush_queue);
>> +
>>       QTAILQ_INSERT_TAIL(&all_bdrv_states, bs, bs_list);
>>   
>>       return bs;
>> @@ -2472,6 +2474,7 @@ int bdrv_truncate(BlockDriverState *bs, int64_t offset)
>>           ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
>>           bdrv_dirty_bitmap_truncate(bs);
>>           bdrv_parent_cb_resize(bs);
>> +        ++bs->write_gen;
>>       }
>>       return ret;
>>   }
>> diff --git a/block/io.c b/block/io.c
>> index 7086908..f181ff7 100644
>> --- a/block/io.c
>> +++ b/block/io.c
>> @@ -1303,6 +1303,7 @@ static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
>>       }
>>       bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
>>   
>> +    ++bs->write_gen;
>>       bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
>>   
>>       if (bs->wr_highest_offset < offset + bytes) {
>> @@ -2235,6 +2236,15 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>>   
>>       tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
>>   
>> +    int current_gen = bs->write_gen;
>> +
>> +    /* Wait until any previous flushes are completed */
>> +    while (bs->flush_started_gen != bs->flushed_gen) {
>> +        qemu_co_queue_wait(&bs->flush_queue);
>> +    }
>> +
>> +    bs->flush_started_gen = current_gen;
>> +
>>       /* Write back all layers by calling one driver function */
>>       if (bs->drv->bdrv_co_flush) {
>>           ret = bs->drv->bdrv_co_flush(bs);
>> @@ -2255,6 +2265,11 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>>           goto flush_parent;
>>       }
>>   
>> +    /* Check if we really need to flush anything */
>> +    if (bs->flushed_gen == current_gen) {
>> +        goto flush_parent;
>> +    }
>> +
>>       BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
>>       if (bs->drv->bdrv_co_flush_to_disk) {
>>           ret = bs->drv->bdrv_co_flush_to_disk(bs);
>> @@ -2285,6 +2300,7 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>>            */
>>           ret = 0;
>>       }
>> +
>>       if (ret < 0) {
>>           goto out;
>>       }
>> @@ -2295,6 +2311,10 @@ int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
>>   flush_parent:
>>       ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
>>   out:
>> +    /* Notify any pending flushes that we have completed */
>> +    bs->flushed_gen = current_gen;
>> +    qemu_co_queue_restart_all(&bs->flush_queue);
>> +
>>       tracked_request_end(&req);
>>       return ret;
>>   }
>> @@ -2420,6 +2440,7 @@ int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
>>       }
>>       ret = 0;
>>   out:
>> +    ++bs->write_gen;
>>       bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
>>                      req.bytes >> BDRV_SECTOR_BITS);
>>       tracked_request_end(&req);
>> diff --git a/include/block/block_int.h b/include/block/block_int.h
>> index 47b9aac..396bd2b 100644
>> --- a/include/block/block_int.h
>> +++ b/include/block/block_int.h
>> @@ -439,6 +439,11 @@ struct BlockDriverState {
>>       int copy_on_read; /* if nonzero, copy read backing sectors into image.
>>                            note this is a reference count */
>>   
>> +    CoQueue flush_queue;            /* Serializing flush queue */
>> +    unsigned int write_gen;         /* Current data generation */
>> +    unsigned int flush_started_gen; /* Generation for which flush has started */
>> +    unsigned int flushed_gen;       /* Flushed write generation */
>> +
>>       BlockDriver *drv; /* NULL means no media */
>>       void *opaque;
>>   
>> diff --git a/tests/qemu-iotests/026.out.nocache b/tests/qemu-iotests/026.out.nocache
>> index 9c2c8a9..672d77c 100644
>> --- a/tests/qemu-iotests/026.out.nocache
>> +++ b/tests/qemu-iotests/026.out.nocache
>> @@ -14,7 +14,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_update; errno: 5; imm: off; once: off; write
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   
>> @@ -23,7 +22,6 @@ This means waste of disk space, but no harm to data.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_update; errno: 5; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   
>> @@ -42,7 +40,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_update; errno: 28; imm: off; once: off; write
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   
>> @@ -51,7 +48,6 @@ This means waste of disk space, but no harm to data.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_update; errno: 28; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   
>> @@ -78,11 +74,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   Event: l2_load; errno: 5; imm: off; once: off; write
>>   wrote 131072/131072 bytes at offset 0
>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   read failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>> @@ -90,11 +82,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   Event: l2_load; errno: 5; imm: off; once: off; write -b
>>   wrote 131072/131072 bytes at offset 0
>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   read failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>> @@ -118,11 +106,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   Event: l2_load; errno: 28; imm: off; once: off; write
>>   wrote 131072/131072 bytes at offset 0
>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> -Failed to flush the L2 table cache: No space left on device
>> -Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>> -Failed to flush the L2 table cache: No space left on device
>> -Failed to flush the refcount block cache: No space left on device
>>   read failed: No space left on device
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>> @@ -130,11 +114,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   Event: l2_load; errno: 28; imm: off; once: off; write -b
>>   wrote 131072/131072 bytes at offset 0
>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> -Failed to flush the L2 table cache: No space left on device
>> -Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>> -Failed to flush the L2 table cache: No space left on device
>> -Failed to flush the refcount block cache: No space left on device
>>   read failed: No space left on device
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>> @@ -314,14 +294,12 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_load; errno: 5; imm: off; once: off; write
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_load; errno: 5; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>> @@ -338,14 +316,12 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_load; errno: 28; imm: off; once: off; write
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_load; errno: 28; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>> @@ -362,14 +338,12 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_update_part; errno: 5; imm: off; once: off; write
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_update_part; errno: 5; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>> @@ -386,14 +360,12 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_update_part; errno: 28; imm: off; once: off; write
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_update_part; errno: 28; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>> @@ -410,14 +382,12 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc; errno: 5; imm: off; once: off; write
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc; errno: 5; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>> @@ -434,14 +404,12 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc; errno: 28; imm: off; once: off; write
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc; errno: 28; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>> @@ -458,15 +426,11 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: cluster_alloc; errno: 5; imm: off; once: off; write
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: cluster_alloc; errno: 5; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>> @@ -482,15 +446,11 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: cluster_alloc; errno: 28; imm: off; once: off; write
>> -Failed to flush the L2 table cache: No space left on device
>> -Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: cluster_alloc; errno: 28; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: No space left on device
>> -Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>>   
>> @@ -561,7 +521,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   
>> @@ -570,7 +529,6 @@ This means waste of disk space, but no harm to data.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   
>> @@ -589,7 +547,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   
>> @@ -598,7 +555,6 @@ This means waste of disk space, but no harm to data.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write -b
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   
>> @@ -643,8 +599,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>> @@ -655,8 +609,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
>> -Failed to flush the L2 table cache: No space left on device
>> -Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>> @@ -667,7 +619,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_grow_write_table; errno: 5; imm: off; once: off
>> -Failed to flush the L2 table cache: Input/output error
>>   Failed to flush the refcount block cache: Input/output error
>>   write failed: Input/output error
>>   No errors were found on the image.
>> @@ -679,7 +630,6 @@ No errors were found on the image.
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>   
>>   Event: l1_grow_write_table; errno: 28; imm: off; once: off
>> -Failed to flush the L2 table cache: No space left on device
>>   Failed to flush the refcount block cache: No space left on device
>>   write failed: No space left on device
>>   No errors were found on the image.
>> diff --git a/tests/qemu-iotests/071.out b/tests/qemu-iotests/071.out
>> index 2b40ead..8c6851e 100644
>> --- a/tests/qemu-iotests/071.out
>> +++ b/tests/qemu-iotests/071.out
>> @@ -30,14 +30,10 @@ blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
>>   
>>   === Testing blkdebug through filename ===
>>   
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   read failed: Input/output error
>>   
>>   === Testing blkdebug through file blockref ===
>>   
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   read failed: Input/output error
>>   
>>   === Testing blkdebug on existing block device ===
>> @@ -51,8 +47,6 @@ read failed: Input/output error
>>   {"return": ""}
>>   {"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
>> -QEMU_PROG: Failed to flush the L2 table cache: Input/output error
>> -QEMU_PROG: Failed to flush the refcount block cache: Input/output error
>>   
>>   
>>   === Testing blkverify on existing block device ===
>> @@ -92,7 +86,5 @@ read failed: Input/output error
>>   {"return": ""}
>>   {"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
>> -QEMU_PROG: Failed to flush the L2 table cache: Input/output error
>> -QEMU_PROG: Failed to flush the refcount block cache: Input/output error
>>   
>>   *** done
>> diff --git a/tests/qemu-iotests/089.out b/tests/qemu-iotests/089.out
>> index 5b541a3..18f5fdd 100644
>> --- a/tests/qemu-iotests/089.out
>> +++ b/tests/qemu-iotests/089.out
>> @@ -24,8 +24,6 @@ read 512/512 bytes at offset 0
>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
>>   wrote 512/512 bytes at offset 229376
>>   512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>> -Failed to flush the L2 table cache: Input/output error
>> -Failed to flush the refcount block cache: Input/output error
>>   read failed: Input/output error
>>   
>>   === Testing qemu-img info output ===
>> diff --git a/tests/qemu-iotests/141.out b/tests/qemu-iotests/141.out
>> index adceac1..eaf1e60 100644
>> --- a/tests/qemu-iotests/141.out
>> +++ b/tests/qemu-iotests/141.out
>> @@ -18,8 +18,8 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.
>>   
>>   {"return": {}}
>>   Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
>> -{"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
>> +{"return": {}}
>>   {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: mirror"}}
>>   {"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
>> @@ -28,8 +28,8 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.
>>   === Testing active block-commit ===
>>   
>>   {"return": {}}
>> -{"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
>> +{"return": {}}
>>   {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}}
>>   {"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
>> diff --git a/tests/qemu-iotests/144.out b/tests/qemu-iotests/144.out
>> index 410d741..387855c 100644
>> --- a/tests/qemu-iotests/144.out
>> +++ b/tests/qemu-iotests/144.out
>> @@ -12,9 +12,9 @@ Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/
>>   
>>   === Performing block-commit on active layer ===
>>   
>> -{"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
>>   {"return": {}}
>> +{"return": {}}
>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
>>   
>>   === Performing Live Snapshot 2 ===
>>
> Patches 1-3: OK.
> Patch 4 still breaks test 026...
>
> I will try to get this merged, but please run the entire test suite on
> patch 4 before re-sending.
>
> --js
ah.... Scheisse. The problem is that 026 has 2 results, one for cached,
one for uncached IO. I have missed that. Give me 5 minutes.

We have fixed uncached mode only.

The problem is clear here.

Den
John Snow July 18, 2016, 7:18 p.m. UTC | #3
On 07/18/2016 03:17 PM, Denis V. Lunev wrote:
> On 07/18/2016 10:10 PM, John Snow wrote:
>>
>> On 07/15/2016 09:47 AM, Denis V. Lunev wrote:
>>> From: Evgeny Yakovlev <eyakovlev@virtuozzo.com>
>>>
>>> Some guests (win2008 server for example) do a lot of unnecessary
>>> flushing when underlying media has not changed. This adds additional
>>> overhead on host when calling fsync/fdatasync.
>>>
>>> This change introduces a write generation scheme in BlockDriverState.
>>> Current write generation is checked against last flushed generation to
>>> avoid unnessesary flushes.
>>>
>>> The problem with excessive flushing was found by a performance test
>>> which does parallel directory tree creation (from 2 processes).
>>> Results improved from 0.424 loops/sec to 0.432 loops/sec.
>>> Each loop creates 10^3 directories with 10 files in each.
>>>
>>> This affected some blkdebug testcases that were expecting error logs
>>> from
>>> failure-injected flushes which are now skipped entirely
>>> (tests 026 071 089).
>>>
>>> This also affects the performance of block jobs and thus BLOCK_JOB_READY
>>> events for driver-mirror and active block-commit commands now arrives
>>> faster, before QMP send successfully returns to caller (tests 141 144).
>>>
>>> Signed-off-by: Evgeny Yakovlev <eyakovlev@virtuozzo.com>
>>> Signed-off-by: Denis V. Lunev <den@openvz.org>
>>> Reviewed-by: Paolo Bonzini <pbonzini@redhat.com>
>>> CC: Kevin Wolf <kwolf@redhat.com>
>>> CC: Max Reitz <mreitz@redhat.com>
>>> CC: Stefan Hajnoczi <stefanha@redhat.com>
>>> CC: Fam Zheng <famz@redhat.com>
>>> CC: John Snow <jsnow@redhat.com>
>>> ---
>>>   block.c                            |  3 +++
>>>   block/io.c                         | 21 ++++++++++++++++
>>>   include/block/block_int.h          |  5 ++++
>>>   tests/qemu-iotests/026.out.nocache | 50
>>> --------------------------------------
>>>   tests/qemu-iotests/071.out         |  8 ------
>>>   tests/qemu-iotests/089.out         |  2 --
>>>   tests/qemu-iotests/141.out         |  4 +--
>>>   tests/qemu-iotests/144.out         |  2 +-
>>>   8 files changed, 32 insertions(+), 63 deletions(-)
>>>
>>> diff --git a/block.c b/block.c
>>> index 823ff1d..060f88e 100644
>>> --- a/block.c
>>> +++ b/block.c
>>> @@ -234,6 +234,8 @@ BlockDriverState *bdrv_new(void)
>>>       bs->refcnt = 1;
>>>       bs->aio_context = qemu_get_aio_context();
>>>   +    qemu_co_queue_init(&bs->flush_queue);
>>> +
>>>       QTAILQ_INSERT_TAIL(&all_bdrv_states, bs, bs_list);
>>>         return bs;
>>> @@ -2472,6 +2474,7 @@ int bdrv_truncate(BlockDriverState *bs, int64_t
>>> offset)
>>>           ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
>>>           bdrv_dirty_bitmap_truncate(bs);
>>>           bdrv_parent_cb_resize(bs);
>>> +        ++bs->write_gen;
>>>       }
>>>       return ret;
>>>   }
>>> diff --git a/block/io.c b/block/io.c
>>> index 7086908..f181ff7 100644
>>> --- a/block/io.c
>>> +++ b/block/io.c
>>> @@ -1303,6 +1303,7 @@ static int coroutine_fn
>>> bdrv_aligned_pwritev(BlockDriverState *bs,
>>>       }
>>>       bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
>>>   +    ++bs->write_gen;
>>>       bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
>>>         if (bs->wr_highest_offset < offset + bytes) {
>>> @@ -2235,6 +2236,15 @@ int coroutine_fn
>>> bdrv_co_flush(BlockDriverState *bs)
>>>         tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
>>>   +    int current_gen = bs->write_gen;
>>> +
>>> +    /* Wait until any previous flushes are completed */
>>> +    while (bs->flush_started_gen != bs->flushed_gen) {
>>> +        qemu_co_queue_wait(&bs->flush_queue);
>>> +    }
>>> +
>>> +    bs->flush_started_gen = current_gen;
>>> +
>>>       /* Write back all layers by calling one driver function */
>>>       if (bs->drv->bdrv_co_flush) {
>>>           ret = bs->drv->bdrv_co_flush(bs);
>>> @@ -2255,6 +2265,11 @@ int coroutine_fn
>>> bdrv_co_flush(BlockDriverState *bs)
>>>           goto flush_parent;
>>>       }
>>>   +    /* Check if we really need to flush anything */
>>> +    if (bs->flushed_gen == current_gen) {
>>> +        goto flush_parent;
>>> +    }
>>> +
>>>       BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
>>>       if (bs->drv->bdrv_co_flush_to_disk) {
>>>           ret = bs->drv->bdrv_co_flush_to_disk(bs);
>>> @@ -2285,6 +2300,7 @@ int coroutine_fn bdrv_co_flush(BlockDriverState
>>> *bs)
>>>            */
>>>           ret = 0;
>>>       }
>>> +
>>>       if (ret < 0) {
>>>           goto out;
>>>       }
>>> @@ -2295,6 +2311,10 @@ int coroutine_fn
>>> bdrv_co_flush(BlockDriverState *bs)
>>>   flush_parent:
>>>       ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
>>>   out:
>>> +    /* Notify any pending flushes that we have completed */
>>> +    bs->flushed_gen = current_gen;
>>> +    qemu_co_queue_restart_all(&bs->flush_queue);
>>> +
>>>       tracked_request_end(&req);
>>>       return ret;
>>>   }
>>> @@ -2420,6 +2440,7 @@ int coroutine_fn
>>> bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
>>>       }
>>>       ret = 0;
>>>   out:
>>> +    ++bs->write_gen;
>>>       bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
>>>                      req.bytes >> BDRV_SECTOR_BITS);
>>>       tracked_request_end(&req);
>>> diff --git a/include/block/block_int.h b/include/block/block_int.h
>>> index 47b9aac..396bd2b 100644
>>> --- a/include/block/block_int.h
>>> +++ b/include/block/block_int.h
>>> @@ -439,6 +439,11 @@ struct BlockDriverState {
>>>       int copy_on_read; /* if nonzero, copy read backing sectors into
>>> image.
>>>                            note this is a reference count */
>>>   +    CoQueue flush_queue;            /* Serializing flush queue */
>>> +    unsigned int write_gen;         /* Current data generation */
>>> +    unsigned int flush_started_gen; /* Generation for which flush
>>> has started */
>>> +    unsigned int flushed_gen;       /* Flushed write generation */
>>> +
>>>       BlockDriver *drv; /* NULL means no media */
>>>       void *opaque;
>>>   diff --git a/tests/qemu-iotests/026.out.nocache
>>> b/tests/qemu-iotests/026.out.nocache
>>> index 9c2c8a9..672d77c 100644
>>> --- a/tests/qemu-iotests/026.out.nocache
>>> +++ b/tests/qemu-iotests/026.out.nocache
>>> @@ -14,7 +14,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_update; errno: 5; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   @@ -23,7 +22,6 @@ This means waste of disk space, but no harm to data.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_update; errno: 5; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   @@ -42,7 +40,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_update; errno: 28; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   @@ -51,7 +48,6 @@ This means waste of disk space, but no harm to data.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_update; errno: 28; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   @@ -78,11 +74,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT
>>> size=1073741824
>>>   Event: l2_load; errno: 5; imm: off; once: off; write
>>>   wrote 131072/131072 bytes at offset 0
>>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   read failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>> @@ -90,11 +82,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT
>>> size=1073741824
>>>   Event: l2_load; errno: 5; imm: off; once: off; write -b
>>>   wrote 131072/131072 bytes at offset 0
>>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   read failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>> @@ -118,11 +106,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT
>>> size=1073741824
>>>   Event: l2_load; errno: 28; imm: off; once: off; write
>>>   wrote 131072/131072 bytes at offset 0
>>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>>> -Failed to flush the L2 table cache: No space left on device
>>> -Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>> -Failed to flush the L2 table cache: No space left on device
>>> -Failed to flush the refcount block cache: No space left on device
>>>   read failed: No space left on device
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>> @@ -130,11 +114,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT
>>> size=1073741824
>>>   Event: l2_load; errno: 28; imm: off; once: off; write -b
>>>   wrote 131072/131072 bytes at offset 0
>>>   128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>>> -Failed to flush the L2 table cache: No space left on device
>>> -Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>> -Failed to flush the L2 table cache: No space left on device
>>> -Failed to flush the refcount block cache: No space left on device
>>>   read failed: No space left on device
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>> @@ -314,14 +294,12 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_load; errno: 5; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_load; errno: 5; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>> @@ -338,14 +316,12 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_load; errno: 28; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_load; errno: 28; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>> @@ -362,14 +338,12 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_update_part; errno: 5; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_update_part; errno: 5; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>> @@ -386,14 +360,12 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_update_part; errno: 28; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_update_part; errno: 28; imm: off; once: off;
>>> write -b
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>> @@ -410,14 +382,12 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc; errno: 5; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc; errno: 5; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>> @@ -434,14 +404,12 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc; errno: 28; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc; errno: 28; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>> @@ -458,15 +426,11 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: cluster_alloc; errno: 5; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: cluster_alloc; errno: 5; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>> @@ -482,15 +446,11 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: cluster_alloc; errno: 28; imm: off; once: off; write
>>> -Failed to flush the L2 table cache: No space left on device
>>> -Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: cluster_alloc; errno: 28; imm: off; once: off; write -b
>>> -Failed to flush the L2 table cache: No space left on device
>>> -Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>>   @@ -561,7 +521,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc_write_blocks; errno: 28; imm: off; once:
>>> off; write
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   @@ -570,7 +529,6 @@ This means waste of disk space, but no harm to
>>> data.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc_write_blocks; errno: 28; imm: off; once:
>>> off; write -b
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   @@ -589,7 +547,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc_write_table; errno: 28; imm: off; once:
>>> off; write
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   @@ -598,7 +555,6 @@ This means waste of disk space, but no harm to
>>> data.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: refblock_alloc_write_table; errno: 28; imm: off; once:
>>> off; write -b
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   @@ -643,8 +599,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>> @@ -655,8 +609,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
>>> -Failed to flush the L2 table cache: No space left on device
>>> -Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>> @@ -667,7 +619,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_grow_write_table; errno: 5; imm: off; once: off
>>> -Failed to flush the L2 table cache: Input/output error
>>>   Failed to flush the refcount block cache: Input/output error
>>>   write failed: Input/output error
>>>   No errors were found on the image.
>>> @@ -679,7 +630,6 @@ No errors were found on the image.
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
>>>     Event: l1_grow_write_table; errno: 28; imm: off; once: off
>>> -Failed to flush the L2 table cache: No space left on device
>>>   Failed to flush the refcount block cache: No space left on device
>>>   write failed: No space left on device
>>>   No errors were found on the image.
>>> diff --git a/tests/qemu-iotests/071.out b/tests/qemu-iotests/071.out
>>> index 2b40ead..8c6851e 100644
>>> --- a/tests/qemu-iotests/071.out
>>> +++ b/tests/qemu-iotests/071.out
>>> @@ -30,14 +30,10 @@ blkverify: read sector_num=0 nb_sectors=1
>>> contents mismatch in sector 0
>>>     === Testing blkdebug through filename ===
>>>   -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   read failed: Input/output error
>>>     === Testing blkdebug through file blockref ===
>>>   -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   read failed: Input/output error
>>>     === Testing blkdebug on existing block device ===
>>> @@ -51,8 +47,6 @@ read failed: Input/output error
>>>   {"return": ""}
>>>   {"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "SHUTDOWN"}
>>> -QEMU_PROG: Failed to flush the L2 table cache: Input/output error
>>> -QEMU_PROG: Failed to flush the refcount block cache: Input/output error
>>>       === Testing blkverify on existing block device ===
>>> @@ -92,7 +86,5 @@ read failed: Input/output error
>>>   {"return": ""}
>>>   {"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "SHUTDOWN"}
>>> -QEMU_PROG: Failed to flush the L2 table cache: Input/output error
>>> -QEMU_PROG: Failed to flush the refcount block cache: Input/output error
>>>     *** done
>>> diff --git a/tests/qemu-iotests/089.out b/tests/qemu-iotests/089.out
>>> index 5b541a3..18f5fdd 100644
>>> --- a/tests/qemu-iotests/089.out
>>> +++ b/tests/qemu-iotests/089.out
>>> @@ -24,8 +24,6 @@ read 512/512 bytes at offset 0
>>>   Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
>>>   wrote 512/512 bytes at offset 229376
>>>   512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
>>> -Failed to flush the L2 table cache: Input/output error
>>> -Failed to flush the refcount block cache: Input/output error
>>>   read failed: Input/output error
>>>     === Testing qemu-img info output ===
>>> diff --git a/tests/qemu-iotests/141.out b/tests/qemu-iotests/141.out
>>> index adceac1..eaf1e60 100644
>>> --- a/tests/qemu-iotests/141.out
>>> +++ b/tests/qemu-iotests/141.out
>>> @@ -18,8 +18,8 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT
>>> size=1048576 backing_file=TEST_DIR/t.
>>>     {"return": {}}
>>>   Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576
>>> backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
>>> -{"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0,
>>> "offset": 0, "speed": 0, "type": "mirror"}}
>>> +{"return": {}}
>>>   {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy:
>>> block device is in use by block job: mirror"}}
>>>   {"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0,
>>> "offset": 0, "speed": 0, "type": "mirror"}}
>>> @@ -28,8 +28,8 @@ Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT
>>> size=1048576 backing_file=TEST_DIR/t.
>>>   === Testing active block-commit ===
>>>     {"return": {}}
>>> -{"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0,
>>> "offset": 0, "speed": 0, "type": "commit"}}
>>> +{"return": {}}
>>>   {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy:
>>> block device is in use by block job: commit"}}
>>>   {"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0,
>>> "offset": 0, "speed": 0, "type": "commit"}}
>>> diff --git a/tests/qemu-iotests/144.out b/tests/qemu-iotests/144.out
>>> index 410d741..387855c 100644
>>> --- a/tests/qemu-iotests/144.out
>>> +++ b/tests/qemu-iotests/144.out
>>> @@ -12,9 +12,9 @@ Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2
>>> size=536870912 backing_file=TEST_DIR/
>>>     === Performing block-commit on active layer ===
>>>   -{"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "BLOCK_JOB_READY", "data": {"device": "virtio0", "len": 0,
>>> "offset": 0, "speed": 0, "type": "commit"}}
>>>   {"return": {}}
>>> +{"return": {}}
>>>   {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP},
>>> "event": "BLOCK_JOB_COMPLETED", "data": {"device": "virtio0", "len":
>>> 0, "offset": 0, "speed": 0, "type": "commit"}}
>>>     === Performing Live Snapshot 2 ===
>>>
>> Patches 1-3: OK.
>> Patch 4 still breaks test 026...
>>
>> I will try to get this merged, but please run the entire test suite on
>> patch 4 before re-sending.
>>
>> --js
> ah.... Scheisse. The problem is that 026 has 2 results, one for cached,
> one for uncached IO. I have missed that. Give me 5 minutes.
> 
> We have fixed uncached mode only.
> 
> The problem is clear here.
> 
> Den

Tch! I didn't notice myself. Thanks for the catch.
diff mbox

Patch

diff --git a/block.c b/block.c
index 823ff1d..060f88e 100644
--- a/block.c
+++ b/block.c
@@ -234,6 +234,8 @@  BlockDriverState *bdrv_new(void)
     bs->refcnt = 1;
     bs->aio_context = qemu_get_aio_context();
 
+    qemu_co_queue_init(&bs->flush_queue);
+
     QTAILQ_INSERT_TAIL(&all_bdrv_states, bs, bs_list);
 
     return bs;
@@ -2472,6 +2474,7 @@  int bdrv_truncate(BlockDriverState *bs, int64_t offset)
         ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
         bdrv_dirty_bitmap_truncate(bs);
         bdrv_parent_cb_resize(bs);
+        ++bs->write_gen;
     }
     return ret;
 }
diff --git a/block/io.c b/block/io.c
index 7086908..f181ff7 100644
--- a/block/io.c
+++ b/block/io.c
@@ -1303,6 +1303,7 @@  static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
     }
     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
 
+    ++bs->write_gen;
     bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
 
     if (bs->wr_highest_offset < offset + bytes) {
@@ -2235,6 +2236,15 @@  int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
 
     tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
 
+    int current_gen = bs->write_gen;
+
+    /* Wait until any previous flushes are completed */
+    while (bs->flush_started_gen != bs->flushed_gen) {
+        qemu_co_queue_wait(&bs->flush_queue);
+    }
+
+    bs->flush_started_gen = current_gen;
+
     /* Write back all layers by calling one driver function */
     if (bs->drv->bdrv_co_flush) {
         ret = bs->drv->bdrv_co_flush(bs);
@@ -2255,6 +2265,11 @@  int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
         goto flush_parent;
     }
 
+    /* Check if we really need to flush anything */
+    if (bs->flushed_gen == current_gen) {
+        goto flush_parent;
+    }
+
     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
     if (bs->drv->bdrv_co_flush_to_disk) {
         ret = bs->drv->bdrv_co_flush_to_disk(bs);
@@ -2285,6 +2300,7 @@  int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
          */
         ret = 0;
     }
+
     if (ret < 0) {
         goto out;
     }
@@ -2295,6 +2311,10 @@  int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
 flush_parent:
     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
 out:
+    /* Notify any pending flushes that we have completed */
+    bs->flushed_gen = current_gen;
+    qemu_co_queue_restart_all(&bs->flush_queue);
+
     tracked_request_end(&req);
     return ret;
 }
@@ -2420,6 +2440,7 @@  int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
     }
     ret = 0;
 out:
+    ++bs->write_gen;
     bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
                    req.bytes >> BDRV_SECTOR_BITS);
     tracked_request_end(&req);
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 47b9aac..396bd2b 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -439,6 +439,11 @@  struct BlockDriverState {
     int copy_on_read; /* if nonzero, copy read backing sectors into image.
                          note this is a reference count */
 
+    CoQueue flush_queue;            /* Serializing flush queue */
+    unsigned int write_gen;         /* Current data generation */
+    unsigned int flush_started_gen; /* Generation for which flush has started */
+    unsigned int flushed_gen;       /* Flushed write generation */
+
     BlockDriver *drv; /* NULL means no media */
     void *opaque;
 
diff --git a/tests/qemu-iotests/026.out.nocache b/tests/qemu-iotests/026.out.nocache
index 9c2c8a9..672d77c 100644
--- a/tests/qemu-iotests/026.out.nocache
+++ b/tests/qemu-iotests/026.out.nocache
@@ -14,7 +14,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write 
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
@@ -23,7 +22,6 @@  This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write -b
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
@@ -42,7 +40,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write 
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
@@ -51,7 +48,6 @@  This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write -b
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
@@ -78,11 +74,7 @@  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 5; imm: off; once: off; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 read failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -90,11 +82,7 @@  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 5; imm: off; once: off; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 read failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -118,11 +106,7 @@  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 28; imm: off; once: off; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Failed to flush the L2 table cache: No space left on device
-Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
-Failed to flush the L2 table cache: No space left on device
-Failed to flush the refcount block cache: No space left on device
 read failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -130,11 +114,7 @@  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 Event: l2_load; errno: 28; imm: off; once: off; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Failed to flush the L2 table cache: No space left on device
-Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
-Failed to flush the L2 table cache: No space left on device
-Failed to flush the refcount block cache: No space left on device
 read failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -314,14 +294,12 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 5; imm: off; once: off; write 
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 5; imm: off; once: off; write -b
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
@@ -338,14 +316,12 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 28; imm: off; once: off; write 
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 28; imm: off; once: off; write -b
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
@@ -362,14 +338,12 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 5; imm: off; once: off; write 
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 5; imm: off; once: off; write -b
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
@@ -386,14 +360,12 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 28; imm: off; once: off; write 
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 28; imm: off; once: off; write -b
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
@@ -410,14 +382,12 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 5; imm: off; once: off; write 
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 5; imm: off; once: off; write -b
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
@@ -434,14 +404,12 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 28; imm: off; once: off; write 
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 28; imm: off; once: off; write -b
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
@@ -458,15 +426,11 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 5; imm: off; once: off; write 
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 5; imm: off; once: off; write -b
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -482,15 +446,11 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 28; imm: off; once: off; write 
-Failed to flush the L2 table cache: No space left on device
-Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: cluster_alloc; errno: 28; imm: off; once: off; write -b
-Failed to flush the L2 table cache: No space left on device
-Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 
@@ -561,7 +521,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write 
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
@@ -570,7 +529,6 @@  This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write -b
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
@@ -589,7 +547,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write 
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
@@ -598,7 +555,6 @@  This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write -b
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
@@ -643,8 +599,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -655,8 +609,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
-Failed to flush the L2 table cache: No space left on device
-Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -667,7 +619,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_grow_write_table; errno: 5; imm: off; once: off
-Failed to flush the L2 table cache: Input/output error
 Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
@@ -679,7 +630,6 @@  No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_grow_write_table; errno: 28; imm: off; once: off
-Failed to flush the L2 table cache: No space left on device
 Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
diff --git a/tests/qemu-iotests/071.out b/tests/qemu-iotests/071.out
index 2b40ead..8c6851e 100644
--- a/tests/qemu-iotests/071.out
+++ b/tests/qemu-iotests/071.out
@@ -30,14 +30,10 @@  blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 === Testing blkdebug through filename ===
 
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 read failed: Input/output error
 
 === Testing blkdebug through file blockref ===
 
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 read failed: Input/output error
 
 === Testing blkdebug on existing block device ===
@@ -51,8 +47,6 @@  read failed: Input/output error
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-QEMU_PROG: Failed to flush the L2 table cache: Input/output error
-QEMU_PROG: Failed to flush the refcount block cache: Input/output error
 
 
 === Testing blkverify on existing block device ===
@@ -92,7 +86,5 @@  read failed: Input/output error
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-QEMU_PROG: Failed to flush the L2 table cache: Input/output error
-QEMU_PROG: Failed to flush the refcount block cache: Input/output error
 
 *** done
diff --git a/tests/qemu-iotests/089.out b/tests/qemu-iotests/089.out
index 5b541a3..18f5fdd 100644
--- a/tests/qemu-iotests/089.out
+++ b/tests/qemu-iotests/089.out
@@ -24,8 +24,6 @@  read 512/512 bytes at offset 0
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 512/512 bytes at offset 229376
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Failed to flush the L2 table cache: Input/output error
-Failed to flush the refcount block cache: Input/output error
 read failed: Input/output error
 
 === Testing qemu-img info output ===
diff --git a/tests/qemu-iotests/141.out b/tests/qemu-iotests/141.out
index adceac1..eaf1e60 100644
--- a/tests/qemu-iotests/141.out
+++ b/tests/qemu-iotests/141.out
@@ -18,8 +18,8 @@  Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.
 
 {"return": {}}
 Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
-{"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
+{"return": {}}
 {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: mirror"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
@@ -28,8 +28,8 @@  Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.
 === Testing active block-commit ===
 
 {"return": {}}
-{"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"return": {}}
 {"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
diff --git a/tests/qemu-iotests/144.out b/tests/qemu-iotests/144.out
index 410d741..387855c 100644
--- a/tests/qemu-iotests/144.out
+++ b/tests/qemu-iotests/144.out
@@ -12,9 +12,9 @@  Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/
 
 === Performing block-commit on active layer ===
 
-{"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
 {"return": {}}
+{"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
 
 === Performing Live Snapshot 2 ===