diff mbox series

[2/3] generic: test shutdowns of a nested filesystem

Message ID 162743103024.3428896.8525632218517299015.stgit@magnolia (mailing list archive)
State New, archived
Headers show
Series fstests: exercise code refactored in 5.14 | expand

Commit Message

Darrick J. Wong July 28, 2021, 12:10 a.m. UTC
From: Darrick J. Wong <djwong@kernel.org>

generic/475, but we're running fsstress on a disk image inside the
scratch filesystem

Signed-off-by: Darrick J. Wong <djwong@kernel.org>
---
 tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
 tests/generic/725.out |    2 +
 2 files changed, 138 insertions(+)
 create mode 100755 tests/generic/725
 create mode 100644 tests/generic/725.out

Comments

Zorro Lang Aug. 12, 2021, 5:44 a.m. UTC | #1
On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <djwong@kernel.org>
> 
> generic/475, but we're running fsstress on a disk image inside the
> scratch filesystem
> 
> Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> ---
>  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/generic/725.out |    2 +
>  2 files changed, 138 insertions(+)
>  create mode 100755 tests/generic/725
>  create mode 100644 tests/generic/725.out
> 
> 
> diff --git a/tests/generic/725 b/tests/generic/725
> new file mode 100755
> index 00000000..f43bcb37
> --- /dev/null
> +++ b/tests/generic/725
> @@ -0,0 +1,136 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> +#
> +# FS QA Test No. 725
> +#
> +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> +# out the underlying scratch device with dm-error to see what happens when the
> +# disk goes down.  Having taken down both fses in this manner, remount them and
> +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> +# in writeback on the host that cause VM guests to fail to recover.
> +#
> +. ./common/preamble
> +_begin_fstest shutdown auto log metadata eio
> +
> +_cleanup()
> +{
> +	cd /
> +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> +	wait
> +	if [ -n "$loopmnt" ]; then
> +		umount $loopmnt 2>/dev/null
> +		rm -r -f $loopmnt
> +	fi
> +	rm -f $tmp.*
> +	_dmerror_unmount
> +	_dmerror_cleanup
> +}
> +
> +# Import common functions.
> +. ./common/dmerror
> +. ./common/reflink
> +
> +# Modify as appropriate.
> +_supported_fs generic
> +
> +_require_scratch_reflink
> +_require_cp_reflink
> +_require_dm_target error
> +_require_command "$KILLALL_PROG" "killall"
> +
> +echo "Silence is golden."
> +
> +_scratch_mkfs >> $seqres.full 2>&1
> +_require_metadata_journaling $SCRATCH_DEV
> +_dmerror_init
> +_dmerror_mount
> +
> +# Create a fs image consuming 1/3 of the scratch fs
> +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
> +loopimg_bytes=$((scratch_freesp_bytes / 3))
> +
> +loopimg=$SCRATCH_MNT/testfs
> +truncate -s $loopimg_bytes $loopimg
> +_mkfs_dev $loopimg

I must say this's a nice test as generic/475, I'd like to have it ASAP :)
Just one question: if the FSTYP is nfs, cifs or virtiofs and so on ... [see below]

> +
> +loopmnt=$tmp.mount
> +mkdir -p $loopmnt
> +
> +scratch_aliveflag=$tmp.runsnap
> +snap_aliveflag=$tmp.snapping
> +
> +snap_loop_fs() {
> +	touch "$snap_aliveflag"
> +	while [ -e "$scratch_aliveflag" ]; do
> +		rm -f $loopimg.a
> +		_cp_reflink $loopimg $loopimg.a
> +		sleep 1
> +	done
> +	rm -f "$snap_aliveflag"
> +}
> +
> +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> +
> +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> +	touch $scratch_aliveflag
> +	snap_loop_fs >> $seqres.full 2>&1 &
> +
> +	if ! _mount $loopimg $loopmnt -o loop; then

... This test will fail directly at here

Thanks,
Zorro

> +		rm -f $scratch_aliveflag
> +		_fail "loop mount failed"
> +		break
> +	fi
> +
> +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> +
> +	# purposely include 0 second sleeps to test shutdown immediately after
> +	# recovery
> +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> +	rm -f $scratch_aliveflag
> +
> +	# This test aims to simulate sudden disk failure, which means that we
> +	# do not want to quiesce the filesystem or otherwise give it a chance
> +	# to flush its logs.  Therefore we want to call dmsetup with the
> +	# --nolockfs parameter; to make this happen we must call the load
> +	# error table helper *without* 'lockfs'.
> +	_dmerror_load_error_table
> +
> +	ps -e | grep fsstress > /dev/null 2>&1
> +	while [ $? -eq 0 ]; do
> +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> +		wait > /dev/null 2>&1
> +		ps -e | grep fsstress > /dev/null 2>&1
> +	done
> +	for ((i = 0; i < 10; i++)); do
> +		test -e "$snap_aliveflag" || break
> +		sleep 1
> +	done
> +
> +	# Mount again to replay log after loading working table, so we have a
> +	# consistent XFS after test.
> +	$UMOUNT_PROG $loopmnt
> +	_dmerror_unmount || _fail "unmount failed"
> +	_dmerror_load_working_table
> +	if ! _dmerror_mount; then
> +		dmsetup table | tee -a /dev/ttyprintk
> +		lsblk | tee -a /dev/ttyprintk
> +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
> +		_fail "mount failed"
> +	fi
> +done
> +
> +# Make sure the fs image file is ok
> +if [ -f "$loopimg" ]; then
> +	if _mount $loopimg $loopmnt -o loop; then
> +		$UMOUNT_PROG $loopmnt &> /dev/null
> +	else
> +		echo "final loop mount failed"
> +	fi
> +	_check_xfs_filesystem $loopimg none none
> +fi
> +
> +# success, all done
> +status=0
> +exit
> diff --git a/tests/generic/725.out b/tests/generic/725.out
> new file mode 100644
> index 00000000..ed73a9fc
> --- /dev/null
> +++ b/tests/generic/725.out
> @@ -0,0 +1,2 @@
> +QA output created by 725
> +Silence is golden.
>
Darrick J. Wong Aug. 12, 2021, 5:07 p.m. UTC | #2
On Thu, Aug 12, 2021 at 01:44:21PM +0800, Zorro Lang wrote:
> On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> > From: Darrick J. Wong <djwong@kernel.org>
> > 
> > generic/475, but we're running fsstress on a disk image inside the
> > scratch filesystem
> > 
> > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > ---
> >  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/generic/725.out |    2 +
> >  2 files changed, 138 insertions(+)
> >  create mode 100755 tests/generic/725
> >  create mode 100644 tests/generic/725.out
> > 
> > 
> > diff --git a/tests/generic/725 b/tests/generic/725
> > new file mode 100755
> > index 00000000..f43bcb37
> > --- /dev/null
> > +++ b/tests/generic/725
> > @@ -0,0 +1,136 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0
> > +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 725
> > +#
> > +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> > +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> > +# out the underlying scratch device with dm-error to see what happens when the
> > +# disk goes down.  Having taken down both fses in this manner, remount them and
> > +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> > +# in writeback on the host that cause VM guests to fail to recover.
> > +#
> > +. ./common/preamble
> > +_begin_fstest shutdown auto log metadata eio
> > +
> > +_cleanup()
> > +{
> > +	cd /
> > +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > +	wait
> > +	if [ -n "$loopmnt" ]; then
> > +		umount $loopmnt 2>/dev/null
> > +		rm -r -f $loopmnt
> > +	fi
> > +	rm -f $tmp.*
> > +	_dmerror_unmount
> > +	_dmerror_cleanup
> > +}
> > +
> > +# Import common functions.
> > +. ./common/dmerror
> > +. ./common/reflink
> > +
> > +# Modify as appropriate.
> > +_supported_fs generic
> > +
> > +_require_scratch_reflink
> > +_require_cp_reflink
> > +_require_dm_target error
> > +_require_command "$KILLALL_PROG" "killall"
> > +
> > +echo "Silence is golden."
> > +
> > +_scratch_mkfs >> $seqres.full 2>&1
> > +_require_metadata_journaling $SCRATCH_DEV
> > +_dmerror_init
> > +_dmerror_mount
> > +
> > +# Create a fs image consuming 1/3 of the scratch fs
> > +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
> > +loopimg_bytes=$((scratch_freesp_bytes / 3))
> > +
> > +loopimg=$SCRATCH_MNT/testfs
> > +truncate -s $loopimg_bytes $loopimg
> > +_mkfs_dev $loopimg
> 
> I must say this's a nice test as generic/475, I'd like to have it ASAP :)
> Just one question: if the FSTYP is nfs, cifs or virtiofs and so on ... [see below]
> 
> > +
> > +loopmnt=$tmp.mount
> > +mkdir -p $loopmnt
> > +
> > +scratch_aliveflag=$tmp.runsnap
> > +snap_aliveflag=$tmp.snapping
> > +
> > +snap_loop_fs() {
> > +	touch "$snap_aliveflag"
> > +	while [ -e "$scratch_aliveflag" ]; do
> > +		rm -f $loopimg.a
> > +		_cp_reflink $loopimg $loopimg.a
> > +		sleep 1
> > +	done
> > +	rm -f "$snap_aliveflag"
> > +}
> > +
> > +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> > +
> > +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> > +	touch $scratch_aliveflag
> > +	snap_loop_fs >> $seqres.full 2>&1 &
> > +
> > +	if ! _mount $loopimg $loopmnt -o loop; then
> 
> ... This test will fail directly at here

It won't, because this test doesn't run if SCRATCH_DEV isn't a block
device.  _require_dm_target calls _require_block_device, which should
prevent that, right?

--D

> 
> Thanks,
> Zorro
> 
> > +		rm -f $scratch_aliveflag
> > +		_fail "loop mount failed"
> > +		break
> > +	fi
> > +
> > +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> > +
> > +	# purposely include 0 second sleeps to test shutdown immediately after
> > +	# recovery
> > +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> > +	rm -f $scratch_aliveflag
> > +
> > +	# This test aims to simulate sudden disk failure, which means that we
> > +	# do not want to quiesce the filesystem or otherwise give it a chance
> > +	# to flush its logs.  Therefore we want to call dmsetup with the
> > +	# --nolockfs parameter; to make this happen we must call the load
> > +	# error table helper *without* 'lockfs'.
> > +	_dmerror_load_error_table
> > +
> > +	ps -e | grep fsstress > /dev/null 2>&1
> > +	while [ $? -eq 0 ]; do
> > +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > +		wait > /dev/null 2>&1
> > +		ps -e | grep fsstress > /dev/null 2>&1
> > +	done
> > +	for ((i = 0; i < 10; i++)); do
> > +		test -e "$snap_aliveflag" || break
> > +		sleep 1
> > +	done
> > +
> > +	# Mount again to replay log after loading working table, so we have a
> > +	# consistent XFS after test.
> > +	$UMOUNT_PROG $loopmnt
> > +	_dmerror_unmount || _fail "unmount failed"
> > +	_dmerror_load_working_table
> > +	if ! _dmerror_mount; then
> > +		dmsetup table | tee -a /dev/ttyprintk
> > +		lsblk | tee -a /dev/ttyprintk
> > +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
> > +		_fail "mount failed"
> > +	fi
> > +done
> > +
> > +# Make sure the fs image file is ok
> > +if [ -f "$loopimg" ]; then
> > +	if _mount $loopimg $loopmnt -o loop; then
> > +		$UMOUNT_PROG $loopmnt &> /dev/null
> > +	else
> > +		echo "final loop mount failed"
> > +	fi
> > +	_check_xfs_filesystem $loopimg none none
> > +fi
> > +
> > +# success, all done
> > +status=0
> > +exit
> > diff --git a/tests/generic/725.out b/tests/generic/725.out
> > new file mode 100644
> > index 00000000..ed73a9fc
> > --- /dev/null
> > +++ b/tests/generic/725.out
> > @@ -0,0 +1,2 @@
> > +QA output created by 725
> > +Silence is golden.
> > 
>
Zorro Lang Aug. 13, 2021, 2:52 p.m. UTC | #3
On Thu, Aug 12, 2021 at 10:07:46AM -0700, Darrick J. Wong wrote:
> On Thu, Aug 12, 2021 at 01:44:21PM +0800, Zorro Lang wrote:
> > On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> > > From: Darrick J. Wong <djwong@kernel.org>
> > > 
> > > generic/475, but we're running fsstress on a disk image inside the
> > > scratch filesystem
> > > 
> > > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > > ---
> > >  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
> > >  tests/generic/725.out |    2 +
> > >  2 files changed, 138 insertions(+)
> > >  create mode 100755 tests/generic/725
> > >  create mode 100644 tests/generic/725.out
> > > 
> > > 
> > > diff --git a/tests/generic/725 b/tests/generic/725
> > > new file mode 100755
> > > index 00000000..f43bcb37
> > > --- /dev/null
> > > +++ b/tests/generic/725
> > > @@ -0,0 +1,136 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0
> > > +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 725
> > > +#
> > > +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> > > +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> > > +# out the underlying scratch device with dm-error to see what happens when the
> > > +# disk goes down.  Having taken down both fses in this manner, remount them and
> > > +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> > > +# in writeback on the host that cause VM guests to fail to recover.
> > > +#
> > > +. ./common/preamble
> > > +_begin_fstest shutdown auto log metadata eio
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > +	wait
> > > +	if [ -n "$loopmnt" ]; then
> > > +		umount $loopmnt 2>/dev/null
> > > +		rm -r -f $loopmnt
> > > +	fi
> > > +	rm -f $tmp.*
> > > +	_dmerror_unmount
> > > +	_dmerror_cleanup
> > > +}
> > > +
> > > +# Import common functions.
> > > +. ./common/dmerror
> > > +. ./common/reflink
> > > +
> > > +# Modify as appropriate.
> > > +_supported_fs generic
> > > +
> > > +_require_scratch_reflink
> > > +_require_cp_reflink
> > > +_require_dm_target error
> > > +_require_command "$KILLALL_PROG" "killall"
> > > +
> > > +echo "Silence is golden."
> > > +
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_require_metadata_journaling $SCRATCH_DEV
> > > +_dmerror_init
> > > +_dmerror_mount
> > > +
> > > +# Create a fs image consuming 1/3 of the scratch fs
> > > +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
> > > +loopimg_bytes=$((scratch_freesp_bytes / 3))
> > > +
> > > +loopimg=$SCRATCH_MNT/testfs
> > > +truncate -s $loopimg_bytes $loopimg
> > > +_mkfs_dev $loopimg
> > 
> > I must say this's a nice test as generic/475, I'd like to have it ASAP :)
> > Just one question: if the FSTYP is nfs, cifs or virtiofs and so on ... [see below]
> > 
> > > +
> > > +loopmnt=$tmp.mount
> > > +mkdir -p $loopmnt
> > > +
> > > +scratch_aliveflag=$tmp.runsnap
> > > +snap_aliveflag=$tmp.snapping
> > > +
> > > +snap_loop_fs() {
> > > +	touch "$snap_aliveflag"
> > > +	while [ -e "$scratch_aliveflag" ]; do
> > > +		rm -f $loopimg.a
> > > +		_cp_reflink $loopimg $loopimg.a
> > > +		sleep 1
> > > +	done
> > > +	rm -f "$snap_aliveflag"
> > > +}
> > > +
> > > +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> > > +
> > > +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> > > +	touch $scratch_aliveflag
> > > +	snap_loop_fs >> $seqres.full 2>&1 &
> > > +
> > > +	if ! _mount $loopimg $loopmnt -o loop; then
> > 
> > ... This test will fail directly at here
> 
> It won't, because this test doesn't run if SCRATCH_DEV isn't a block
> device.  _require_dm_target calls _require_block_device, which should
> prevent that, right?

Oh, you're right[1], I forgot that. If so, this case is good to me.
Hope it get merged soon :)
Reviewed-by: Zorro Lang <zlang@redhat.com>

Thanks,
Zorro

[1]
# ./check generic/725
FSTYP         -- nfs
PLATFORM      -- Linux/x86_64 xx-xxx-xx 4.18.0-xxx.el8.x86_64+debug #1 SMP Wed Jul 14 12:35:49 EDT 2021
MKFS_OPTIONS  -- xxx-xxx-xxx-xxxxxxx:/mnt/scratch/nfs-server
MOUNT_OPTIONS -- -o context=system_u:object_r:root_t:s0 xx-xxxx-xxxx.xxxxx.xx:/mnt/scratch/nfs-server /mnt/nfs-scratch

generic/725     [not run] require xx-xxxx-xxxx.xxxxx.xx:/mnt/scratch/nfs-server to be valid block disk
Ran: generic/725
Not run: generic/725
Passed all 1 tests

# ./check generic/725
FSTYP         -- glusterfs
PLATFORM      -- Linux/x86_64 xx-xxx-xx 4.18.0-xxx.el8.x86_64+debug #1 SMP Wed Jul 14 12:35:49 EDT 2021
MKFS_OPTIONS  -- xxx-xxx-xxx-xxxxxxx:/SCRATCH_VOL
MOUNT_OPTIONS -- -o context=system_u:object_r:root_t:s0 xx-xxxx-xxxx.xxxxx.xx:/SCRATCH_VOL /mnt/gluster-scratch

generic/725     [not run] Reflink not supported by scratch filesystem type: glusterfs
Ran: generic/725
Not run: generic/725
Passed all 1 tests

> 
> --D
> 
> > 
> > Thanks,
> > Zorro
> > 
> > > +		rm -f $scratch_aliveflag
> > > +		_fail "loop mount failed"
> > > +		break
> > > +	fi
> > > +
> > > +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> > > +
> > > +	# purposely include 0 second sleeps to test shutdown immediately after
> > > +	# recovery
> > > +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> > > +	rm -f $scratch_aliveflag
> > > +
> > > +	# This test aims to simulate sudden disk failure, which means that we
> > > +	# do not want to quiesce the filesystem or otherwise give it a chance
> > > +	# to flush its logs.  Therefore we want to call dmsetup with the
> > > +	# --nolockfs parameter; to make this happen we must call the load
> > > +	# error table helper *without* 'lockfs'.
> > > +	_dmerror_load_error_table
> > > +
> > > +	ps -e | grep fsstress > /dev/null 2>&1
> > > +	while [ $? -eq 0 ]; do
> > > +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > +		wait > /dev/null 2>&1
> > > +		ps -e | grep fsstress > /dev/null 2>&1
> > > +	done
> > > +	for ((i = 0; i < 10; i++)); do
> > > +		test -e "$snap_aliveflag" || break
> > > +		sleep 1
> > > +	done
> > > +
> > > +	# Mount again to replay log after loading working table, so we have a
> > > +	# consistent XFS after test.
> > > +	$UMOUNT_PROG $loopmnt
> > > +	_dmerror_unmount || _fail "unmount failed"
> > > +	_dmerror_load_working_table
> > > +	if ! _dmerror_mount; then
> > > +		dmsetup table | tee -a /dev/ttyprintk
> > > +		lsblk | tee -a /dev/ttyprintk
> > > +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
> > > +		_fail "mount failed"
> > > +	fi
> > > +done
> > > +
> > > +# Make sure the fs image file is ok
> > > +if [ -f "$loopimg" ]; then
> > > +	if _mount $loopimg $loopmnt -o loop; then
> > > +		$UMOUNT_PROG $loopmnt &> /dev/null
> > > +	else
> > > +		echo "final loop mount failed"
> > > +	fi
> > > +	_check_xfs_filesystem $loopimg none none
> > > +fi
> > > +
> > > +# success, all done
> > > +status=0
> > > +exit
> > > diff --git a/tests/generic/725.out b/tests/generic/725.out
> > > new file mode 100644
> > > index 00000000..ed73a9fc
> > > --- /dev/null
> > > +++ b/tests/generic/725.out
> > > @@ -0,0 +1,2 @@
> > > +QA output created by 725
> > > +Silence is golden.
> > > 
> > 
>
Eryu Guan Aug. 15, 2021, 4:28 p.m. UTC | #4
On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> From: Darrick J. Wong <djwong@kernel.org>
> 
> generic/475, but we're running fsstress on a disk image inside the
> scratch filesystem
> 
> Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> ---
>  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/generic/725.out |    2 +
>  2 files changed, 138 insertions(+)
>  create mode 100755 tests/generic/725
>  create mode 100644 tests/generic/725.out
> 
> 
> diff --git a/tests/generic/725 b/tests/generic/725
> new file mode 100755
> index 00000000..f43bcb37
> --- /dev/null
> +++ b/tests/generic/725
> @@ -0,0 +1,136 @@
> +#! /bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> +#
> +# FS QA Test No. 725
> +#
> +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> +# out the underlying scratch device with dm-error to see what happens when the
> +# disk goes down.  Having taken down both fses in this manner, remount them and
> +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> +# in writeback on the host that cause VM guests to fail to recover.

It currently fails for me on btrfs, the loop mount failed in 2nd
iteration, seems like a bug in btrfs.

> +#
> +. ./common/preamble
> +_begin_fstest shutdown auto log metadata eio
> +
> +_cleanup()
> +{
> +	cd /
> +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> +	wait
> +	if [ -n "$loopmnt" ]; then
> +		umount $loopmnt 2>/dev/null

$UMOUNT_PROG

> +		rm -r -f $loopmnt
> +	fi
> +	rm -f $tmp.*
> +	_dmerror_unmount
> +	_dmerror_cleanup
> +}
> +
> +# Import common functions.
> +. ./common/dmerror
> +. ./common/reflink
> +
> +# Modify as appropriate.
> +_supported_fs generic
> +
> +_require_scratch_reflink
> +_require_cp_reflink
> +_require_dm_target error
> +_require_command "$KILLALL_PROG" "killall"
> +
> +echo "Silence is golden."
> +
> +_scratch_mkfs >> $seqres.full 2>&1
> +_require_metadata_journaling $SCRATCH_DEV
> +_dmerror_init
> +_dmerror_mount
> +
> +# Create a fs image consuming 1/3 of the scratch fs
> +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)

_get_available_space $SCRATCH_MNT ?

> +loopimg_bytes=$((scratch_freesp_bytes / 3))
> +
> +loopimg=$SCRATCH_MNT/testfs
> +truncate -s $loopimg_bytes $loopimg
> +_mkfs_dev $loopimg
> +
> +loopmnt=$tmp.mount
> +mkdir -p $loopmnt
> +
> +scratch_aliveflag=$tmp.runsnap
> +snap_aliveflag=$tmp.snapping
> +
> +snap_loop_fs() {
> +	touch "$snap_aliveflag"
> +	while [ -e "$scratch_aliveflag" ]; do
> +		rm -f $loopimg.a
> +		_cp_reflink $loopimg $loopimg.a
> +		sleep 1
> +	done
> +	rm -f "$snap_aliveflag"
> +}
> +
> +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> +
> +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> +	touch $scratch_aliveflag
> +	snap_loop_fs >> $seqres.full 2>&1 &
> +
> +	if ! _mount $loopimg $loopmnt -o loop; then
> +		rm -f $scratch_aliveflag
> +		_fail "loop mount failed"

I found it a bit easier to debug if print $i here.

> +		break
> +	fi
> +
> +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> +
> +	# purposely include 0 second sleeps to test shutdown immediately after
> +	# recovery
> +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> +	rm -f $scratch_aliveflag
> +
> +	# This test aims to simulate sudden disk failure, which means that we
> +	# do not want to quiesce the filesystem or otherwise give it a chance
> +	# to flush its logs.  Therefore we want to call dmsetup with the
> +	# --nolockfs parameter; to make this happen we must call the load
> +	# error table helper *without* 'lockfs'.
> +	_dmerror_load_error_table
> +
> +	ps -e | grep fsstress > /dev/null 2>&1
> +	while [ $? -eq 0 ]; do
> +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> +		wait > /dev/null 2>&1
> +		ps -e | grep fsstress > /dev/null 2>&1
> +	done
> +	for ((i = 0; i < 10; i++)); do
> +		test -e "$snap_aliveflag" || break
> +		sleep 1
> +	done
> +
> +	# Mount again to replay log after loading working table, so we have a
> +	# consistent XFS after test.

This is a generic test, fix the XFS specific comments?

> +	$UMOUNT_PROG $loopmnt
> +	_dmerror_unmount || _fail "unmount failed"
> +	_dmerror_load_working_table
> +	if ! _dmerror_mount; then
> +		dmsetup table | tee -a /dev/ttyprintk
> +		lsblk | tee -a /dev/ttyprintk
> +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md

Above logs all should go to $seqres.full ?

And $XFS_METADUMP_PROG is not suitable for a generic test.

> +		_fail "mount failed"
> +	fi
> +done
> +
> +# Make sure the fs image file is ok
> +if [ -f "$loopimg" ]; then
> +	if _mount $loopimg $loopmnt -o loop; then
> +		$UMOUNT_PROG $loopmnt &> /dev/null
> +	else
> +		echo "final loop mount failed"
> +	fi
> +	_check_xfs_filesystem $loopimg none none

Same here, use _check_scratch_fs?

Thanks,
Eryu

> +fi
> +
> +# success, all done
> +status=0
> +exit
> diff --git a/tests/generic/725.out b/tests/generic/725.out
> new file mode 100644
> index 00000000..ed73a9fc
> --- /dev/null
> +++ b/tests/generic/725.out
> @@ -0,0 +1,2 @@
> +QA output created by 725
> +Silence is golden.
Darrick J. Wong Aug. 16, 2021, 4:35 p.m. UTC | #5
On Mon, Aug 16, 2021 at 12:28:20AM +0800, Eryu Guan wrote:
> On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> > From: Darrick J. Wong <djwong@kernel.org>
> > 
> > generic/475, but we're running fsstress on a disk image inside the
> > scratch filesystem
> > 
> > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > ---
> >  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
> >  tests/generic/725.out |    2 +
> >  2 files changed, 138 insertions(+)
> >  create mode 100755 tests/generic/725
> >  create mode 100644 tests/generic/725.out
> > 
> > 
> > diff --git a/tests/generic/725 b/tests/generic/725
> > new file mode 100755
> > index 00000000..f43bcb37
> > --- /dev/null
> > +++ b/tests/generic/725
> > @@ -0,0 +1,136 @@
> > +#! /bin/bash
> > +# SPDX-License-Identifier: GPL-2.0
> > +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> > +#
> > +# FS QA Test No. 725
> > +#
> > +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> > +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> > +# out the underlying scratch device with dm-error to see what happens when the
> > +# disk goes down.  Having taken down both fses in this manner, remount them and
> > +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> > +# in writeback on the host that cause VM guests to fail to recover.
> 
> It currently fails for me on btrfs, the loop mount failed in 2nd
> iteration, seems like a bug in btrfs.

Yep.  Until recently (aka the Big Xfs Log Recovery Bughunt of 2021) it
wouldn't pass xfs either. :/

> > +#
> > +. ./common/preamble
> > +_begin_fstest shutdown auto log metadata eio
> > +
> > +_cleanup()
> > +{
> > +	cd /
> > +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > +	wait
> > +	if [ -n "$loopmnt" ]; then
> > +		umount $loopmnt 2>/dev/null
> 
> $UMOUNT_PROG
> 
> > +		rm -r -f $loopmnt
> > +	fi
> > +	rm -f $tmp.*
> > +	_dmerror_unmount
> > +	_dmerror_cleanup
> > +}
> > +
> > +# Import common functions.
> > +. ./common/dmerror
> > +. ./common/reflink
> > +
> > +# Modify as appropriate.
> > +_supported_fs generic
> > +
> > +_require_scratch_reflink
> > +_require_cp_reflink
> > +_require_dm_target error
> > +_require_command "$KILLALL_PROG" "killall"
> > +
> > +echo "Silence is golden."
> > +
> > +_scratch_mkfs >> $seqres.full 2>&1
> > +_require_metadata_journaling $SCRATCH_DEV
> > +_dmerror_init
> > +_dmerror_mount
> > +
> > +# Create a fs image consuming 1/3 of the scratch fs
> > +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
> 
> _get_available_space $SCRATCH_MNT ?
> 
> > +loopimg_bytes=$((scratch_freesp_bytes / 3))
> > +
> > +loopimg=$SCRATCH_MNT/testfs
> > +truncate -s $loopimg_bytes $loopimg
> > +_mkfs_dev $loopimg
> > +
> > +loopmnt=$tmp.mount
> > +mkdir -p $loopmnt
> > +
> > +scratch_aliveflag=$tmp.runsnap
> > +snap_aliveflag=$tmp.snapping
> > +
> > +snap_loop_fs() {
> > +	touch "$snap_aliveflag"
> > +	while [ -e "$scratch_aliveflag" ]; do
> > +		rm -f $loopimg.a
> > +		_cp_reflink $loopimg $loopimg.a
> > +		sleep 1
> > +	done
> > +	rm -f "$snap_aliveflag"
> > +}
> > +
> > +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> > +
> > +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> > +	touch $scratch_aliveflag
> > +	snap_loop_fs >> $seqres.full 2>&1 &
> > +
> > +	if ! _mount $loopimg $loopmnt -o loop; then
> > +		rm -f $scratch_aliveflag
> > +		_fail "loop mount failed"
> 
> I found it a bit easier to debug if print $i here.

Ok, I'll change it to "loop $i mount failed".

> > +		break
> > +	fi
> > +
> > +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> > +
> > +	# purposely include 0 second sleeps to test shutdown immediately after
> > +	# recovery
> > +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> > +	rm -f $scratch_aliveflag
> > +
> > +	# This test aims to simulate sudden disk failure, which means that we
> > +	# do not want to quiesce the filesystem or otherwise give it a chance
> > +	# to flush its logs.  Therefore we want to call dmsetup with the
> > +	# --nolockfs parameter; to make this happen we must call the load
> > +	# error table helper *without* 'lockfs'.
> > +	_dmerror_load_error_table
> > +
> > +	ps -e | grep fsstress > /dev/null 2>&1
> > +	while [ $? -eq 0 ]; do
> > +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > +		wait > /dev/null 2>&1
> > +		ps -e | grep fsstress > /dev/null 2>&1
> > +	done
> > +	for ((i = 0; i < 10; i++)); do
> > +		test -e "$snap_aliveflag" || break
> > +		sleep 1
> > +	done
> > +
> > +	# Mount again to replay log after loading working table, so we have a
> > +	# consistent XFS after test.
> 
> This is a generic test, fix the XFS specific comments?

Oops.  "...a consistent fs after test."

> > +	$UMOUNT_PROG $loopmnt
> > +	_dmerror_unmount || _fail "unmount failed"
> > +	_dmerror_load_working_table
> > +	if ! _dmerror_mount; then
> > +		dmsetup table | tee -a /dev/ttyprintk
> > +		lsblk | tee -a /dev/ttyprintk
> > +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
> 
> Above logs all should go to $seqres.full ?

Oops, yeah.  I'll remove them since I was only using them to check the
system state.

> And $XFS_METADUMP_PROG is not suitable for a generic test.

I'll create _metadump_dev so that this at least works for the two
filesystems for which we have dump creation helpers (ext* and xfs).

> > +		_fail "mount failed"
> > +	fi
> > +done
> > +
> > +# Make sure the fs image file is ok
> > +if [ -f "$loopimg" ]; then
> > +	if _mount $loopimg $loopmnt -o loop; then
> > +		$UMOUNT_PROG $loopmnt &> /dev/null
> > +	else
> > +		echo "final loop mount failed"
> > +	fi
> > +	_check_xfs_filesystem $loopimg none none
> 
> Same here, use _check_scratch_fs?

$loopimg is a file within the scratch fs.

--D

> Thanks,
> Eryu
> 
> > +fi
> > +
> > +# success, all done
> > +status=0
> > +exit
> > diff --git a/tests/generic/725.out b/tests/generic/725.out
> > new file mode 100644
> > index 00000000..ed73a9fc
> > --- /dev/null
> > +++ b/tests/generic/725.out
> > @@ -0,0 +1,2 @@
> > +QA output created by 725
> > +Silence is golden.
Eryu Guan Aug. 17, 2021, 3:16 a.m. UTC | #6
On Mon, Aug 16, 2021 at 09:35:24AM -0700, Darrick J. Wong wrote:
> On Mon, Aug 16, 2021 at 12:28:20AM +0800, Eryu Guan wrote:
> > On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> > > From: Darrick J. Wong <djwong@kernel.org>
> > > 
> > > generic/475, but we're running fsstress on a disk image inside the
> > > scratch filesystem
> > > 
> > > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > > ---
> > >  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
> > >  tests/generic/725.out |    2 +
> > >  2 files changed, 138 insertions(+)
> > >  create mode 100755 tests/generic/725
> > >  create mode 100644 tests/generic/725.out
> > > 
> > > 
> > > diff --git a/tests/generic/725 b/tests/generic/725
> > > new file mode 100755
> > > index 00000000..f43bcb37
> > > --- /dev/null
> > > +++ b/tests/generic/725
> > > @@ -0,0 +1,136 @@
> > > +#! /bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0
> > > +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> > > +#
> > > +# FS QA Test No. 725
> > > +#
> > > +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> > > +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> > > +# out the underlying scratch device with dm-error to see what happens when the
> > > +# disk goes down.  Having taken down both fses in this manner, remount them and
> > > +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> > > +# in writeback on the host that cause VM guests to fail to recover.
> > 
> > It currently fails for me on btrfs, the loop mount failed in 2nd
> > iteration, seems like a bug in btrfs.
> 
> Yep.  Until recently (aka the Big Xfs Log Recovery Bughunt of 2021) it
> wouldn't pass xfs either. :/
> 
> > > +#
> > > +. ./common/preamble
> > > +_begin_fstest shutdown auto log metadata eio
> > > +
> > > +_cleanup()
> > > +{
> > > +	cd /
> > > +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > +	wait
> > > +	if [ -n "$loopmnt" ]; then
> > > +		umount $loopmnt 2>/dev/null
> > 
> > $UMOUNT_PROG
> > 
> > > +		rm -r -f $loopmnt
> > > +	fi
> > > +	rm -f $tmp.*
> > > +	_dmerror_unmount
> > > +	_dmerror_cleanup
> > > +}
> > > +
> > > +# Import common functions.
> > > +. ./common/dmerror
> > > +. ./common/reflink
> > > +
> > > +# Modify as appropriate.
> > > +_supported_fs generic
> > > +
> > > +_require_scratch_reflink
> > > +_require_cp_reflink
> > > +_require_dm_target error
> > > +_require_command "$KILLALL_PROG" "killall"
> > > +
> > > +echo "Silence is golden."
> > > +
> > > +_scratch_mkfs >> $seqres.full 2>&1
> > > +_require_metadata_journaling $SCRATCH_DEV
> > > +_dmerror_init
> > > +_dmerror_mount
> > > +
> > > +# Create a fs image consuming 1/3 of the scratch fs
> > > +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
> > 
> > _get_available_space $SCRATCH_MNT ?
> > 
> > > +loopimg_bytes=$((scratch_freesp_bytes / 3))
> > > +
> > > +loopimg=$SCRATCH_MNT/testfs
> > > +truncate -s $loopimg_bytes $loopimg
> > > +_mkfs_dev $loopimg
> > > +
> > > +loopmnt=$tmp.mount
> > > +mkdir -p $loopmnt
> > > +
> > > +scratch_aliveflag=$tmp.runsnap
> > > +snap_aliveflag=$tmp.snapping
> > > +
> > > +snap_loop_fs() {
> > > +	touch "$snap_aliveflag"
> > > +	while [ -e "$scratch_aliveflag" ]; do
> > > +		rm -f $loopimg.a
> > > +		_cp_reflink $loopimg $loopimg.a
> > > +		sleep 1
> > > +	done
> > > +	rm -f "$snap_aliveflag"
> > > +}
> > > +
> > > +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> > > +
> > > +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> > > +	touch $scratch_aliveflag
> > > +	snap_loop_fs >> $seqres.full 2>&1 &
> > > +
> > > +	if ! _mount $loopimg $loopmnt -o loop; then
> > > +		rm -f $scratch_aliveflag
> > > +		_fail "loop mount failed"
> > 
> > I found it a bit easier to debug if print $i here.
> 
> Ok, I'll change it to "loop $i mount failed".
> 
> > > +		break
> > > +	fi
> > > +
> > > +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> > > +
> > > +	# purposely include 0 second sleeps to test shutdown immediately after
> > > +	# recovery
> > > +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> > > +	rm -f $scratch_aliveflag
> > > +
> > > +	# This test aims to simulate sudden disk failure, which means that we
> > > +	# do not want to quiesce the filesystem or otherwise give it a chance
> > > +	# to flush its logs.  Therefore we want to call dmsetup with the
> > > +	# --nolockfs parameter; to make this happen we must call the load
> > > +	# error table helper *without* 'lockfs'.
> > > +	_dmerror_load_error_table
> > > +
> > > +	ps -e | grep fsstress > /dev/null 2>&1
> > > +	while [ $? -eq 0 ]; do
> > > +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > +		wait > /dev/null 2>&1
> > > +		ps -e | grep fsstress > /dev/null 2>&1
> > > +	done
> > > +	for ((i = 0; i < 10; i++)); do
> > > +		test -e "$snap_aliveflag" || break
> > > +		sleep 1
> > > +	done
> > > +
> > > +	# Mount again to replay log after loading working table, so we have a
> > > +	# consistent XFS after test.
> > 
> > This is a generic test, fix the XFS specific comments?
> 
> Oops.  "...a consistent fs after test."
> 
> > > +	$UMOUNT_PROG $loopmnt
> > > +	_dmerror_unmount || _fail "unmount failed"
> > > +	_dmerror_load_working_table
> > > +	if ! _dmerror_mount; then
> > > +		dmsetup table | tee -a /dev/ttyprintk
> > > +		lsblk | tee -a /dev/ttyprintk
> > > +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
> > 
> > Above logs all should go to $seqres.full ?
> 
> Oops, yeah.  I'll remove them since I was only using them to check the
> system state.
> 
> > And $XFS_METADUMP_PROG is not suitable for a generic test.
> 
> I'll create _metadump_dev so that this at least works for the two
> filesystems for which we have dump creation helpers (ext* and xfs).

Sounds great!

> 
> > > +		_fail "mount failed"
> > > +	fi
> > > +done
> > > +
> > > +# Make sure the fs image file is ok
> > > +if [ -f "$loopimg" ]; then
> > > +	if _mount $loopimg $loopmnt -o loop; then
> > > +		$UMOUNT_PROG $loopmnt &> /dev/null
> > > +	else
> > > +		echo "final loop mount failed"
> > > +	fi
> > > +	_check_xfs_filesystem $loopimg none none
> > 
> > Same here, use _check_scratch_fs?
> 
> $loopimg is a file within the scratch fs.

_check_scratch_fs can take dev as argument, and default to $SCRATCH_DEV,
I think that works in this case?

Thanks,
Eryu
Darrick J. Wong Aug. 17, 2021, 4:16 a.m. UTC | #7
On Tue, Aug 17, 2021 at 11:16:49AM +0800, Eryu Guan wrote:
> On Mon, Aug 16, 2021 at 09:35:24AM -0700, Darrick J. Wong wrote:
> > On Mon, Aug 16, 2021 at 12:28:20AM +0800, Eryu Guan wrote:
> > > On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> > > > From: Darrick J. Wong <djwong@kernel.org>
> > > > 
> > > > generic/475, but we're running fsstress on a disk image inside the
> > > > scratch filesystem
> > > > 
> > > > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > > > ---
> > > >  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
> > > >  tests/generic/725.out |    2 +
> > > >  2 files changed, 138 insertions(+)
> > > >  create mode 100755 tests/generic/725
> > > >  create mode 100644 tests/generic/725.out
> > > > 
> > > > 
> > > > diff --git a/tests/generic/725 b/tests/generic/725
> > > > new file mode 100755
> > > > index 00000000..f43bcb37
> > > > --- /dev/null
> > > > +++ b/tests/generic/725
> > > > @@ -0,0 +1,136 @@
> > > > +#! /bin/bash
> > > > +# SPDX-License-Identifier: GPL-2.0
> > > > +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> > > > +#
> > > > +# FS QA Test No. 725
> > > > +#
> > > > +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> > > > +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> > > > +# out the underlying scratch device with dm-error to see what happens when the
> > > > +# disk goes down.  Having taken down both fses in this manner, remount them and
> > > > +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> > > > +# in writeback on the host that cause VM guests to fail to recover.
> > > 
> > > It currently fails for me on btrfs, the loop mount failed in 2nd
> > > iteration, seems like a bug in btrfs.
> > 
> > Yep.  Until recently (aka the Big Xfs Log Recovery Bughunt of 2021) it
> > wouldn't pass xfs either. :/
> > 
> > > > +#
> > > > +. ./common/preamble
> > > > +_begin_fstest shutdown auto log metadata eio
> > > > +
> > > > +_cleanup()
> > > > +{
> > > > +	cd /
> > > > +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > > +	wait
> > > > +	if [ -n "$loopmnt" ]; then
> > > > +		umount $loopmnt 2>/dev/null
> > > 
> > > $UMOUNT_PROG
> > > 
> > > > +		rm -r -f $loopmnt
> > > > +	fi
> > > > +	rm -f $tmp.*
> > > > +	_dmerror_unmount
> > > > +	_dmerror_cleanup
> > > > +}
> > > > +
> > > > +# Import common functions.
> > > > +. ./common/dmerror
> > > > +. ./common/reflink
> > > > +
> > > > +# Modify as appropriate.
> > > > +_supported_fs generic
> > > > +
> > > > +_require_scratch_reflink
> > > > +_require_cp_reflink
> > > > +_require_dm_target error
> > > > +_require_command "$KILLALL_PROG" "killall"
> > > > +
> > > > +echo "Silence is golden."
> > > > +
> > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > +_require_metadata_journaling $SCRATCH_DEV
> > > > +_dmerror_init
> > > > +_dmerror_mount
> > > > +
> > > > +# Create a fs image consuming 1/3 of the scratch fs
> > > > +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
> > > 
> > > _get_available_space $SCRATCH_MNT ?
> > > 
> > > > +loopimg_bytes=$((scratch_freesp_bytes / 3))
> > > > +
> > > > +loopimg=$SCRATCH_MNT/testfs
> > > > +truncate -s $loopimg_bytes $loopimg
> > > > +_mkfs_dev $loopimg
> > > > +
> > > > +loopmnt=$tmp.mount
> > > > +mkdir -p $loopmnt
> > > > +
> > > > +scratch_aliveflag=$tmp.runsnap
> > > > +snap_aliveflag=$tmp.snapping
> > > > +
> > > > +snap_loop_fs() {
> > > > +	touch "$snap_aliveflag"
> > > > +	while [ -e "$scratch_aliveflag" ]; do
> > > > +		rm -f $loopimg.a
> > > > +		_cp_reflink $loopimg $loopimg.a
> > > > +		sleep 1
> > > > +	done
> > > > +	rm -f "$snap_aliveflag"
> > > > +}
> > > > +
> > > > +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> > > > +
> > > > +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> > > > +	touch $scratch_aliveflag
> > > > +	snap_loop_fs >> $seqres.full 2>&1 &
> > > > +
> > > > +	if ! _mount $loopimg $loopmnt -o loop; then
> > > > +		rm -f $scratch_aliveflag
> > > > +		_fail "loop mount failed"
> > > 
> > > I found it a bit easier to debug if print $i here.
> > 
> > Ok, I'll change it to "loop $i mount failed".
> > 
> > > > +		break
> > > > +	fi
> > > > +
> > > > +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> > > > +
> > > > +	# purposely include 0 second sleeps to test shutdown immediately after
> > > > +	# recovery
> > > > +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> > > > +	rm -f $scratch_aliveflag
> > > > +
> > > > +	# This test aims to simulate sudden disk failure, which means that we
> > > > +	# do not want to quiesce the filesystem or otherwise give it a chance
> > > > +	# to flush its logs.  Therefore we want to call dmsetup with the
> > > > +	# --nolockfs parameter; to make this happen we must call the load
> > > > +	# error table helper *without* 'lockfs'.
> > > > +	_dmerror_load_error_table
> > > > +
> > > > +	ps -e | grep fsstress > /dev/null 2>&1
> > > > +	while [ $? -eq 0 ]; do
> > > > +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > > +		wait > /dev/null 2>&1
> > > > +		ps -e | grep fsstress > /dev/null 2>&1
> > > > +	done
> > > > +	for ((i = 0; i < 10; i++)); do
> > > > +		test -e "$snap_aliveflag" || break
> > > > +		sleep 1
> > > > +	done
> > > > +
> > > > +	# Mount again to replay log after loading working table, so we have a
> > > > +	# consistent XFS after test.
> > > 
> > > This is a generic test, fix the XFS specific comments?
> > 
> > Oops.  "...a consistent fs after test."
> > 
> > > > +	$UMOUNT_PROG $loopmnt
> > > > +	_dmerror_unmount || _fail "unmount failed"
> > > > +	_dmerror_load_working_table
> > > > +	if ! _dmerror_mount; then
> > > > +		dmsetup table | tee -a /dev/ttyprintk
> > > > +		lsblk | tee -a /dev/ttyprintk
> > > > +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
> > > 
> > > Above logs all should go to $seqres.full ?
> > 
> > Oops, yeah.  I'll remove them since I was only using them to check the
> > system state.
> > 
> > > And $XFS_METADUMP_PROG is not suitable for a generic test.
> > 
> > I'll create _metadump_dev so that this at least works for the two
> > filesystems for which we have dump creation helpers (ext* and xfs).
> 
> Sounds great!
> 
> > 
> > > > +		_fail "mount failed"
> > > > +	fi
> > > > +done
> > > > +
> > > > +# Make sure the fs image file is ok
> > > > +if [ -f "$loopimg" ]; then
> > > > +	if _mount $loopimg $loopmnt -o loop; then
> > > > +		$UMOUNT_PROG $loopmnt &> /dev/null
> > > > +	else
> > > > +		echo "final loop mount failed"
> > > > +	fi
> > > > +	_check_xfs_filesystem $loopimg none none
> > > 
> > > Same here, use _check_scratch_fs?
> > 
> > $loopimg is a file within the scratch fs.
> 
> _check_scratch_fs can take dev as argument, and default to $SCRATCH_DEV,
> I think that works in this case?

It could be made to work with a large enough crowbar, but that's
seriously overkill because $loopimg is a file *within* the scratch
filesystem.  The $loopimg fs gets formatted without the
SCRATCH_LOGDEV/SCRATCH_RTDEV options (because it is not itself the
scratch filesystem), which means that in order to (ab)use
_check_scratch_fs to do the same thing as _check_xfs_filesystem, you
have to exclude those options.  So yes, this:

	SCRATCH_RTDEV= SCRATCH_LOGDEV= _check_scratch_fs $loopimg

is the equivalent of this:

	_check_xfs_filesystem $loopimg none none

But the first is longer and pointless.

--D

> Thanks,
> Eryu
Darrick J. Wong Aug. 17, 2021, 3:54 p.m. UTC | #8
On Mon, Aug 16, 2021 at 09:16:16PM -0700, Darrick J. Wong wrote:
> On Tue, Aug 17, 2021 at 11:16:49AM +0800, Eryu Guan wrote:
> > On Mon, Aug 16, 2021 at 09:35:24AM -0700, Darrick J. Wong wrote:
> > > On Mon, Aug 16, 2021 at 12:28:20AM +0800, Eryu Guan wrote:
> > > > On Tue, Jul 27, 2021 at 05:10:30PM -0700, Darrick J. Wong wrote:
> > > > > From: Darrick J. Wong <djwong@kernel.org>
> > > > > 
> > > > > generic/475, but we're running fsstress on a disk image inside the
> > > > > scratch filesystem
> > > > > 
> > > > > Signed-off-by: Darrick J. Wong <djwong@kernel.org>
> > > > > ---
> > > > >  tests/generic/725     |  136 +++++++++++++++++++++++++++++++++++++++++++++++++
> > > > >  tests/generic/725.out |    2 +
> > > > >  2 files changed, 138 insertions(+)
> > > > >  create mode 100755 tests/generic/725
> > > > >  create mode 100644 tests/generic/725.out
> > > > > 
> > > > > 
> > > > > diff --git a/tests/generic/725 b/tests/generic/725
> > > > > new file mode 100755
> > > > > index 00000000..f43bcb37
> > > > > --- /dev/null
> > > > > +++ b/tests/generic/725
> > > > > @@ -0,0 +1,136 @@
> > > > > +#! /bin/bash
> > > > > +# SPDX-License-Identifier: GPL-2.0
> > > > > +# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
> > > > > +#
> > > > > +# FS QA Test No. 725
> > > > > +#
> > > > > +# Test nested log recovery with repeated (simulated) disk failures.  We kick
> > > > > +# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
> > > > > +# out the underlying scratch device with dm-error to see what happens when the
> > > > > +# disk goes down.  Having taken down both fses in this manner, remount them and
> > > > > +# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
> > > > > +# in writeback on the host that cause VM guests to fail to recover.
> > > > 
> > > > It currently fails for me on btrfs, the loop mount failed in 2nd
> > > > iteration, seems like a bug in btrfs.
> > > 
> > > Yep.  Until recently (aka the Big Xfs Log Recovery Bughunt of 2021) it
> > > wouldn't pass xfs either. :/
> > > 
> > > > > +#
> > > > > +. ./common/preamble
> > > > > +_begin_fstest shutdown auto log metadata eio
> > > > > +
> > > > > +_cleanup()
> > > > > +{
> > > > > +	cd /
> > > > > +	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > > > +	wait
> > > > > +	if [ -n "$loopmnt" ]; then
> > > > > +		umount $loopmnt 2>/dev/null
> > > > 
> > > > $UMOUNT_PROG
> > > > 
> > > > > +		rm -r -f $loopmnt
> > > > > +	fi
> > > > > +	rm -f $tmp.*
> > > > > +	_dmerror_unmount
> > > > > +	_dmerror_cleanup
> > > > > +}
> > > > > +
> > > > > +# Import common functions.
> > > > > +. ./common/dmerror
> > > > > +. ./common/reflink
> > > > > +
> > > > > +# Modify as appropriate.
> > > > > +_supported_fs generic
> > > > > +
> > > > > +_require_scratch_reflink
> > > > > +_require_cp_reflink
> > > > > +_require_dm_target error
> > > > > +_require_command "$KILLALL_PROG" "killall"
> > > > > +
> > > > > +echo "Silence is golden."
> > > > > +
> > > > > +_scratch_mkfs >> $seqres.full 2>&1
> > > > > +_require_metadata_journaling $SCRATCH_DEV
> > > > > +_dmerror_init
> > > > > +_dmerror_mount
> > > > > +
> > > > > +# Create a fs image consuming 1/3 of the scratch fs
> > > > > +scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
> > > > 
> > > > _get_available_space $SCRATCH_MNT ?
> > > > 
> > > > > +loopimg_bytes=$((scratch_freesp_bytes / 3))
> > > > > +
> > > > > +loopimg=$SCRATCH_MNT/testfs
> > > > > +truncate -s $loopimg_bytes $loopimg
> > > > > +_mkfs_dev $loopimg
> > > > > +
> > > > > +loopmnt=$tmp.mount
> > > > > +mkdir -p $loopmnt
> > > > > +
> > > > > +scratch_aliveflag=$tmp.runsnap
> > > > > +snap_aliveflag=$tmp.snapping
> > > > > +
> > > > > +snap_loop_fs() {
> > > > > +	touch "$snap_aliveflag"
> > > > > +	while [ -e "$scratch_aliveflag" ]; do
> > > > > +		rm -f $loopimg.a
> > > > > +		_cp_reflink $loopimg $loopimg.a
> > > > > +		sleep 1
> > > > > +	done
> > > > > +	rm -f "$snap_aliveflag"
> > > > > +}
> > > > > +
> > > > > +fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
> > > > > +
> > > > > +for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
> > > > > +	touch $scratch_aliveflag
> > > > > +	snap_loop_fs >> $seqres.full 2>&1 &
> > > > > +
> > > > > +	if ! _mount $loopimg $loopmnt -o loop; then
> > > > > +		rm -f $scratch_aliveflag
> > > > > +		_fail "loop mount failed"
> > > > 
> > > > I found it a bit easier to debug if print $i here.
> > > 
> > > Ok, I'll change it to "loop $i mount failed".
> > > 
> > > > > +		break
> > > > > +	fi
> > > > > +
> > > > > +	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
> > > > > +
> > > > > +	# purposely include 0 second sleeps to test shutdown immediately after
> > > > > +	# recovery
> > > > > +	sleep $((RANDOM % (3 * TIME_FACTOR) ))
> > > > > +	rm -f $scratch_aliveflag
> > > > > +
> > > > > +	# This test aims to simulate sudden disk failure, which means that we
> > > > > +	# do not want to quiesce the filesystem or otherwise give it a chance
> > > > > +	# to flush its logs.  Therefore we want to call dmsetup with the
> > > > > +	# --nolockfs parameter; to make this happen we must call the load
> > > > > +	# error table helper *without* 'lockfs'.
> > > > > +	_dmerror_load_error_table
> > > > > +
> > > > > +	ps -e | grep fsstress > /dev/null 2>&1
> > > > > +	while [ $? -eq 0 ]; do
> > > > > +		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
> > > > > +		wait > /dev/null 2>&1
> > > > > +		ps -e | grep fsstress > /dev/null 2>&1
> > > > > +	done
> > > > > +	for ((i = 0; i < 10; i++)); do
> > > > > +		test -e "$snap_aliveflag" || break
> > > > > +		sleep 1
> > > > > +	done
> > > > > +
> > > > > +	# Mount again to replay log after loading working table, so we have a
> > > > > +	# consistent XFS after test.
> > > > 
> > > > This is a generic test, fix the XFS specific comments?
> > > 
> > > Oops.  "...a consistent fs after test."
> > > 
> > > > > +	$UMOUNT_PROG $loopmnt
> > > > > +	_dmerror_unmount || _fail "unmount failed"
> > > > > +	_dmerror_load_working_table
> > > > > +	if ! _dmerror_mount; then
> > > > > +		dmsetup table | tee -a /dev/ttyprintk
> > > > > +		lsblk | tee -a /dev/ttyprintk
> > > > > +		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
> > > > 
> > > > Above logs all should go to $seqres.full ?
> > > 
> > > Oops, yeah.  I'll remove them since I was only using them to check the
> > > system state.
> > > 
> > > > And $XFS_METADUMP_PROG is not suitable for a generic test.
> > > 
> > > I'll create _metadump_dev so that this at least works for the two
> > > filesystems for which we have dump creation helpers (ext* and xfs).
> > 
> > Sounds great!
> > 
> > > 
> > > > > +		_fail "mount failed"
> > > > > +	fi
> > > > > +done
> > > > > +
> > > > > +# Make sure the fs image file is ok
> > > > > +if [ -f "$loopimg" ]; then
> > > > > +	if _mount $loopimg $loopmnt -o loop; then
> > > > > +		$UMOUNT_PROG $loopmnt &> /dev/null
> > > > > +	else
> > > > > +		echo "final loop mount failed"
> > > > > +	fi
> > > > > +	_check_xfs_filesystem $loopimg none none
> > > > 
> > > > Same here, use _check_scratch_fs?
> > > 
> > > $loopimg is a file within the scratch fs.
> > 
> > _check_scratch_fs can take dev as argument, and default to $SCRATCH_DEV,
> > I think that works in this case?
> 
> It could be made to work with a large enough crowbar, but that's
> seriously overkill because $loopimg is a file *within* the scratch
> filesystem.  The $loopimg fs gets formatted without the
> SCRATCH_LOGDEV/SCRATCH_RTDEV options (because it is not itself the
> scratch filesystem), which means that in order to (ab)use
> _check_scratch_fs to do the same thing as _check_xfs_filesystem, you
> have to exclude those options.  So yes, this:
> 
> 	SCRATCH_RTDEV= SCRATCH_LOGDEV= _check_scratch_fs $loopimg
> 
> is the equivalent of this:
> 
> 	_check_xfs_filesystem $loopimg none none
> 
> But the first is longer and pointless.

...and now that it's morning and I've had coffee again, I now understand
what you're actually asking, which is "Don't use _foo_xfs* functions in
a generic test!", not "rototill in this helper for stylistic reasons".

Judging from my immediate defensive reaction, I've clearly been worn
down by all the bikeshedding the past year.  Have they resumed flights
to Nobikeshed Island?

Anyway, I'll go fix that.  Thank you for catching the mistake.

--D

> --D
> 
> > Thanks,
> > Eryu
diff mbox series

Patch

diff --git a/tests/generic/725 b/tests/generic/725
new file mode 100755
index 00000000..f43bcb37
--- /dev/null
+++ b/tests/generic/725
@@ -0,0 +1,136 @@ 
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2021 Oracle, Inc.  All Rights Reserved.
+#
+# FS QA Test No. 725
+#
+# Test nested log recovery with repeated (simulated) disk failures.  We kick
+# off fsstress on a loopback filesystem mounted on the scratch fs, then switch
+# out the underlying scratch device with dm-error to see what happens when the
+# disk goes down.  Having taken down both fses in this manner, remount them and
+# repeat.  This test simulates VM hosts crashing to try to shake out CoW bugs
+# in writeback on the host that cause VM guests to fail to recover.
+#
+. ./common/preamble
+_begin_fstest shutdown auto log metadata eio
+
+_cleanup()
+{
+	cd /
+	$KILLALL_PROG -9 fsstress > /dev/null 2>&1
+	wait
+	if [ -n "$loopmnt" ]; then
+		umount $loopmnt 2>/dev/null
+		rm -r -f $loopmnt
+	fi
+	rm -f $tmp.*
+	_dmerror_unmount
+	_dmerror_cleanup
+}
+
+# Import common functions.
+. ./common/dmerror
+. ./common/reflink
+
+# Modify as appropriate.
+_supported_fs generic
+
+_require_scratch_reflink
+_require_cp_reflink
+_require_dm_target error
+_require_command "$KILLALL_PROG" "killall"
+
+echo "Silence is golden."
+
+_scratch_mkfs >> $seqres.full 2>&1
+_require_metadata_journaling $SCRATCH_DEV
+_dmerror_init
+_dmerror_mount
+
+# Create a fs image consuming 1/3 of the scratch fs
+scratch_freesp_bytes=$(stat -f -c '%a * %S' $SCRATCH_MNT | bc)
+loopimg_bytes=$((scratch_freesp_bytes / 3))
+
+loopimg=$SCRATCH_MNT/testfs
+truncate -s $loopimg_bytes $loopimg
+_mkfs_dev $loopimg
+
+loopmnt=$tmp.mount
+mkdir -p $loopmnt
+
+scratch_aliveflag=$tmp.runsnap
+snap_aliveflag=$tmp.snapping
+
+snap_loop_fs() {
+	touch "$snap_aliveflag"
+	while [ -e "$scratch_aliveflag" ]; do
+		rm -f $loopimg.a
+		_cp_reflink $loopimg $loopimg.a
+		sleep 1
+	done
+	rm -f "$snap_aliveflag"
+}
+
+fsstress=($FSSTRESS_PROG $FSSTRESS_AVOID -d "$loopmnt" -n 999999 -p "$((LOAD_FACTOR * 4))")
+
+for i in $(seq 1 $((25 * TIME_FACTOR)) ); do
+	touch $scratch_aliveflag
+	snap_loop_fs >> $seqres.full 2>&1 &
+
+	if ! _mount $loopimg $loopmnt -o loop; then
+		rm -f $scratch_aliveflag
+		_fail "loop mount failed"
+		break
+	fi
+
+	("${fsstress[@]}" >> $seqres.full &) > /dev/null 2>&1
+
+	# purposely include 0 second sleeps to test shutdown immediately after
+	# recovery
+	sleep $((RANDOM % (3 * TIME_FACTOR) ))
+	rm -f $scratch_aliveflag
+
+	# This test aims to simulate sudden disk failure, which means that we
+	# do not want to quiesce the filesystem or otherwise give it a chance
+	# to flush its logs.  Therefore we want to call dmsetup with the
+	# --nolockfs parameter; to make this happen we must call the load
+	# error table helper *without* 'lockfs'.
+	_dmerror_load_error_table
+
+	ps -e | grep fsstress > /dev/null 2>&1
+	while [ $? -eq 0 ]; do
+		$KILLALL_PROG -9 fsstress > /dev/null 2>&1
+		wait > /dev/null 2>&1
+		ps -e | grep fsstress > /dev/null 2>&1
+	done
+	for ((i = 0; i < 10; i++)); do
+		test -e "$snap_aliveflag" || break
+		sleep 1
+	done
+
+	# Mount again to replay log after loading working table, so we have a
+	# consistent XFS after test.
+	$UMOUNT_PROG $loopmnt
+	_dmerror_unmount || _fail "unmount failed"
+	_dmerror_load_working_table
+	if ! _dmerror_mount; then
+		dmsetup table | tee -a /dev/ttyprintk
+		lsblk | tee -a /dev/ttyprintk
+		$XFS_METADUMP_PROG -a -g -o $DMERROR_DEV $seqres.dmfail.md
+		_fail "mount failed"
+	fi
+done
+
+# Make sure the fs image file is ok
+if [ -f "$loopimg" ]; then
+	if _mount $loopimg $loopmnt -o loop; then
+		$UMOUNT_PROG $loopmnt &> /dev/null
+	else
+		echo "final loop mount failed"
+	fi
+	_check_xfs_filesystem $loopimg none none
+fi
+
+# success, all done
+status=0
+exit
diff --git a/tests/generic/725.out b/tests/generic/725.out
new file mode 100644
index 00000000..ed73a9fc
--- /dev/null
+++ b/tests/generic/725.out
@@ -0,0 +1,2 @@ 
+QA output created by 725
+Silence is golden.