@@ -26,6 +26,7 @@ do_report=false
DUMP_OUTPUT=false
iterations=1
istop=false
+loop_on_fail=0
# This is a global variable used to pass test failure text to reporting gunk
_err_msg=""
@@ -75,6 +76,7 @@ check options
--large-fs optimise scratch device for large filesystems
-s section run only specified section from config file
-S section exclude the specified section from the config file
+ -L <n> loop tests <n> times following a failure, measuring aggregate pass/fail metrics
testlist options
-g group[,group...] include tests from these groups
@@ -333,6 +335,7 @@ while [ $# -gt 0 ]; do
;;
--large-fs) export LARGE_SCRATCH_DEV=yes ;;
--extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
+ -L) loop_on_fail=$2; shift ;;
-*) usage ;;
*) # not an argument, we've got tests now.
@@ -606,6 +609,40 @@ _run_seq() {
fi
}
+# Check whether the last test should be rerun according to loop-on-error state
+# and return "0" if so, otherwise return "1".
+_ix_inc() {
+ local test_status="$1"
+ local loop_len="$2"
+
+ if ((!loop_on_fail)); then
+ echo 1
+ return
+ fi
+
+ if [ "$test_status" == "fail" ] && ((!loop_len)); then
+ echo 0 # initial failure of this test, start loop-on-fail
+ elif ((loop_len > 0)) && ((loop_len < loop_on_fail)); then
+ echo 0 # continue loop following initial failure
+ else
+ echo 1 # completed or not currently in a failure loop
+ fi
+}
+
+_failure_loop_dump_stats() {
+ local t="$1" && shift
+ awk "BEGIN {
+ n=split(\"$*\", arr);"'
+ for (i = 1; i <= n; i++)
+ stats[arr[i]]++;
+ printf("'"$t"' aggregate results across %d runs: ", n);
+ for (x in stats)
+ printf("%s=%d (%.1f%%)", (i-- > n ? x : ", " x),
+ stats[x], 100 * stats[x] / n);
+ print;
+ }'
+}
+
_detect_kmemleak
_prepare_test_list
@@ -746,13 +783,33 @@ function run_section()
seqres="$check"
_check_test_fs
- local tc_status="init"
+ local tc_status="init" ix
prev_seq=""
- for seq in $list ; do
- # Run report for previous test!
- _stash_test_status "$seqnum" "$tc_status"
- if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
- _make_testcase_report "$prev_seq" "$tc_status"
+ local -a _list=( $list ) loop_status=()
+ for ((ix = 0; ix < ${#_list[*]};
+ ix += $(_ix_inc "$tc_status" "${#loop_status[*]}"))); do
+ seq="${_list[$ix]}"
+
+ if ((!loop_on_fail)); then
+ # Run report for previous test!
+ _stash_test_status "$seqnum" "$tc_status"
+ if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
+ _make_testcase_report "$prev_seq" "$tc_status"
+ fi
+ else
+ if [ "$seq" == "$prev_seq" ]; then
+ if ((!${#loop_status[*]})); then
+ _stash_test_status "$seqnum" "$tc_status"
+ if $do_report; then
+ _make_testcase_report "$prev_seq" "$tc_status"
+ fi
+ fi
+ loop_status+=("$tc_status")
+ elif ((${#loop_status[*]})); then
+ loop_status+=("$tc_status")
+ _failure_loop_dump_stats "$seqnum" "${loop_status[@]}"
+ loop_status=()
+ fi
fi
prev_seq="$seq"
@@ -822,7 +879,9 @@ function run_section()
fi
# record that we really tried to run this test.
- try+=("$seqnum")
+ if ((!${#loop_status[*]})); then
+ try+=("$seqnum")
+ fi
awk 'BEGIN {lasttime=" "} \
$1 == "'$seqnum'" {lasttime=" " $2 "s ... "; exit} \
@@ -949,10 +1008,15 @@ function run_section()
fi
done
- # make sure we record the status of the last test we ran.
- _stash_test_status "$seqnum" "$tc_status"
- if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
- _make_testcase_report "$prev_seq" "$tc_status"
+ if ((${#loop_status[*]})); then
+ loop_status+=("$tc_status")
+ _failure_loop_dump_stats "$seqnum" "${loop_status[@]}"
+ else
+ # make sure we record the status of the last test we ran.
+ _stash_test_status "$seqnum" "$tc_status"
+ if $do_report && [[ ! $tc_status =~ ^(init|expunge)$ ]]; then
+ _make_testcase_report "$prev_seq" "$tc_status"
+ fi
fi
sect_stop=`_wallclock`
If check is run with -L <n>, then a failed test will be rerun <n> times before proceeding to the next test. Following completion of the rerun loop, aggregate pass/fail statistics are printed. Caveats: - rerun tests will be tracked as a single failure in @try and @bad + xunit reports do not include any rerun details - .bad files generated on failure will be overwritten by test reruns Suggested-by: Theodore Ts'o <tytso@mit.edu> Link: https://lwn.net/Articles/897061/ Signed-off-by: David Disseldorp <ddiss@suse.de> --- check | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 11 deletions(-)