@@ -11,6 +11,10 @@ if [ ! -x "$default_path/sparse-llvm" ]; then
disabled_cmds="sparsec sparsei sparse-llvm"
fi
+# flags:
+# - some tests gave an unexpected result
+failed=0
+
# counts:
# - tests that have not been converted to test-suite format
# - tests that are disabled
@@ -163,18 +167,27 @@ do_test()
test_failed=1
fi
- if [ "$test_failed" -eq "1" ]; then
- ko_tests=`expr $ko_tests + 1`
- get_tag "check-known-to-fail" $file
- if [ "$?" -eq "0" ]; then
+ get_tag "check-known-to-fail" $file
+ must_fail=`expr "$?" = 0`
+ known_ko_tests=`expr $known_ko_tests + $must_fail`
+
+ [ "$test_failed" -eq "$must_fail" ] || failed=1
+
+ if [ "$must_fail" -eq "1" ]; then
+ if [ "$test_failed" -eq "1" ]; then
echo "info: test '$file' is known to fail"
- known_ko_tests=`expr $known_ko_tests + 1`
+ else
+ echo "info: test '$file' is known to fail but succeed!"
+ test_failed=1
fi
- return 1
+ fi
+
+ if [ "$test_failed" -eq "1" ]; then
+ ko_tests=`expr $ko_tests + 1`
else
ok_tests=`expr $ok_tests + 1`
- return 0
fi
+ return $test_failed
}
do_test_suite()
@@ -273,5 +286,5 @@ case "$1" in
;;
esac
-exit 0
+exit $failed
Such situation may simply show that what was tested is now fixed and that it's juste the test annotation which need to be adapted, but can be a sign that something else is broken. Reporting the exact result (failure/success, known-to-fail/expect-to-succeed) make the testsuite more useful and allow to use more efficiently git-bisect or other automated testing tools. Signed-off-by: Luc Van Oostenryck <luc.vanoostenryck@gmail.com> --- validation/test-suite | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-)