diff mbox series

[RFC,3/5] check: Improve pass/fail metrics and section config output

Message ID 05768096e43da32f50ef337c0676d06fb8c9454e.1736496620.git.nirjhar.roy.lists@gmail.com (mailing list archive)
State New
Headers show
Series Add support for central fsconfig and -q <n> unconditional loop | expand

Commit Message

Nirjhar Roy (IBM) Jan. 10, 2025, 9:10 a.m. UTC
This patch improves the output by:
- Adding pass/fail aggregate results in the stdout for each section's
  summary. Also adds the location of the xunit xml files in the
  section's summary.
- Adds section config details into the result's global log file.

e.g.

$ ./check -q 2 -R xunit-quiet   selftest/001

SECTION       -- s1
=========================
selftest/001 aggregate results across 2 runs: pass=2 (100.0%)
Ran: selftest/001
Passed all 1 tests
Xunit report: /home/ubuntu/xfstests/results//s1/result.xml

SECTION       -- s2
=========================
selftest/001 aggregate results across 2 runs: pass=2 (100.0%)
Ran: selftest/001
Passed all 1 tests
Xunit report: /home/ubuntu/xfstests/results//s2/result.xml

Output in results/check.log file

Kernel version: 6.13.0-rc1-00182-gb8f52214c61a-dirty
Wed Jan  8 11:23:57 UTC 2025
SECTION       -- s1
=========================
FSTYP         -- ext4
PLATFORM      -- Linux/x86_64 citest-1 6.13.0-rc1-00182-gb8f52214c61a-dirty #5 SMP PREEMPT_DYNAMIC Wed Dec 18 14:03:34 IST 2024
MKFS_OPTIONS  -- -F /dev/loop1
MOUNT_OPTIONS -- -o acl,user_xattr /dev/loop1 /mnt1/scratch
selftest/001 aggregate results across 2 runs: pass=2 (100.0%)
Ran: selftest/001
Passed all 1 tests
Xunit report: /home/ubuntu/xfstests/results//s1/result.xml

Kernel version: 6.13.0-rc1-00182-gb8f52214c61a-dirty
Wed Jan  8 11:24:02 UTC 2025
SECTION       -- s2
=========================
FSTYP         -- ext4
PLATFORM      -- Linux/x86_64 citest-1 6.13.0-rc1-00182-gb8f52214c61a-dirty #5 SMP PREEMPT_DYNAMIC Wed Dec 18 14:03:34 IST 2024
MKFS_OPTIONS  -- -F /dev/loop1
MOUNT_OPTIONS -- -o acl,user_xattr /dev/loop1 /mnt1/scratch
selftest/001 aggregate results across 2 runs: pass=2 (100.0%)
Ran: selftest/001
Passed all 1 tests
Xunit report: /home/ubuntu/xfstests/results//s2/result.xml

Suggested-by: Ritesh Harjani <ritesh.list@gmail.com>
Co-developed-by: Ritesh Harjani (IBM) <ritesh.list@gmail.com>
Signed-off-by: Ritesh Harjani (IBM) <ritesh.list@gmail.com>
Signed-off-by: Nirjhar Roy (IBM) <nirjhar.roy.lists@gmail.com>
---
 check | 67 ++++++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 46 insertions(+), 21 deletions(-)
diff mbox series

Patch

diff --git a/check b/check
index e8ac0ce9..f7028836 100755
--- a/check
+++ b/check
@@ -468,6 +468,25 @@  if [ -n "$REPORT_GCOV" ]; then
 	_gcov_check_report_gcov
 fi
 
+_print_list()
+{
+	local -n list=$1
+	local item
+	for item in "${list[@]}"; do
+		echo "$item"
+	done
+}
+
+_display_test_configuration()
+{
+	echo "FSTYP         -- `_full_fstyp_details`"
+	echo "PLATFORM      -- `_full_platform_details`"
+	if [ ! -z "$SCRATCH_DEV" ]; then
+		echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
+		echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
+	fi
+}
+
 _wrapup()
 {
 	seq="check.$$"
@@ -505,11 +524,19 @@  _wrapup()
 
 		echo "SECTION       -- $section" >>$tmp.summary
 		echo "=========================" >>$tmp.summary
+
+		_global_log "SECTION       -- $section"
+		_global_log "=========================" >>$tmp.summary
+		_global_log "$(_display_test_configuration)"
 		if ((${#try[*]} > 0)); then
+			local test_aggr_stats=$(_print_list loop_test_stats_per_section)
 			if [ $brief_test_summary == "false" ]; then
+				[[ -n "$test_aggr_stats" ]] && echo "$test_aggr_stats" >> \
+					$tmp.summary
 				echo "Ran: ${try[*]}"
 				echo "Ran: ${try[*]}" >>$tmp.summary
 			fi
+			_global_log "$test_aggr_stats"
 			_global_log "Ran: ${try[*]}"
 		fi
 
@@ -539,12 +566,15 @@  _wrapup()
 			_global_log "Passed all ${#try[*]} tests"
 			echo "Passed all ${#try[*]} tests" >>$tmp.summary
 		fi
-		echo "" >>$tmp.summary
 		if $do_report; then
 			_make_section_report "$section" "${#try[*]}" \
 					     "${#bad[*]}" "${#notrun[*]}" \
 					     "$((sect_stop - sect_start))"
+			local out_fn="$REPORT_DIR/result.xml"
+			echo "Xunit report: $out_fn" >> $tmp.summary
+			_global_log "Xunit report: $out_fn"
 		fi
+		echo "" >>$tmp.summary
 
 		# Generate code coverage report
 		if [ -n "$REPORT_GCOV" ]; then
@@ -655,18 +685,19 @@  _stash_test_status() {
 
 		if ((loop_on_fail && ${#loop_status[*]} > loop_on_fail)) || \
 		   ((loop_unconditional && ${#loop_status[*]} > loop_unconditional)); then
-			printf "%s aggregate results across %d runs: " \
-				"$test_seq" "${#loop_status[*]}"
-			awk "BEGIN {
-				n=split(\"${loop_status[*]}\", arr);"'
-				for (i = 1; i <= n; i++)
-					stats[arr[i]]++;
-				for (x in stats)
-					printf("%s=%d (%.1f%%)",
-					       (i-- > n ? x : ", " x),
-					       stats[x], 100 * stats[x] / n);
-				}'
-			echo
+			local test_stats=$(printf "%s aggregate results across %d runs: " \
+				"$test_seq" "${#loop_status[*]}")
+			test_stats+=$(awk "BEGIN {
+					n=split(\"${loop_status[*]}\", arr);"'
+					for (i = 1; i <= n; i++)
+						stats[arr[i]]++;
+					for (x in stats)
+						printf("%s=%d (%.1f%%)",
+							   (i-- > n ? x : ", " x),
+							   stats[x], 100 * stats[x] / n);
+					}')
+			echo "$test_stats"
+			loop_test_stats_per_section+=("$test_stats")
 			loop_status=()
 			is_bad_test=false
 		fi
@@ -839,14 +870,7 @@  function run_section()
 	rm -f $check.full
 
 	[ -f $check.time ] || touch $check.time
-
-	# print out our test configuration
-	echo "FSTYP         -- `_full_fstyp_details`"
-	echo "PLATFORM      -- `_full_platform_details`"
-	if [ ! -z "$SCRATCH_DEV" ]; then
-	  echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
-	  echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
-	fi
+	_display_test_configuration
 	echo
 	test -n "$REPORT_GCOV" && _gcov_reset
 	needwrap=true
@@ -884,6 +908,7 @@  function run_section()
 
 	is_bad_test=false
 	loop_status=()	# track loop rerun state
+	loop_test_stats_per_section=() # store loop test statistics per section
 	local tc_status ix
 	local -a _list=( $list )
 	for ((ix = 0; ix < ${#_list[*]}; !${#loop_status[*]} && ix++)); do