diff mbox

[2/2] report: Add xunit format report generator

Message ID 1485882983-16883-3-git-send-email-dmonakhov@openvz.org (mailing list archive)
State New, archived
Headers show

Commit Message

Dmitry Monakhov Jan. 31, 2017, 5:16 p.m. UTC 
xunit[1]/junit[2] are well known report formats for tests frameworks which
supported by most of test CI frameworks(such as Jenkins [3], Bamboo [4], Avocado [5])
Basically this is just xml document which can be easily parsed later by external tools.

EXAMPLE:
#./check --xunit -s ext4 generic/001 generic/010 generic/013
#cat results/ext4/result.xml
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="xfstests" errors="0" skipped="1" tests="3" time="12" timestamp="2017-01-31 18:24:06" >
	<testcase classname="ext4" name="generic/001" time="6">
	</testcase>
	<testcase classname="ext4" name="generic/010" time="0">
		<skipped message="src/dbtest not built" />
	</testcase>
	<testcase classname="ext4" name="generic/013" time="3">
	</testcase>
</testsuite>

Footnotes:
[1] https://xunit.github.io/docs/format-xml-v2.html
[2] http://help.catchsoftware.com/display/ET/JUnit+Format
[3] jenkins.io
[4] https://www.atlassian.com/software/bamboo
[5] https://github.com/avocado-framework/avocado

Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org>
---
 check         |  22 +++++++++---
 common/rc     |   3 ++
 common/report | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 128 insertions(+), 5 deletions(-)
 create mode 100644 common/report
diff mbox

Patch

diff --git a/check b/check
index edc0899..1442b6c 100755
--- a/check
+++ b/check
@@ -54,7 +54,7 @@  export DIFF_LENGTH=${DIFF_LENGTH:=10}
 # by default don't output timestamps
 timestamp=${TIMESTAMP:=false}
 
-rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist
+rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.*
 
 SRC_GROUPS="generic shared"
 export SRC_DIR="tests"
@@ -75,6 +75,7 @@  check options
     -r			randomize test order
     -d			dump test output to stdout
     -b			brief test summary
+    --xunit		generate summary in xunit format
     --large-fs		optimise scratch device for large filesystems
     -s section		run only specified section from config file
     -S section		exclude the specified section from the config file
@@ -296,6 +297,7 @@  while [ $# -gt 0 ]; do
 	-d)	DUMP_OUTPUT=true ;;
 	-b)	brief_test_summary=true;;
 
+	--xunit) REPORT_XUNIT=yes; DO_REPORT=yes ;;
 	--large-fs) export LARGE_SCRATCH_DEV=yes ;;
 	--extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
 
@@ -380,7 +382,10 @@  _wrapup()
 	check="$RESULT_BASE/check"
 
 	if $showme; then
-	:
+		if $needwrap; then
+			_make_section_report
+			needwrap=false
+		fi
 	elif $needwrap; then
 		if [ -f $check.time -a -f $tmp.time ]; then
 			cat $check.time $tmp.time  \
@@ -431,9 +436,9 @@  _wrapup()
 			echo "Passed all $n_try tests" >>$tmp.summary
 		fi
 		echo "" >>$tmp.summary
+		_make_section_report
 		needwrap=false
 	fi
-
 	sum_bad=`expr $sum_bad + $n_bad`
 	_wipe_counters
 	rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
@@ -519,6 +524,7 @@  for section in $HOST_OPTIONS_SECTIONS; do
 		echo "SECTION       -- $section"
 	fi
 
+	sect_start=`_wallclock`
 	if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
 		echo "RECREATING    -- $FSTYP on $TEST_DEV"
 		_test_unmount 2> /dev/null
@@ -624,9 +630,11 @@  for section in $HOST_OPTIONS_SECTIONS; do
 	    if $OPTIONS_HAVE_SECTIONS; then
 		export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
 		seqres="$RESULT_BASE/$section/$seqnum"
+		sectdir_res="$RESULT_BASE/$section"
 	    else
 		export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
 		seqres="$RESULT_BASE/$seqnum"
+		sectdir_res="$RESULT_BASE"
 	    fi
 
 	    mkdir -p $RESULT_DIR
@@ -638,9 +646,10 @@  for section in $HOST_OPTIONS_SECTIONS; do
 		start=0
 		stop=0
 		n_notrun=`expr $n_notrun + 1`
+		_make_testcase_report "list"
 		continue
 	    fi
-
+	    tc_status="pass"
 	    if [ ! -f $seq ]; then
 		echo " - no such test?"
 	    else
@@ -704,6 +713,7 @@  for section in $HOST_OPTIONS_SECTIONS; do
 		    cat $seqres.notrun
 		    notrun="$notrun $seqnum"
 		    n_notrun=`expr $n_notrun + 1`
+		    tc_status="notrun"
 		else
 		    if [ $sts -ne 0 ]
 		    then
@@ -763,10 +773,12 @@  for section in $HOST_OPTIONS_SECTIONS; do
 		bad="$bad $seqnum"
 		n_bad=`expr $n_bad + 1`
 		quick=false
+		tc_status="fail"
 	    fi
-
+	    _make_testcase_report "$tc_status"
 	    seq="after_$seqnum"
 	done
+	sect_stop=`_wallclock`
 	_wrapup
 	echo
 
diff --git a/common/rc b/common/rc
index f97b4d2..9f1a068 100644
--- a/common/rc
+++ b/common/rc
@@ -3246,6 +3246,9 @@  _get_fs_sysfs_attr()
 }
 
 
+if [ ! -z "$DO_REPORT" ]; then
+   . ./common/report
+fi
 
 init_rc
 
diff --git a/common/report b/common/report
new file mode 100644
index 0000000..05496c5
--- /dev/null
+++ b/common/report
@@ -0,0 +1,108 @@ 
+#
+# Reports generator funcitons lives here
+#
+
+
+#
+# Xunit format report functions
+_xunit_make_section_report()
+{
+	# section ==> testsuite
+	#<?xml version="1.0" encoding="UTF-8"?>
+	#<testsuite errors="0" failures="1" name="avocado" skipped="1" tests="3" time="0.0942118167877" timestamp="2017-01-30 21:43:42.117398">
+	sect_name=$section
+	sect_time=`expr $sect_stop - $sect_start`
+
+	if [ $sect_name == '-no-sections-' ]; then
+	    sect_name='global'
+	fi
+	report=$tmp.report.xunit.$sect_name.xml
+
+	n_total=`expr $n_try + $n_notrun`
+	echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" > $sectdir_res/result.xml
+	stats="errors=\"$n_bad\" skipped=\"$n_notrun\" tests=\"$n_total\" time=\"$sect_time\" timestamp=\"$date_time\""
+	echo "<testsuite name=\"xfstests\" $stats >" >> $sectdir_res/result.xml
+	cat $tmp.report.xunit.$sect_name.xml >> $sectdir_res/result.xml
+	echo "</testsuite>" >> $sectdir_res/result.xml
+	echo "Xunit report: $sectdir_res/result.xml"
+}
+
+_xunit_make_testcase_report()
+{
+	test_status="$1"
+	test_time=`expr $stop - $start`
+	sect_name=$section
+
+	#TODO: other places may also win if no-section mode will be named like 'default/global'
+	if [ $sect_name == '-no-sections-' ]; then
+		sect_name='global'
+	fi
+	report=$tmp.report.xunit.$sect_name.xml
+	strip="$SRC_DIR/"
+	test_name=${seq#$strip}
+	echo -e "\t<testcase classname=\"$sect_name\" name=\"$test_name\" time=\"$test_time\">" >> $report
+	case $test_status in
+		"pass")
+		;;
+		"notrun")
+			if [ -f $seqres.notrun ]; then
+			    msg=`cat $seqres.notrun`
+			    echo -e "\t\t<skipped message=\"$msg\" />" >> $report
+			else
+				echo -e "\t\t<skipped/>" >> $report
+			fi
+			;;
+		"list")
+			echo -e "\t\t<skipped/>" >> $report
+			;;
+		"fail")
+			if [ -z "$err_msg" ]; then
+			    err_msg="Test $sequm failed, reason unknown"
+			fi
+			echo -e "\t\t<failure message=\"$err_msg\" type=\"TestFail\">" >> $report
+			if [ -f $seqres.full ]; then
+			    echo -e "\t\t\t<system-out>" >> $report
+			    printf  '<![CDATA[\n' >>$report
+			    cat $seqres.full | tr -dc '[:print:][:space:]' >>$report
+			    printf ']]>\n'  >>$report
+			    echo -e "\t\t\t</system-out>" >> $report
+			fi
+			if [ -f $seqres.dmesg ]; then
+			    echo -e "\t\t\t<system-err>" >> $report
+			    printf  '<![CDATA[\n' >>$report
+			    cat $seqres.dmesg | tr -dc '[:print:][:space:]' >>$report
+			    printf ']]>\n'  >>$report
+			    echo -e "\t\t\t</system-err>" >> $report
+			elif [ -f $seqres.out.bad ]; then
+			    echo -e "\t\t\t<system-err>" >> $report
+			    printf  '<![CDATA[\n' >>$report
+			    $diff $seq.out $seqres.out.bad >>$report
+			    printf ']]>\n'  >>$report
+			    echo -e "\t\t\t</system-err>" >> $report
+			fi
+			echo -e "\t\t</failure>" >> $report
+			;;
+		*)
+			echo -e "\t\t<failure message=\"Unknown ret_state=$ret_state\" type=\"TestFail\"/>" >> $report
+			;;
+	esac
+	echo -e "\t</testcase>" >> $report
+}
+
+
+#
+#  Common report generator entry points
+_make_section_report()
+{
+	if [ ! -z "$REPORT_XUNIT" ]; then
+	    _xunit_make_section_report
+	fi
+}
+
+_make_testcase_report()
+{
+	test_status="$1"
+	if [ ! -z "$REPORT_XUNIT" ]; then
+	    _xunit_make_testcase_report "$test_status"
+	fi
+}