diff mbox series

[6/8] report: collect basic information about a test run

Message ID 167149449737.332657.1308561091226926848.stgit@magnolia (mailing list archive)
State New, archived
Headers show
Series fstests: improve junit xml reporting | expand

Commit Message

Darrick J. Wong Dec. 20, 2022, 12:01 a.m. UTC 
From: Darrick J. Wong <djwong@kernel.org>

Record various generic information about an fstests run when generating
a junit xml report.  This includes the cpu architecture, the kernel
revision, the CPU, memory, and numa node counts, and some information
about the block devices passed in.

Signed-off-by: Darrick J. Wong <djwong@kernel.org>
---
 common/report |   44 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 1 deletion(-)

Comments

Theodore Ts'o Dec. 20, 2022, 3:29 a.m. UTC | #1 
On Mon, Dec 19, 2022 at 04:01:37PM -0800, Darrick J. Wong wrote:
> From: Darrick J. Wong <djwong@kernel.org>
> 
> Record various generic information about an fstests run when generating
> a junit xml report.  This includes the cpu architecture, the kernel
> revision, the CPU, memory, and numa node counts, and some information
> about the block devices passed in.

It would be nice if there was a way that the test runner could pass
information that would be added to the xunit properties.  As I
mentioned in another e-mail, I currently do this via a post-processing
step which adds the properties to the junit xml file via a python
script.  And there are a number of additional properties that are used
by my report generator[1] which takes the junit xml file as input, and
generates a summary report which is convenient for humans.

[1] https://github.com/tytso/xfstests-bld/blob/master/test-appliance/files/usr/local/bin/gen_results_summary

Some of these properties include the version of xfstests, xfsprogs,
and other key software components (for example, I've had test failures
traced to bugs in fio, so knowing the version of fio that is used is
super-handy).

So maybe we could pass in a properties file, either via a command-line
option or an environment variable?  My script[2] uses a colon
separated format, but I'm not wedded to that delimiter.

CMDLINE: "-c f2fs/default -g auto"
FSTESTIMG: gce-xfstests/xfstests-amd64-202212131454
FSTESTPRJ: gce-xfstests
KERNEL: kernel	6.1.0-xfstests #2 SMP PREEMPT_DYNAMIC Mon Dec 12 16:09:40 EST 2022 x86_64
FSTESTVER: blktests	068bd2a (Fri, 18 Nov 2022 08:38:35 +0900)
FSTESTVER: fio		fio-3.31 (Tue, 9 Aug 2022 14:41:25 -0600)
FSTESTVER: fsverity	v1.5 (Sun, 6 Feb 2022 10:59:13 -0800)
FSTESTVER: ima-evm-utils	v1.3.2 (Wed, 28 Oct 2020 13:18:08 -0400)
FSTESTVER: nvme-cli	v1.16 (Thu, 11 Nov 2021 13:09:06 -0800)
FSTESTVER: quota		v4.05-52-gf7e24ee (Tue, 1 Nov 2022 11:45:06 +0100)
FSTESTVER: util-linux	v2.38.1 (Thu, 4 Aug 2022 11:06:21 +0200)
FSTESTVER: xfsprogs	v6.0.0 (Mon, 14 Nov 2022 12:06:23 +0100)
FSTESTVER: xfstests-bld	65edab38 (Wed, 30 Nov 2022 12:11:57 -0500)
FSTESTVER: xfstests	v2022.11.27-8-g3c178050c (Wed, 30 Nov 2022 10:25:39 -0500)
FSTESTVER: zz_build-distro	bullseye
FSTESTCFG: "f2fs/default"
FSTESTSET: "-g auto"
FSTESTEXC: ""
FSTESTOPT: "aex"
MNTOPTS: ""
CPUS: "2"
MEM: "7680"
DMI_MEM: 8 GB (Max capacity)
PARAM_MEM: 7680 (restricted by cmdline)
GCE ID: "3198461547210171740"
MACHINE TYPE: "e2-standard-2"
TESTRUNID: tytso-20221213150813

[2] https://github.com/tytso/xfstests-bld/blob/master/test-appliance/files/usr/local/bin/update_properties_xunit

Cheers,

					- Ted
Darrick J. Wong Feb. 14, 2023, 6:59 p.m. UTC | #2 
On Mon, Dec 19, 2022 at 10:29:58PM -0500, Theodore Ts'o wrote:
> On Mon, Dec 19, 2022 at 04:01:37PM -0800, Darrick J. Wong wrote:
> > From: Darrick J. Wong <djwong@kernel.org>
> > 
> > Record various generic information about an fstests run when generating
> > a junit xml report.  This includes the cpu architecture, the kernel
> > revision, the CPU, memory, and numa node counts, and some information
> > about the block devices passed in.
> 
> It would be nice if there was a way that the test runner could pass
> information that would be added to the xunit properties.  As I
> mentioned in another e-mail, I currently do this via a post-processing
> step which adds the properties to the junit xml file via a python
> script.  And there are a number of additional properties that are used
> by my report generator[1] which takes the junit xml file as input, and
> generates a summary report which is convenient for humans.
> 
> [1] https://github.com/tytso/xfstests-bld/blob/master/test-appliance/files/usr/local/bin/gen_results_summary
> 
> Some of these properties include the version of xfstests, xfsprogs,
> and other key software components (for example, I've had test failures
> traced to bugs in fio, so knowing the version of fio that is used is
> super-handy).
> 
> So maybe we could pass in a properties file, either via a command-line
> option or an environment variable?  My script[2] uses a colon
> separated format, but I'm not wedded to that delimiter.
> 
> CMDLINE: "-c f2fs/default -g auto"
> FSTESTIMG: gce-xfstests/xfstests-amd64-202212131454
> FSTESTPRJ: gce-xfstests
> KERNEL: kernel	6.1.0-xfstests #2 SMP PREEMPT_DYNAMIC Mon Dec 12 16:09:40 EST 2022 x86_64
> FSTESTVER: blktests	068bd2a (Fri, 18 Nov 2022 08:38:35 +0900)
> FSTESTVER: fio		fio-3.31 (Tue, 9 Aug 2022 14:41:25 -0600)
> FSTESTVER: fsverity	v1.5 (Sun, 6 Feb 2022 10:59:13 -0800)
> FSTESTVER: ima-evm-utils	v1.3.2 (Wed, 28 Oct 2020 13:18:08 -0400)
> FSTESTVER: nvme-cli	v1.16 (Thu, 11 Nov 2021 13:09:06 -0800)
> FSTESTVER: quota		v4.05-52-gf7e24ee (Tue, 1 Nov 2022 11:45:06 +0100)
> FSTESTVER: util-linux	v2.38.1 (Thu, 4 Aug 2022 11:06:21 +0200)
> FSTESTVER: xfsprogs	v6.0.0 (Mon, 14 Nov 2022 12:06:23 +0100)
> FSTESTVER: xfstests-bld	65edab38 (Wed, 30 Nov 2022 12:11:57 -0500)
> FSTESTVER: xfstests	v2022.11.27-8-g3c178050c (Wed, 30 Nov 2022 10:25:39 -0500)

Do you want the version numbers of each dependency to have a unique
name attribute here?

<property name="FSTESTVER: xfstests" value="v2022.11.27-8-g3c178050c..."/>

Though ... technically speaking, the @name attributes aren't required to
be unique, so this is valid:

<property name="FSTESTVER" value="xfstests-bld 65edab38..."/>
<property name="FSTESTVER" value="xfstests v2022.11.27-8-g3c178050c..."/>

Or I could go with what I've been rambling about on the ext4 concall for
some time now:  set EXTRA_REPORT_VARS to a path to a file containing
"name: value" strings, one per line, split on the colon.  You all can
translate this into such a format however you like. :)

--D

> FSTESTVER: zz_build-distro	bullseye
> FSTESTCFG: "f2fs/default"
> FSTESTSET: "-g auto"
> FSTESTEXC: ""
> FSTESTOPT: "aex"
> MNTOPTS: ""
> CPUS: "2"
> MEM: "7680"
> DMI_MEM: 8 GB (Max capacity)
> PARAM_MEM: 7680 (restricted by cmdline)
> GCE ID: "3198461547210171740"
> MACHINE TYPE: "e2-standard-2"
> TESTRUNID: tytso-20221213150813
> 
> [2] https://github.com/tytso/xfstests-bld/blob/master/test-appliance/files/usr/local/bin/update_properties_xunit
> 
> Cheers,
> 
> 					- Ted
diff mbox series

Patch

diff --git a/common/report b/common/report
index 642e0426a6..68646a7709 100644
--- a/common/report
+++ b/common/report
@@ -19,6 +19,42 @@  encode_xml()
 		-e 's/"/\&quot;/g'
 }
 
+# Fill out REPORT_VARS with information about the block device referred to by
+# the passed-in bash variable.
+__generate_blockdev_report_vars() {
+	local bdev_var="$1"
+	local bdev="${!bdev_var}"
+
+	test -z "$bdev" && return
+	test -b "$bdev" || return
+	local sysfs_bdev="$(_sysfs_dev "$bdev")"
+
+	REPORT_VARS["${bdev_var}_SIZE_KB"]="$(( "$(blockdev --getsz "$bdev")" / 2 ))"
+	REPORT_VARS["${bdev_var}_ROTATIONAL"]="$(cat "$sysfs_bdev/queue/rotational" 2>/dev/null)"
+	REPORT_VARS["${bdev_var}_DAX"]="$(cat "$sysfs_bdev/queue/dax" 2>/dev/null)"
+	REPORT_VARS["${bdev_var}_DISCARD"]="$(sed -e 's/[1-9][0-9]*/1/g' "$sysfs_bdev/queue/discard_max_bytes" 2>/dev/null)"
+	REPORT_VARS["${bdev_var}_WRITE_ZEROES"]="$(sed -e 's/[1-9][0-9]*/1/g' "$sysfs_bdev/queue/write_zeroes_max_bytes" 2>/dev/null)"
+	REPORT_VARS["${bdev_var}_PHYS_BLOCK_BYTES"]="$(cat "$sysfs_bdev/queue/physical_block_size" 2>/dev/null)"
+	REPORT_VARS["${bdev_var}_LBA_BYTES"]="$(cat "$sysfs_bdev/queue/logical_block_size" 2>/dev/null)"
+	REPORT_VARS["${bdev_var}_ZONES"]="$(cat "$sysfs_bdev/queue/nr_zones" 2>/dev/null)"
+}
+
+# Fill out REPORT_VARS with tidbits about our test runner configuration.
+# Caller is required to declare REPORT_VARS to be an associative array.
+__generate_report_vars() {
+	REPORT_VARS["ARCH"]="$(uname -m)"
+	REPORT_VARS["KERNEL"]="$(uname -r)"
+	REPORT_VARS["CPUS"]="$(getconf _NPROCESSORS_ONLN 2>/dev/null)"
+	REPORT_VARS["MEM_KB"]="$(grep MemTotal: /proc/meminfo | awk '{print $2}')"
+	REPORT_VARS["SWAP_KB"]="$(grep SwapTotal: /proc/meminfo | awk '{print $2}')"
+
+	test -e /sys/devices/system/node/possible && \
+		REPORT_VARS["NUMA_NODES"]="$(cat /sys/devices/system/node/possible 2>/dev/null)"
+
+	__generate_blockdev_report_vars "TEST_DEV"
+	__generate_blockdev_report_vars "SCRATCH_DEV"
+}
+
 #
 # Xunit format report functions
 _xunit_add_property()
@@ -64,11 +100,17 @@  _xunit_make_section_report()
  hostname="$HOST" timestamp="$timestamp">
 ENDL
 
+	declare -A REPORT_VARS
+	__generate_report_vars
+
 	# Properties
 	echo -e "\t<properties>" >> $REPORT_DIR/result.xml
+	(for key in "${!REPORT_VARS[@]}"; do
+		_xunit_add_property "$key" "${REPORT_VARS["$key"]}"
+	done;
 	for p in "${REPORT_ENV_LIST[@]}"; do
 		_xunit_add_property "$p" "${!p}"
-	done | sort >> $REPORT_DIR/result.xml
+	done) | sort >> $REPORT_DIR/result.xml
 	echo -e "\t</properties>" >> $REPORT_DIR/result.xml
 	if [ -f $report ]; then
 		cat $report >> $REPORT_DIR/result.xml