diff mbox series

[v2,2/4] automation: add jobs running tests from tools/tests/*

Message ID cafc69b6c01805e7ccc0fcd6ccebe0b7088c4bd5.1739496480.git-series.marmarek@invisiblethingslab.com (mailing list archive)
State Superseded
Headers show
Series Few CI improvements | expand

Commit Message

Marek Marczykowski-Górecki Feb. 14, 2025, 1:28 a.m. UTC 
There are a bunch of tests in tools/tests/, let them run in CI.
For each subdirectory expect "make run" will run the test, and observe
its exit code. This way, adding new tests is easy, and they will be
automatically picked up.

For better visibility, log test output to junit xml format, and let
gitlab ingest it. Set SUT_ADDR variable with name/address of the system
under test, so a network can be used to extract the file. The actual
address is set using DHCP. And for the test internal network, still add
the 192.168.0.1 IP (but don't replace the DHCP-provided one).

Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
---
Changes in v2:
 - use bash shebang
 - clarify skipped message
 - cleanup extra printf params
 - limit calling DHCP in dom0 to only tests that need it
---
 automation/gitlab-ci/test.yaml     | 23 +++++++++++++++-
 automation/scripts/build           |  1 +-
 automation/scripts/qubes-x86-64.sh | 28 ++++++++++++++++++-
 automation/scripts/run-tools-tests | 47 +++++++++++++++++++++++++++++++-
 4 files changed, 99 insertions(+)
 create mode 100755 automation/scripts/run-tools-tests

Comments

Stefano Stabellini Feb. 15, 2025, 12:26 a.m. UTC | #1 
On Fri, 14 Feb 2025, Marek Marczykowski-Górecki wrote:
> There are a bunch of tests in tools/tests/, let them run in CI.
> For each subdirectory expect "make run" will run the test, and observe
> its exit code. This way, adding new tests is easy, and they will be
> automatically picked up.
> 
> For better visibility, log test output to junit xml format, and let
> gitlab ingest it. Set SUT_ADDR variable with name/address of the system
> under test, so a network can be used to extract the file. The actual
> address is set using DHCP. And for the test internal network, still add
> the 192.168.0.1 IP (but don't replace the DHCP-provided one).
> 
> Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>

Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>


> ---
> Changes in v2:
>  - use bash shebang
>  - clarify skipped message
>  - cleanup extra printf params
>  - limit calling DHCP in dom0 to only tests that need it
> ---
>  automation/gitlab-ci/test.yaml     | 23 +++++++++++++++-
>  automation/scripts/build           |  1 +-
>  automation/scripts/qubes-x86-64.sh | 28 ++++++++++++++++++-
>  automation/scripts/run-tools-tests | 47 +++++++++++++++++++++++++++++++-
>  4 files changed, 99 insertions(+)
>  create mode 100755 automation/scripts/run-tools-tests
> 
> diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> index 1822e3ea5fd7..c21a37933881 100644
> --- a/automation/gitlab-ci/test.yaml
> +++ b/automation/gitlab-ci/test.yaml
> @@ -130,6 +130,7 @@
>      PCIDEV: "03:00.0"
>      PCIDEV_INTR: "MSI-X"
>      CONSOLE_OPTS: "console=com1 com1=115200,8n1"
> +    SUT_ADDR: test-2.testnet
>    artifacts:
>      paths:
>        - smoke.serial
> @@ -263,6 +264,28 @@ adl-pvshim-x86-64-gcc-debug:
>      - *x86-64-test-needs
>      - alpine-3.18-gcc-debug
>  
> +adl-tools-tests-pv-x86-64-gcc-debug:
> +  extends: .adl-x86-64
> +  script:
> +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE}
> +  artifacts:
> +    reports:
> +      junit: tests-junit.xml
> +  needs:
> +    - *x86-64-test-needs
> +    - alpine-3.18-gcc-debug
> +
> +adl-tools-tests-pvh-x86-64-gcc-debug:
> +  extends: .adl-x86-64
> +  script:
> +    - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE}
> +  artifacts:
> +    reports:
> +      junit: tests-junit.xml
> +  needs:
> +    - *x86-64-test-needs
> +    - alpine-3.18-gcc-debug
> +
>  zen3p-smoke-x86-64-gcc-debug:
>    extends: .zen3p-x86-64
>    script:
> diff --git a/automation/scripts/build b/automation/scripts/build
> index 952599cc25c2..522efe774ef3 100755
> --- a/automation/scripts/build
> +++ b/automation/scripts/build
> @@ -109,5 +109,6 @@ else
>      # even though dist/ contains everything, while some containers don't even
>      # build Xen
>      cp -r dist binaries/
> +    cp -r tools/tests binaries/
>      collect_xen_artefacts
>  fi
> diff --git a/automation/scripts/qubes-x86-64.sh b/automation/scripts/qubes-x86-64.sh
> index 7eb3ce1bf703..7c80e0c23318 100755
> --- a/automation/scripts/qubes-x86-64.sh
> +++ b/automation/scripts/qubes-x86-64.sh
> @@ -10,6 +10,8 @@ set -ex
>  #  - pci-pv         PV dom0,  PV domU + PCI Passthrough
>  #  - pvshim         PV dom0,  PVSHIM domU
>  #  - s3             PV dom0,  S3 suspend/resume
> +#  - tools-tests-pv PV dom0, run tests from tools/tests/*
> +#  - tools-tests-pvh PVH dom0, run tests from tools/tests/*
>  test_variant=$1
>  
>  ### defaults
> @@ -19,6 +21,7 @@ timeout=120
>  domU_type="pvh"
>  domU_vif="'bridge=xenbr0',"
>  domU_extra_config=
> +retrieve_xml=
>  
>  case "${test_variant}" in
>      ### test: smoke test & smoke test PVH & smoke test HVM & smoke test PVSHIM
> @@ -126,6 +129,21 @@ done
>  "
>          ;;
>  
> +    ### tests: tools-tests-pv, tools-tests-pvh
> +    "tools-tests-pv"|"tools-tests-pvh")
> +        retrieve_xml=1
> +        passed="test passed"
> +        domU_check=""
> +        dom0_check="
> +/tests/run-tools-tests /tests /tmp/tests-junit.xml && echo \"${passed}\"
> +nc -l -p 8080 < /tmp/tests-junit.xml >/dev/null &
> +"
> +        if [ "${test_variant}" = "tools-tests-pvh" ]; then
> +            extra_xen_opts="dom0=pvh"
> +        fi
> +
> +        ;;
> +
>      *)
>          echo "Unrecognised test_variant '${test_variant}'" >&2
>          exit 1
> @@ -178,6 +196,8 @@ mkdir srv
>  mkdir sys
>  rm var/run
>  cp -ar ../binaries/dist/install/* .
> +cp -ar ../binaries/tests .
> +cp -a ../automation/scripts/run-tools-tests tests/
>  
>  echo "#!/bin/bash
>  
> @@ -192,6 +212,10 @@ ifconfig xenbr0 192.168.0.1
>  
>  " > etc/local.d/xen.start
>  
> +if [ -n "$retrieve_xml" ]; then
> +    echo "timeout 30s udhcpc -i xenbr0" >> etc/local.d/xen.start
> +fi
> +
>  if [ -n "$domU_check" ]; then
>      echo "
>  # get domU console content into test log
> @@ -272,6 +296,10 @@ if [ $timeout -le 0 ]; then
>      exit 1
>  fi
>  
> +if [ -n "$retrieve_xml" ]; then
> +    nc -w 10 "$SUT_ADDR" 8080 > tests-junit.xml </dev/null
> +fi
> +
>  sleep 1
>  
>  (grep -q "^Welcome to Alpine Linux" smoke.serial && grep -q "${passed}" smoke.serial) || exit 1
> diff --git a/automation/scripts/run-tools-tests b/automation/scripts/run-tools-tests
> new file mode 100755
> index 000000000000..770e97c3e943
> --- /dev/null
> +++ b/automation/scripts/run-tools-tests
> @@ -0,0 +1,47 @@
> +#!/bin/bash
> +
> +usage() {
> +    echo "Usage: $0 tests-dir xml-out"
> +}
> +
> +xml_out=$2
> +if [ -z "$xml_out" ]; then
> +  xml_out=/dev/null
> +fi
> +printf '<?xml version="1.0" encoding="UTF-8"?>\n' > "$xml_out"
> +printf '<testsuites name="tools.tests">\n' >> "$xml_out"
> +printf ' <testsuite name="tools.tests">\n' >> "$xml_out"
> +failed=
> +for dir in "$1"/*; do
> +    [ -d "$dir" ] || continue
> +    echo "Running test in $dir"
> +    printf '  <testcase name="%s">\n' "$dir" >> "$xml_out"
> +    ret=
> +    for f in "$dir"/*; do
> +        [ -f "$f" ] || continue
> +        [ -x "$f" ] || continue
> +        "$f" 2>&1 | tee /tmp/out
> +        ret=$?
> +        if [ "$ret" -ne 0 ]; then
> +            echo "FAILED: $ret"
> +            failed+=" $dir"
> +            printf '   <failure type="failure" message="binary %s exited with code %d">\n' "$f" "$ret" >> "$xml_out"
> +            # TODO: could use xml escaping... but current tests seems to
> +            # produce sane output
> +            cat /tmp/out >> "$xml_out"
> +            printf '   </failure>\n' >> "$xml_out"
> +        else
> +            echo "PASSED"
> +        fi
> +    done
> +    if [ -z "$ret" ]; then
> +        printf '   <skipped type="skipped" message="no executable test found in %s"/>\n' "$dir" >> "$xml_out"
> +    fi
> +    printf '  </testcase>\n' >> "$xml_out"
> +done
> +printf ' </testsuite>\n' >> "$xml_out"
> +printf '</testsuites>\n' >> "$xml_out"
> +
> +if [ -n "$failed" ]; then
> +    exit 1
> +fi
> -- 
> git-series 0.9.1
>
diff mbox series

Patch

diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 1822e3ea5fd7..c21a37933881 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -130,6 +130,7 @@ 
     PCIDEV: "03:00.0"
     PCIDEV_INTR: "MSI-X"
     CONSOLE_OPTS: "console=com1 com1=115200,8n1"
+    SUT_ADDR: test-2.testnet
   artifacts:
     paths:
       - smoke.serial
@@ -263,6 +264,28 @@  adl-pvshim-x86-64-gcc-debug:
     - *x86-64-test-needs
     - alpine-3.18-gcc-debug
 
+adl-tools-tests-pv-x86-64-gcc-debug:
+  extends: .adl-x86-64
+  script:
+    - ./automation/scripts/qubes-x86-64.sh tools-tests-pv 2>&1 | tee ${LOGFILE}
+  artifacts:
+    reports:
+      junit: tests-junit.xml
+  needs:
+    - *x86-64-test-needs
+    - alpine-3.18-gcc-debug
+
+adl-tools-tests-pvh-x86-64-gcc-debug:
+  extends: .adl-x86-64
+  script:
+    - ./automation/scripts/qubes-x86-64.sh tools-tests-pvh 2>&1 | tee ${LOGFILE}
+  artifacts:
+    reports:
+      junit: tests-junit.xml
+  needs:
+    - *x86-64-test-needs
+    - alpine-3.18-gcc-debug
+
 zen3p-smoke-x86-64-gcc-debug:
   extends: .zen3p-x86-64
   script:
diff --git a/automation/scripts/build b/automation/scripts/build
index 952599cc25c2..522efe774ef3 100755
--- a/automation/scripts/build
+++ b/automation/scripts/build
@@ -109,5 +109,6 @@  else
     # even though dist/ contains everything, while some containers don't even
     # build Xen
     cp -r dist binaries/
+    cp -r tools/tests binaries/
     collect_xen_artefacts
 fi
diff --git a/automation/scripts/qubes-x86-64.sh b/automation/scripts/qubes-x86-64.sh
index 7eb3ce1bf703..7c80e0c23318 100755
--- a/automation/scripts/qubes-x86-64.sh
+++ b/automation/scripts/qubes-x86-64.sh
@@ -10,6 +10,8 @@  set -ex
 #  - pci-pv         PV dom0,  PV domU + PCI Passthrough
 #  - pvshim         PV dom0,  PVSHIM domU
 #  - s3             PV dom0,  S3 suspend/resume
+#  - tools-tests-pv PV dom0, run tests from tools/tests/*
+#  - tools-tests-pvh PVH dom0, run tests from tools/tests/*
 test_variant=$1
 
 ### defaults
@@ -19,6 +21,7 @@  timeout=120
 domU_type="pvh"
 domU_vif="'bridge=xenbr0',"
 domU_extra_config=
+retrieve_xml=
 
 case "${test_variant}" in
     ### test: smoke test & smoke test PVH & smoke test HVM & smoke test PVSHIM
@@ -126,6 +129,21 @@  done
 "
         ;;
 
+    ### tests: tools-tests-pv, tools-tests-pvh
+    "tools-tests-pv"|"tools-tests-pvh")
+        retrieve_xml=1
+        passed="test passed"
+        domU_check=""
+        dom0_check="
+/tests/run-tools-tests /tests /tmp/tests-junit.xml && echo \"${passed}\"
+nc -l -p 8080 < /tmp/tests-junit.xml >/dev/null &
+"
+        if [ "${test_variant}" = "tools-tests-pvh" ]; then
+            extra_xen_opts="dom0=pvh"
+        fi
+
+        ;;
+
     *)
         echo "Unrecognised test_variant '${test_variant}'" >&2
         exit 1
@@ -178,6 +196,8 @@  mkdir srv
 mkdir sys
 rm var/run
 cp -ar ../binaries/dist/install/* .
+cp -ar ../binaries/tests .
+cp -a ../automation/scripts/run-tools-tests tests/
 
 echo "#!/bin/bash
 
@@ -192,6 +212,10 @@  ifconfig xenbr0 192.168.0.1
 
 " > etc/local.d/xen.start
 
+if [ -n "$retrieve_xml" ]; then
+    echo "timeout 30s udhcpc -i xenbr0" >> etc/local.d/xen.start
+fi
+
 if [ -n "$domU_check" ]; then
     echo "
 # get domU console content into test log
@@ -272,6 +296,10 @@  if [ $timeout -le 0 ]; then
     exit 1
 fi
 
+if [ -n "$retrieve_xml" ]; then
+    nc -w 10 "$SUT_ADDR" 8080 > tests-junit.xml </dev/null
+fi
+
 sleep 1
 
 (grep -q "^Welcome to Alpine Linux" smoke.serial && grep -q "${passed}" smoke.serial) || exit 1
diff --git a/automation/scripts/run-tools-tests b/automation/scripts/run-tools-tests
new file mode 100755
index 000000000000..770e97c3e943
--- /dev/null
+++ b/automation/scripts/run-tools-tests
@@ -0,0 +1,47 @@ 
+#!/bin/bash
+
+usage() {
+    echo "Usage: $0 tests-dir xml-out"
+}
+
+xml_out=$2
+if [ -z "$xml_out" ]; then
+  xml_out=/dev/null
+fi
+printf '<?xml version="1.0" encoding="UTF-8"?>\n' > "$xml_out"
+printf '<testsuites name="tools.tests">\n' >> "$xml_out"
+printf ' <testsuite name="tools.tests">\n' >> "$xml_out"
+failed=
+for dir in "$1"/*; do
+    [ -d "$dir" ] || continue
+    echo "Running test in $dir"
+    printf '  <testcase name="%s">\n' "$dir" >> "$xml_out"
+    ret=
+    for f in "$dir"/*; do
+        [ -f "$f" ] || continue
+        [ -x "$f" ] || continue
+        "$f" 2>&1 | tee /tmp/out
+        ret=$?
+        if [ "$ret" -ne 0 ]; then
+            echo "FAILED: $ret"
+            failed+=" $dir"
+            printf '   <failure type="failure" message="binary %s exited with code %d">\n' "$f" "$ret" >> "$xml_out"
+            # TODO: could use xml escaping... but current tests seems to
+            # produce sane output
+            cat /tmp/out >> "$xml_out"
+            printf '   </failure>\n' >> "$xml_out"
+        else
+            echo "PASSED"
+        fi
+    done
+    if [ -z "$ret" ]; then
+        printf '   <skipped type="skipped" message="no executable test found in %s"/>\n' "$dir" >> "$xml_out"
+    fi
+    printf '  </testcase>\n' >> "$xml_out"
+done
+printf ' </testsuite>\n' >> "$xml_out"
+printf '</testsuites>\n' >> "$xml_out"
+
+if [ -n "$failed" ]; then
+    exit 1
+fi