[1/3] auto-t: make test timeout configurable

Commit Message

James Prestwood Nov. 13, 2023, 2:32 p.m. UTC

With the addition of DPP PKEX autotests some of the timeouts are
quite long and hit test-runners maximum timeouts. For UML we should
allow this since time-travel lets us skip idle waits. Move the test
timeout out of a global define and into the argument list so QEMU
and UML can define it differently.
---
 tools/run-tests | 12 +++++-------
 tools/runner.py |  7 +++++++
 2 files changed, 12 insertions(+), 7 deletions(-)

Comments

Denis Kenzior Nov. 13, 2023, 3:48 p.m. UTC | #1

Hi James,

On 11/13/23 08:32, James Prestwood wrote:
> With the addition of DPP PKEX autotests some of the timeouts are
> quite long and hit test-runners maximum timeouts. For UML we should
> allow this since time-travel lets us skip idle waits. Move the test
> timeout out of a global define and into the argument list so QEMU
> and UML can define it differently.
> ---
>   tools/run-tests | 12 +++++-------
>   tools/runner.py |  7 +++++++
>   2 files changed, 12 insertions(+), 7 deletions(-)
> 

All applied, thanks.

Regards,
-Denis

Patch

diff --git a/tools/run-tests b/tools/run-tests
index 32c09723..11ad73c0 100755
--- a/tools/run-tests
+++ b/tools/run-tests
@@ -27,8 +27,6 @@  from utils import Process, Namespace, BarChart
 config = None
 intf_id = 0
 
-TEST_MAX_TIMEOUT = 240
-
 dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
 
 def dbg(*s, **kwargs):
@@ -52,7 +50,7 @@  def exit_vm():
 			p.kill()
 
 		if config.ctx and config.ctx.results:
-			success = print_results(config.ctx.results)
+			success = print_results(config.ctx.results, int(config.ctx.args.timeout))
 
 			if config.ctx.args.result:
 				write_results(config.ctx.args.result, config.ctx.results)
@@ -917,7 +915,7 @@  def post_test(ctx, to_copy):
 		except:
 			pass
 
-def print_results(results):
+def print_results(results, max_timeout):
 	table = PrettyTable(['Test', colored('Passed', 'green'), colored('Failed', 'red'), \
 				colored('Skipped', 'cyan'), colored('Time', 'yellow')])
 
@@ -928,7 +926,7 @@  def print_results(results):
 
 	for test, result in results.items():
 
-		if result.time == TEST_MAX_TIMEOUT:
+		if result.time == max_timeout:
 			failed = "Timed out"
 			passed = "Timed out"
 		elif result.time == 0:
@@ -975,7 +973,7 @@  def run_auto_tests(ctx, args):
 			p.start()
 			# Rather than time each subtest we just time the total but
 			# mutiply the default time by the number of tests being run.
-			p.join(TEST_MAX_TIMEOUT * len(subtests))
+			p.join(int(args.timeout) * len(subtests))
 
 			if p.is_alive():
 				# Timeout
@@ -983,7 +981,7 @@  def run_auto_tests(ctx, args):
 
 				ctx.results[os.path.basename(test)] = SimpleResult(run=0,
 								failures=0, errors=0,
-								skipped=0, time=TEST_MAX_TIMEOUT)
+								skipped=0, time=int(args.timeout))
 			else:
 				ctx.results[os.path.basename(test)] = rqueue.get()
 
diff --git a/tools/runner.py b/tools/runner.py
index 004bf46d..03f44611 100644
--- a/tools/runner.py
+++ b/tools/runner.py
@@ -120,9 +120,12 @@  class RunnerCoreArgParse(ArgumentParser):
 				const=True,
 				action='store',
 				help='Use physical adapters for tests (passthrough)')
+
+		# Hidden options only meant to be passed to the kernel
 		self.add_argument('--testhome', help=SUPPRESS)
 		self.add_argument('--monitor-parent', help=SUPPRESS)
 		self.add_argument('--result-parent', help=SUPPRESS)
+		self.add_argument('--timeout', help=SUPPRESS)
 
 		# Prevent --autotest/--unittest from being used together
 		auto_unit_group = self.add_mutually_exclusive_group()
@@ -385,6 +388,8 @@  class QemuRunner(RunnerAbstract):
 
 		self._prepare_outfiles()
 
+		self.args.timeout = 240
+
 		if args.hw:
 			if os.path.isfile(args.hw):
 				hw_conf = ConfigParser()
@@ -543,6 +548,8 @@  class UmlRunner(RunnerAbstract):
 
 		self._prepare_outfiles()
 
+		self.args.timeout = 1000
+
 		kern_log = "ignore_loglevel" if "kernel" in args.verbose else "quiet"
 
 		cmd = [args.kernel, 'rootfstype=hostfs', 'ro', 'mem=256M', 'mac80211_hwsim.radios=0',