diff mbox

[i-g-t,2/3,v3] Unify handling of slow/combinatorial tests

Message ID 1446211111-3940-3-git-send-email-david.weinehall@linux.intel.com (mailing list archive)
State New, archived
Headers show

Commit Message

David Weinehall Oct. 30, 2015, 1:18 p.m. UTC
Some subtests are not run by default, for various reasons;
be it because they're only for debugging, because they're slow,
or because they are not of high enough quality.

This patch aims to introduce a common mechanism for categorising
the subtests and introduces a flag (--all) that runs/lists all
subtests instead of just the default set.

Signed-off-by: David Weinehall <david.weinehall@linux.intel.com>
---
 lib/igt_core.c                   |  43 ++++++--
 lib/igt_core.h                   |  42 ++++++++
 tests/gem_concurrent_blit.c      |  50 +++++-----
 tests/kms_frontbuffer_tracking.c | 207 ++++++++++++++++++++++-----------------
 4 files changed, 218 insertions(+), 124 deletions(-)

Comments

Chris Wilson Oct. 30, 2015, 1:52 p.m. UTC | #1
On Fri, Oct 30, 2015 at 03:18:30PM +0200, David Weinehall wrote:
> @@ -931,16 +930,20 @@ run_basic_modes(const struct access_mode *mode,
>  	struct buffers buffers;
>  
>  	for (h = hangs; h->suffix; h++) {
> -		if (!all && *h->suffix)
> -			continue;
> +		unsigned int subtest_flags;
>  
> -		for (p = all ? pipelines : pskip; p->prefix; p++) {
> +		if (*h->suffix)
> +			subtest_flags = SUBTEST_TYPE_SLOW;

They aren't all slow though. The hang tests are (because it takes a long
time for a hang to occur and we need to race many times for reasonable
coverage). Many of the tests here were being skipped because QA couldn't
handle the full set.

Now that you have flags, adding h->flags would be better than heuristic
based on h->suffix. Similary we can use p->flags, then we get
igt_subtest_flags_f(h->flags | p->flags, "foo"); And if we have more
faith in QA in being able to dtrt we really only need to mark the known
slow cases (many the hang injection) as being truly slow.
-Chris
David Weinehall Nov. 12, 2015, 11 a.m. UTC | #2
On Fri, Oct 30, 2015 at 01:52:48PM +0000, Chris Wilson wrote:
> On Fri, Oct 30, 2015 at 03:18:30PM +0200, David Weinehall wrote:
> > @@ -931,16 +930,20 @@ run_basic_modes(const struct access_mode *mode,
> >  	struct buffers buffers;
> >  
> >  	for (h = hangs; h->suffix; h++) {
> > -		if (!all && *h->suffix)
> > -			continue;
> > +		unsigned int subtest_flags;
> >  
> > -		for (p = all ? pipelines : pskip; p->prefix; p++) {
> > +		if (*h->suffix)
> > +			subtest_flags = SUBTEST_TYPE_SLOW;
> 
> They aren't all slow though. The hang tests are (because it takes a long
> time for a hang to occur and we need to race many times for reasonable
> coverage). Many of the tests here were being skipped because QA couldn't
> handle the full set.

Of course. But unlike the creators of the tests in question I have very
little knowledge about why the tests in question weren't included in the
standard test set.  Sorting tests into slow/cornercase/whatever is
something that should be done by people who properly know what
categories the bugs best belong to.


Kind regards, David
diff mbox

Patch

diff --git a/lib/igt_core.c b/lib/igt_core.c
index 59127cafe606..2034bc33ad78 100644
--- a/lib/igt_core.c
+++ b/lib/igt_core.c
@@ -216,6 +216,7 @@  const char *igt_interactive_debug;
 
 /* subtests helpers */
 static bool list_subtests = false;
+static unsigned int subtest_types_mask = SUBTEST_TYPE_NORMAL;
 static char *run_single_subtest = NULL;
 static bool run_single_subtest_found = false;
 static const char *in_subtest = NULL;
@@ -234,12 +235,13 @@  int test_children_sz;
 bool test_child;
 
 enum {
- OPT_LIST_SUBTESTS,
- OPT_RUN_SUBTEST,
- OPT_DESCRIPTION,
- OPT_DEBUG,
- OPT_INTERACTIVE_DEBUG,
- OPT_HELP = 'h'
+	OPT_LIST_SUBTESTS,
+	OPT_WITH_ALL_SUBTESTS,
+	OPT_RUN_SUBTEST,
+	OPT_DESCRIPTION,
+	OPT_DEBUG,
+	OPT_INTERACTIVE_DEBUG,
+	OPT_HELP = 'h'
 };
 
 static int igt_exitcode = IGT_EXIT_SUCCESS;
@@ -478,6 +480,7 @@  static void print_usage(const char *help_str, bool output_on_stderr)
 
 	fprintf(f, "Usage: %s [OPTIONS]\n", command_str);
 	fprintf(f, "  --list-subtests\n"
+		   "  --all\n"
 		   "  --run-subtest <pattern>\n"
 		   "  --debug[=log-domain]\n"
 		   "  --interactive-debug[=domain]\n"
@@ -510,6 +513,7 @@  static int common_init(int *argc, char **argv,
 	int c, option_index = 0, i, x;
 	static struct option long_options[] = {
 		{"list-subtests", 0, 0, OPT_LIST_SUBTESTS},
+		{"all", 0, 0, OPT_WITH_ALL_SUBTESTS},
 		{"run-subtest", 1, 0, OPT_RUN_SUBTEST},
 		{"help-description", 0, 0, OPT_DESCRIPTION},
 		{"debug", optional_argument, 0, OPT_DEBUG},
@@ -617,6 +621,10 @@  static int common_init(int *argc, char **argv,
 			if (!run_single_subtest)
 				list_subtests = true;
 			break;
+		case OPT_WITH_ALL_SUBTESTS:
+			if (!run_single_subtest)
+				subtest_types_mask = SUBTEST_TYPE_ALL;
+			break;
 		case OPT_RUN_SUBTEST:
 			if (!list_subtests)
 				run_single_subtest = strdup(optarg);
@@ -1629,6 +1637,29 @@  void igt_skip_on_simulation(void)
 		igt_require(!igt_run_in_simulation());
 }
 
+/**
+ * igt_match_subtest_flags:
+ *
+ * This function is used to check whether the attributes of the subtest
+ * makes it a candidate for inclusion in the test run; this is used to
+ * categorise tests, for instance to exclude tests that are purely for
+ * debug purposes, tests that are specific to certain environments,
+ * or tests that are very slow.
+ *
+ * Note that a test has to have all its flags met to be run; for instance
+ * a subtest with the flags SUBTEST_TYPE_SLOW | SUBTEST_TYPE_DEBUG requires
+ * "--subtest-types=slow,debug" or "--all" to be executed
+ *
+ * @subtest_flags: The subtests to check for
+ *
+ * Returns: true if the subtest test should be run,
+ *          false if the subtest should be skipped
+ */
+bool igt_match_subtest_flags(unsigned long subtest_flags)
+{
+	return ((subtest_flags & subtest_types_mask) == subtest_flags);
+}
+
 /* structured logging */
 
 /**
diff --git a/lib/igt_core.h b/lib/igt_core.h
index 5ae09653fd55..495cb77a8aea 100644
--- a/lib/igt_core.h
+++ b/lib/igt_core.h
@@ -191,6 +191,48 @@  bool __igt_run_subtest(const char *subtest_name);
 #define igt_subtest_f(f...) \
 	__igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
 
+enum {
+	/* The set of tests run if nothing else is specified */
+	SUBTEST_TYPE_NORMAL = 1 << 0,
+	/* Basic Acceptance Testing set */
+	SUBTEST_TYPE_BASIC = 1 << 1,
+	/* Tests that are very slow */
+	SUBTEST_TYPE_SLOW = 1 << 2,
+	/* Tests that mainly intended for debugging */
+	SUBTEST_TYPE_DEBUG = 1 << 3,
+	SUBTEST_TYPE_ALL = ~0
+} subtest_types;
+
+bool igt_match_subtest_flags(unsigned long subtest_flags);
+
+/**
+ * igt_subtest_flags:
+ * @name: name of the subtest
+ * @__subtest_flags: the categories the subtest belongs to
+ *
+ * This is a wrapper around igt_subtest that will only execute the
+ * testcase if all of the flags passed to this function match those
+ * specified by the list of subtest categories passed from the
+ * command line; the default category is SUBTEST_TYPE_NORMAL.
+ */
+#define igt_subtest_flags(name, __subtest_flags) \
+	if (igt_match_subtest_flags(__subtest_flags)) \
+		igt_subtest(name)
+
+/**
+ * igt_subtest_flags_f:
+ * @...: format string and optional arguments
+ * @__subtest_flags: the categories the subtest belongs to
+ *
+ * This is a wrapper around igt_subtest_f that will only execute the
+ * testcase if all of the flags passed to this function match those
+ * specified by the list of subtest categories passed from the
+ * command line; the default category is SUBTEST_TYPE_NORMAL.
+ */
+#define igt_subtest_flags_f(__subtest_flags, f...) \
+	if (igt_match_subtest_flags(__subtest_flags)) \
+		__igt_subtest_f(igt_tokencat(__tmpchar, __LINE__), f)
+
 const char *igt_subtest_name(void);
 bool igt_only_list_subtests(void);
 
diff --git a/tests/gem_concurrent_blit.c b/tests/gem_concurrent_blit.c
index 1d2d787202df..a4be0e79fbd1 100644
--- a/tests/gem_concurrent_blit.c
+++ b/tests/gem_concurrent_blit.c
@@ -55,7 +55,6 @@  IGT_TEST_DESCRIPTION("Test of pread/pwrite/mmap behavior when writing to active"
 
 int fd, devid, gen;
 struct intel_batchbuffer *batch;
-int all;
 
 static void
 nop_release_bo(drm_intel_bo *bo)
@@ -931,16 +930,20 @@  run_basic_modes(const struct access_mode *mode,
 	struct buffers buffers;
 
 	for (h = hangs; h->suffix; h++) {
-		if (!all && *h->suffix)
-			continue;
+		unsigned int subtest_flags;
 
-		for (p = all ? pipelines : pskip; p->prefix; p++) {
+		if (*h->suffix)
+			subtest_flags = SUBTEST_TYPE_SLOW;
+		else
+			subtest_flags = SUBTEST_TYPE_NORMAL;
+
+		for (p = igt_match_subtest_flags(SUBTEST_TYPE_SLOW) ? pipelines : pskip; p->prefix; p++) {
 			igt_fixture {
 				batch = buffers_init(&buffers, mode, fd);
 			}
 
 			/* try to overwrite the source values */
-			igt_subtest_f("%s-%s-overwrite-source-one%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-overwrite-source-one%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				buffers_create(&buffers, num_buffers);
@@ -949,7 +952,7 @@  run_basic_modes(const struct access_mode *mode,
 					      p->copy, h->hang);
 			}
 
-			igt_subtest_f("%s-%s-overwrite-source%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-overwrite-source%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				buffers_create(&buffers, num_buffers);
@@ -958,7 +961,7 @@  run_basic_modes(const struct access_mode *mode,
 					      p->copy, h->hang);
 			}
 
-			igt_subtest_f("%s-%s-overwrite-source-read-bcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-overwrite-source-read-bcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				buffers_create(&buffers, num_buffers);
@@ -967,7 +970,7 @@  run_basic_modes(const struct access_mode *mode,
 					      p->copy, h->hang);
 			}
 
-			igt_subtest_f("%s-%s-overwrite-source-read-rcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-overwrite-source-read-rcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				igt_require(rendercopy);
@@ -977,7 +980,7 @@  run_basic_modes(const struct access_mode *mode,
 					      p->copy, h->hang);
 			}
 
-			igt_subtest_f("%s-%s-overwrite-source-rev%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-overwrite-source-rev%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				buffers_create(&buffers, num_buffers);
@@ -987,7 +990,7 @@  run_basic_modes(const struct access_mode *mode,
 			}
 
 			/* try to intermix copies with GPU copies*/
-			igt_subtest_f("%s-%s-intermix-rcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-intermix-rcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				igt_require(rendercopy);
@@ -996,7 +999,7 @@  run_basic_modes(const struct access_mode *mode,
 					      do_intermix_rcs,
 					      p->copy, h->hang);
 			}
-			igt_subtest_f("%s-%s-intermix-bcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-intermix-bcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				igt_require(rendercopy);
@@ -1005,7 +1008,7 @@  run_basic_modes(const struct access_mode *mode,
 					      do_intermix_bcs,
 					      p->copy, h->hang);
 			}
-			igt_subtest_f("%s-%s-intermix-both%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-intermix-both%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				igt_require(rendercopy);
@@ -1016,7 +1019,7 @@  run_basic_modes(const struct access_mode *mode,
 			}
 
 			/* try to read the results before the copy completes */
-			igt_subtest_f("%s-%s-early-read%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-early-read%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				buffers_create(&buffers, num_buffers);
@@ -1026,7 +1029,7 @@  run_basic_modes(const struct access_mode *mode,
 			}
 
 			/* concurrent reads */
-			igt_subtest_f("%s-%s-read-read-bcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-read-read-bcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				buffers_create(&buffers, num_buffers);
@@ -1034,7 +1037,7 @@  run_basic_modes(const struct access_mode *mode,
 					      do_read_read_bcs,
 					      p->copy, h->hang);
 			}
-			igt_subtest_f("%s-%s-read-read-rcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			igt_subtest_flags_f(subtest_flags, "%s-%s-read-read-rcs%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				igt_require(rendercopy);
@@ -1044,8 +1047,8 @@  run_basic_modes(const struct access_mode *mode,
 					      p->copy, h->hang);
 			}
 
-			/* and finally try to trick the kernel into loosing the pending write */
-			igt_subtest_f("%s-%s-gpu-read-after-write%s%s", mode->name, p->prefix, suffix, h->suffix) {
+			/* and finally try to trick the kernel into losing the pending write */
+			igt_subtest_flags_f(subtest_flags, "%s-%s-gpu-read-after-write%s%s", mode->name, p->prefix, suffix, h->suffix) {
 				h->require();
 				p->require();
 				buffers_create(&buffers, num_buffers);
@@ -1064,13 +1067,11 @@  run_basic_modes(const struct access_mode *mode,
 static void
 run_modes(const struct access_mode *mode)
 {
-	if (all) {
-		run_basic_modes(mode, "", run_single);
+	run_basic_modes(mode, "", run_single);
 
-		igt_fork_signal_helper();
-		run_basic_modes(mode, "-interruptible", run_interruptible);
-		igt_stop_signal_helper();
-	}
+	igt_fork_signal_helper();
+	run_basic_modes(mode, "-interruptible", run_interruptible);
+	igt_stop_signal_helper();
 
 	igt_fork_signal_helper();
 	run_basic_modes(mode, "-forked", run_forked);
@@ -1083,9 +1084,6 @@  igt_main
 
 	igt_skip_on_simulation();
 
-	if (strstr(igt_test_name(), "all"))
-		all = true;
-
 	igt_fixture {
 		fd = drm_open_driver(DRIVER_INTEL);
 		devid = intel_get_drm_devid(fd);
diff --git a/tests/kms_frontbuffer_tracking.c b/tests/kms_frontbuffer_tracking.c
index 15707b9b9040..39f3c37a8f00 100644
--- a/tests/kms_frontbuffer_tracking.c
+++ b/tests/kms_frontbuffer_tracking.c
@@ -47,8 +47,7 @@  IGT_TEST_DESCRIPTION("Test the Kernel's frontbuffer tracking mechanism and "
  * combinations that are somewhat redundant and don't add much value to the
  * test. For example, since we already do the offscreen testing with a single
  * pipe enabled, there's no much value in doing it again with dual pipes. If you
- * still want to try these redundant tests, you need to use the --show-hidden
- * option.
+ * still want to try these redundant tests, you need to use the --all option.
  *
  * The most important hidden thing is the FEATURE_NONE set of tests. Whenever
  * you get a failure on any test, it is important to check whether the same test
@@ -116,6 +115,9 @@  struct test_mode {
 	} format;
 
 	enum igt_draw_method method;
+
+	/* Specifies the subtest categories this subtest belongs to */
+	unsigned long subtest_flags;
 };
 
 enum flip_type {
@@ -237,7 +239,6 @@  struct {
 	bool fbc_check_last_action;
 	bool no_edp;
 	bool small_modes;
-	bool show_hidden;
 	int step;
 	int only_pipes;
 	int shared_fb_x_offset;
@@ -249,7 +250,6 @@  struct {
 	.fbc_check_last_action = true,
 	.no_edp = false,
 	.small_modes = false,
-	.show_hidden= false,
 	.step = 0,
 	.only_pipes = PIPE_COUNT,
 	.shared_fb_x_offset = 500,
@@ -2933,9 +2933,6 @@  static int opt_handler(int option, int option_index, void *data)
 	case 'm':
 		opt.small_modes = true;
 		break;
-	case 'i':
-		opt.show_hidden = true;
-		break;
 	case 't':
 		opt.step++;
 		break;
@@ -2971,7 +2968,6 @@  const char *help_str =
 "  --no-fbc-action-check       Don't check for the FBC last action\n"
 "  --no-edp                    Don't use eDP monitors\n"
 "  --use-small-modes           Use smaller resolutions for the modes\n"
-"  --show-hidden               Show hidden subtests\n"
 "  --step                      Stop on each step so you can check the screen\n"
 "  --shared-fb-x offset        Use 'offset' as the X offset for the shared FB\n"
 "  --shared-fb-y offset        Use 'offset' as the Y offset for the shared FB\n"
@@ -3068,18 +3064,19 @@  static const char *format_str(enum pixel_format format)
 	for (t.plane = 0; t.plane < PLANE_COUNT; t.plane++) {		   \
 	for (t.fbs = 0; t.fbs < FBS_COUNT; t.fbs++) {			   \
 	for (t.method = 0; t.method < IGT_DRAW_METHOD_COUNT; t.method++) { \
+		t.subtest_flags = SUBTEST_TYPE_NORMAL;			   \
 		if (t.pipes == PIPE_SINGLE && t.screen == SCREEN_SCND)	   \
 			continue;					   \
 		if (t.screen == SCREEN_OFFSCREEN && t.plane != PLANE_PRI)  \
 			continue;					   \
-		if (!opt.show_hidden && t.pipes == PIPE_DUAL &&		   \
+		if (t.pipes == PIPE_DUAL &&				   \
 		    t.screen == SCREEN_OFFSCREEN)			   \
-			continue;					   \
-		if (!opt.show_hidden && t.feature == FEATURE_NONE)	   \
-			continue;					   \
-		if (!opt.show_hidden && t.fbs == FBS_SHARED &&		   \
+			t.subtest_flags = SUBTEST_TYPE_SLOW;					   \
+		if (t.feature == FEATURE_NONE)				   \
+			t.subtest_flags = SUBTEST_TYPE_SLOW;					   \
+		if (t.fbs == FBS_SHARED &&				   \
 		    (t.plane == PLANE_CUR || t.plane == PLANE_SPR))	   \
-			continue;
+			t.subtest_flags = SUBTEST_TYPE_SLOW;
 
 
 #define TEST_MODE_ITER_END } } } } } }
@@ -3094,7 +3091,6 @@  int main(int argc, char *argv[])
 		{ "no-fbc-action-check",      0, 0, 'a'},
 		{ "no-edp",                   0, 0, 'e'},
 		{ "use-small-modes",          0, 0, 'm'},
-		{ "show-hidden",              0, 0, 'i'},
 		{ "step",                     0, 0, 't'},
 		{ "shared-fb-x",              1, 0, 'x'},
 		{ "shared-fb-y",              1, 0, 'y'},
@@ -3110,8 +3106,9 @@  int main(int argc, char *argv[])
 		setup_environment();
 
 	for (t.feature = 0; t.feature < FEATURE_COUNT; t.feature++) {
-		if (!opt.show_hidden && t.feature == FEATURE_NONE)
-			continue;
+		t.subtest_flags = SUBTEST_TYPE_NORMAL;
+		if (t.feature == FEATURE_NONE)
+			t.subtest_flags = SUBTEST_TYPE_SLOW;
 		for (t.pipes = 0; t.pipes < PIPE_COUNT; t.pipes++) {
 			t.screen = SCREEN_PRIM;
 			t.plane = PLANE_PRI;
@@ -3120,52 +3117,58 @@  int main(int argc, char *argv[])
 			/* Make sure nothing is using this value. */
 			t.method = -1;
 
-			igt_subtest_f("%s-%s-rte",
-				      feature_str(t.feature),
-				      pipes_str(t.pipes))
+			igt_subtest_flags_f(t.subtest_flags,
+					    "%s-%s-rte",
+					    feature_str(t.feature),
+					    pipes_str(t.pipes))
 				rte_subtest(&t);
 		}
 	}
 
 	TEST_MODE_ITER_BEGIN(t)
-		igt_subtest_f("%s-%s-%s-%s-%s-draw-%s",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      screen_str(t.screen),
-			      plane_str(t.plane),
-			      fbs_str(t.fbs),
-			      igt_draw_get_method_name(t.method))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-%s-draw-%s",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    screen_str(t.screen),
+				    plane_str(t.plane),
+				    fbs_str(t.fbs),
+				    igt_draw_get_method_name(t.method))
 			draw_subtest(&t);
 	TEST_MODE_ITER_END
 
 	TEST_MODE_ITER_BEGIN(t)
 		if (t.plane != PLANE_PRI ||
-		    t.screen == SCREEN_OFFSCREEN ||
-		    (!opt.show_hidden && t.method != IGT_DRAW_BLT))
+		    t.screen == SCREEN_OFFSCREEN)
 			continue;
-
-		igt_subtest_f("%s-%s-%s-%s-flip-%s",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      screen_str(t.screen),
-			      fbs_str(t.fbs),
-			      igt_draw_get_method_name(t.method))
+		if (t.method != IGT_DRAW_BLT)
+			t.subtest_flags = SUBTEST_TYPE_SLOW;
+
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-flip-%s",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    screen_str(t.screen),
+				    fbs_str(t.fbs),
+				    igt_draw_get_method_name(t.method))
 			flip_subtest(&t, FLIP_PAGEFLIP);
 
-		igt_subtest_f("%s-%s-%s-%s-evflip-%s",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      screen_str(t.screen),
-			      fbs_str(t.fbs),
-			      igt_draw_get_method_name(t.method))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-evflip-%s",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    screen_str(t.screen),
+				    fbs_str(t.fbs),
+				    igt_draw_get_method_name(t.method))
 			flip_subtest(&t, FLIP_PAGEFLIP_EVENT);
 
-		igt_subtest_f("%s-%s-%s-%s-msflip-%s",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      screen_str(t.screen),
-			      fbs_str(t.fbs),
-			      igt_draw_get_method_name(t.method))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-msflip-%s",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    screen_str(t.screen),
+				    fbs_str(t.fbs),
+				    igt_draw_get_method_name(t.method))
 			flip_subtest(&t, FLIP_MODESET);
 
 	TEST_MODE_ITER_END
@@ -3177,10 +3180,11 @@  int main(int argc, char *argv[])
 		    (t.feature & FEATURE_FBC) == 0)
 			continue;
 
-		igt_subtest_f("%s-%s-%s-fliptrack",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      fbs_str(t.fbs))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-fliptrack",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    fbs_str(t.fbs))
 			fliptrack_subtest(&t, FLIP_PAGEFLIP);
 	TEST_MODE_ITER_END
 
@@ -3190,20 +3194,22 @@  int main(int argc, char *argv[])
 		    t.plane == PLANE_PRI)
 			continue;
 
-		igt_subtest_f("%s-%s-%s-%s-%s-move",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      screen_str(t.screen),
-			      plane_str(t.plane),
-			      fbs_str(t.fbs))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-%s-move",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    screen_str(t.screen),
+				    plane_str(t.plane),
+				    fbs_str(t.fbs))
 			move_subtest(&t);
 
-		igt_subtest_f("%s-%s-%s-%s-%s-onoff",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      screen_str(t.screen),
-			      plane_str(t.plane),
-			      fbs_str(t.fbs))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-%s-onoff",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    screen_str(t.screen),
+				    plane_str(t.plane),
+				    fbs_str(t.fbs))
 			onoff_subtest(&t);
 	TEST_MODE_ITER_END
 
@@ -3213,27 +3219,30 @@  int main(int argc, char *argv[])
 		    t.plane != PLANE_SPR)
 			continue;
 
-		igt_subtest_f("%s-%s-%s-%s-%s-fullscreen",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      screen_str(t.screen),
-			      plane_str(t.plane),
-			      fbs_str(t.fbs))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-%s-fullscreen",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    screen_str(t.screen),
+				    plane_str(t.plane),
+				    fbs_str(t.fbs))
 			fullscreen_plane_subtest(&t);
 	TEST_MODE_ITER_END
 
 	TEST_MODE_ITER_BEGIN(t)
 		if (t.screen != SCREEN_PRIM ||
-		    t.method != IGT_DRAW_BLT ||
-		    (!opt.show_hidden && t.plane != PLANE_PRI) ||
-		    (!opt.show_hidden && t.fbs != FBS_INDIVIDUAL))
+		    t.method != IGT_DRAW_BLT)
 			continue;
-
-		igt_subtest_f("%s-%s-%s-%s-multidraw",
-			      feature_str(t.feature),
-			      pipes_str(t.pipes),
-			      plane_str(t.plane),
-			      fbs_str(t.fbs))
+		if (t.plane != PLANE_PRI ||
+		    t.fbs != FBS_INDIVIDUAL)
+			t.subtest_flags = SUBTEST_TYPE_SLOW;
+
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-%s-%s-multidraw",
+				    feature_str(t.feature),
+				    pipes_str(t.pipes),
+				    plane_str(t.plane),
+				    fbs_str(t.fbs))
 			multidraw_subtest(&t);
 	TEST_MODE_ITER_END
 
@@ -3245,7 +3254,9 @@  int main(int argc, char *argv[])
 		    t.method != IGT_DRAW_MMAP_GTT)
 			continue;
 
-		igt_subtest_f("%s-farfromfence", feature_str(t.feature))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-farfromfence",
+				    feature_str(t.feature))
 			farfromfence_subtest(&t);
 	TEST_MODE_ITER_END
 
@@ -3261,10 +3272,11 @@  int main(int argc, char *argv[])
 			if (t.format == FORMAT_DEFAULT)
 				continue;
 
-			igt_subtest_f("%s-%s-draw-%s",
-				      feature_str(t.feature),
-				      format_str(t.format),
-				      igt_draw_get_method_name(t.method))
+			igt_subtest_flags_f(t.subtest_flags,
+					    "%s-%s-draw-%s",
+					    feature_str(t.feature),
+					    format_str(t.format),
+					    igt_draw_get_method_name(t.method))
 				format_draw_subtest(&t);
 		}
 	TEST_MODE_ITER_END
@@ -3275,9 +3287,10 @@  int main(int argc, char *argv[])
 		    t.plane != PLANE_PRI ||
 		    t.method != IGT_DRAW_MMAP_CPU)
 			continue;
-		igt_subtest_f("%s-%s-scaledprimary",
-			      feature_str(t.feature),
-			      fbs_str(t.fbs))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-%s-scaledprimary",
+				    feature_str(t.feature),
+				    fbs_str(t.fbs))
 			scaledprimary_subtest(&t);
 	TEST_MODE_ITER_END
 
@@ -3289,22 +3302,32 @@  int main(int argc, char *argv[])
 		    t.method != IGT_DRAW_MMAP_CPU)
 			continue;
 
-		igt_subtest_f("%s-modesetfrombusy", feature_str(t.feature))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-modesetfrombusy",
+				    feature_str(t.feature))
 			modesetfrombusy_subtest(&t);
 
 		if (t.feature & FEATURE_FBC) {
-			igt_subtest_f("%s-badstride", feature_str(t.feature))
+			igt_subtest_flags_f(t.subtest_flags,
+					    "%s-badstride",
+					    feature_str(t.feature))
 				badstride_subtest(&t);
 
-			igt_subtest_f("%s-stridechange", feature_str(t.feature))
+			igt_subtest_flags_f(t.subtest_flags,
+					    "%s-stridechange",
+					    feature_str(t.feature))
 				stridechange_subtest(&t);
 		}
 
 		if (t.feature & FEATURE_PSR)
-			igt_subtest_f("%s-slowdraw", feature_str(t.feature))
+			igt_subtest_flags_f(t.subtest_flags,
+					    "%s-slowdraw",
+					    feature_str(t.feature))
 				slow_draw_subtest(&t);
 
-		igt_subtest_f("%s-suspend", feature_str(t.feature))
+		igt_subtest_flags_f(t.subtest_flags,
+				    "%s-suspend",
+				    feature_str(t.feature))
 			suspend_subtest(&t);
 	TEST_MODE_ITER_END