| Message ID | 20201109080747.2408489-1-f4bug@amsat.org (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | [RFC,PATCH-for-5.2] tests/acceptance: Disable Spartan-3A DSP 1800A test | expand |
On 09/11/2020 09.07, Philippe Mathieu-Daudé wrote: > This test is regularly failing on CI :( Do not run it automatically. > > Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> > --- > tests/acceptance/boot_linux_console.py | 1 + > tests/acceptance/replay_kernel.py | 1 + > 2 files changed, 2 insertions(+) > > diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py > index 8f433a67f84..010e8790c0f 100644 > --- a/tests/acceptance/boot_linux_console.py > +++ b/tests/acceptance/boot_linux_console.py > @@ -1025,6 +1025,7 @@ def test_m68k_mcf5208evb(self): > tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c' > self.do_test_advcal_2018('07', tar_hash, 'sanity-clause.elf') > > + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > def test_microblaze_s3adsp1800(self): > """ > :avocado: tags=arch:microblaze > diff --git a/tests/acceptance/replay_kernel.py b/tests/acceptance/replay_kernel.py > index 00c228382bd..c1f5fa4de71 100644 > --- a/tests/acceptance/replay_kernel.py > +++ b/tests/acceptance/replay_kernel.py > @@ -280,6 +280,7 @@ def test_m68k_mcf5208evb(self): > file_path = self.fetch_asset(tar_url, asset_hash=tar_hash) > self.do_test_advcal_2018(file_path, 'sanity-clause.elf') > > + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > def test_microblaze_s3adsp1800(self): > """ > :avocado: tags=arch:microblaze I think this is a recent regression - it hasn't been failing in the past. We should first try to find out why it is failing now before sending it to the @skipUnless nirvana... could you maybe add it to the "Known issues" at https://wiki.qemu.org/Planning/5.2 instead? Thomas
On 11/9/20 10:04 AM, Thomas Huth wrote: > On 09/11/2020 09.07, Philippe Mathieu-Daudé wrote: >> This test is regularly failing on CI :( Do not run it automatically. >> >> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> >> --- >> tests/acceptance/boot_linux_console.py | 1 + >> tests/acceptance/replay_kernel.py | 1 + >> 2 files changed, 2 insertions(+) >> >> diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py >> index 8f433a67f84..010e8790c0f 100644 >> --- a/tests/acceptance/boot_linux_console.py >> +++ b/tests/acceptance/boot_linux_console.py >> @@ -1025,6 +1025,7 @@ def test_m68k_mcf5208evb(self): >> tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c' >> self.do_test_advcal_2018('07', tar_hash, 'sanity-clause.elf') >> >> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >> def test_microblaze_s3adsp1800(self): >> """ >> :avocado: tags=arch:microblaze >> diff --git a/tests/acceptance/replay_kernel.py b/tests/acceptance/replay_kernel.py >> index 00c228382bd..c1f5fa4de71 100644 >> --- a/tests/acceptance/replay_kernel.py >> +++ b/tests/acceptance/replay_kernel.py >> @@ -280,6 +280,7 @@ def test_m68k_mcf5208evb(self): >> file_path = self.fetch_asset(tar_url, asset_hash=tar_hash) >> self.do_test_advcal_2018(file_path, 'sanity-clause.elf') >> >> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >> def test_microblaze_s3adsp1800(self): >> """ >> :avocado: tags=arch:microblaze > > I think this is a recent regression - it hasn't been failing in the past. We > should first try to find out why it is failing now before sending it to the > @skipUnless nirvana... could you maybe add it to the "Known issues" at > https://wiki.qemu.org/Planning/5.2 instead? I agree it looks like a regression. I disagree we should keep broken tests failing the pipeline, even if we are not using a Gating CI. I see test_leon3_helenos_uimage has @skip("Test currently broken") meaning we know there is a problem somewhere, but we don't have clue/resources to fix the bug, so we demote the test to avoid users avoid running CI tests. I'll replace by that. Regards, Phil.
On 09/11/2020 10.09, Philippe Mathieu-Daudé wrote: > On 11/9/20 10:04 AM, Thomas Huth wrote: >> On 09/11/2020 09.07, Philippe Mathieu-Daudé wrote: >>> This test is regularly failing on CI :( Do not run it automatically. >>> >>> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> >>> --- >>> tests/acceptance/boot_linux_console.py | 1 + >>> tests/acceptance/replay_kernel.py | 1 + >>> 2 files changed, 2 insertions(+) >>> >>> diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py >>> index 8f433a67f84..010e8790c0f 100644 >>> --- a/tests/acceptance/boot_linux_console.py >>> +++ b/tests/acceptance/boot_linux_console.py >>> @@ -1025,6 +1025,7 @@ def test_m68k_mcf5208evb(self): >>> tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c' >>> self.do_test_advcal_2018('07', tar_hash, 'sanity-clause.elf') >>> >>> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >>> def test_microblaze_s3adsp1800(self): >>> """ >>> :avocado: tags=arch:microblaze >>> diff --git a/tests/acceptance/replay_kernel.py b/tests/acceptance/replay_kernel.py >>> index 00c228382bd..c1f5fa4de71 100644 >>> --- a/tests/acceptance/replay_kernel.py >>> +++ b/tests/acceptance/replay_kernel.py >>> @@ -280,6 +280,7 @@ def test_m68k_mcf5208evb(self): >>> file_path = self.fetch_asset(tar_url, asset_hash=tar_hash) >>> self.do_test_advcal_2018(file_path, 'sanity-clause.elf') >>> >>> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >>> def test_microblaze_s3adsp1800(self): >>> """ >>> :avocado: tags=arch:microblaze >> >> I think this is a recent regression - it hasn't been failing in the past. We >> should first try to find out why it is failing now before sending it to the >> @skipUnless nirvana... could you maybe add it to the "Known issues" at >> https://wiki.qemu.org/Planning/5.2 instead? > > I agree it looks like a regression. > > I disagree we should keep broken tests failing the pipeline, > even if we are not using a Gating CI. But what happens if you disable the test at this point in time now? I think nobody is going to look into this issue anymore since nobody feels responsible. Thus the bug simply get completely ignored. Please add it at least the the "Known issues" section. Thomas
On 11/9/20 10:16 AM, Thomas Huth wrote: > On 09/11/2020 10.09, Philippe Mathieu-Daudé wrote: >> On 11/9/20 10:04 AM, Thomas Huth wrote: >>> On 09/11/2020 09.07, Philippe Mathieu-Daudé wrote: >>>> This test is regularly failing on CI :( Do not run it automatically. >>>> >>>> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> >>>> --- >>>> tests/acceptance/boot_linux_console.py | 1 + >>>> tests/acceptance/replay_kernel.py | 1 + >>>> 2 files changed, 2 insertions(+) >>>> >>>> diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py >>>> index 8f433a67f84..010e8790c0f 100644 >>>> --- a/tests/acceptance/boot_linux_console.py >>>> +++ b/tests/acceptance/boot_linux_console.py >>>> @@ -1025,6 +1025,7 @@ def test_m68k_mcf5208evb(self): >>>> tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c' >>>> self.do_test_advcal_2018('07', tar_hash, 'sanity-clause.elf') >>>> >>>> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >>>> def test_microblaze_s3adsp1800(self): >>>> """ >>>> :avocado: tags=arch:microblaze >>>> diff --git a/tests/acceptance/replay_kernel.py b/tests/acceptance/replay_kernel.py >>>> index 00c228382bd..c1f5fa4de71 100644 >>>> --- a/tests/acceptance/replay_kernel.py >>>> +++ b/tests/acceptance/replay_kernel.py >>>> @@ -280,6 +280,7 @@ def test_m68k_mcf5208evb(self): >>>> file_path = self.fetch_asset(tar_url, asset_hash=tar_hash) >>>> self.do_test_advcal_2018(file_path, 'sanity-clause.elf') >>>> >>>> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >>>> def test_microblaze_s3adsp1800(self): >>>> """ >>>> :avocado: tags=arch:microblaze >>> >>> I think this is a recent regression - it hasn't been failing in the past. We >>> should first try to find out why it is failing now before sending it to the >>> @skipUnless nirvana... could you maybe add it to the "Known issues" at >>> https://wiki.qemu.org/Planning/5.2 instead? >> >> I agree it looks like a regression. >> >> I disagree we should keep broken tests failing the pipeline, >> even if we are not using a Gating CI. > > But what happens if you disable the test at this point in time now? I think > nobody is going to look into this issue anymore since nobody feels > responsible. Thus the bug simply get completely ignored. Corollary: Nobody will use or trust GitLab CI... > Please add it at > least the the "Known issues" section. Will do.
On 11/9/20 7:16 AM, Thomas Huth wrote: > On 09/11/2020 10.09, Philippe Mathieu-Daudé wrote: >> On 11/9/20 10:04 AM, Thomas Huth wrote: >>> On 09/11/2020 09.07, Philippe Mathieu-Daudé wrote: >>>> This test is regularly failing on CI :( Do not run it automatically. >>>> >>>> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> >>>> --- >>>> tests/acceptance/boot_linux_console.py | 1 + >>>> tests/acceptance/replay_kernel.py | 1 + >>>> 2 files changed, 2 insertions(+) >>>> >>>> diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py >>>> index 8f433a67f84..010e8790c0f 100644 >>>> --- a/tests/acceptance/boot_linux_console.py >>>> +++ b/tests/acceptance/boot_linux_console.py >>>> @@ -1025,6 +1025,7 @@ def test_m68k_mcf5208evb(self): >>>> tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c' >>>> self.do_test_advcal_2018('07', tar_hash, 'sanity-clause.elf') >>>> >>>> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >>>> def test_microblaze_s3adsp1800(self): >>>> """ >>>> :avocado: tags=arch:microblaze >>>> diff --git a/tests/acceptance/replay_kernel.py b/tests/acceptance/replay_kernel.py >>>> index 00c228382bd..c1f5fa4de71 100644 >>>> --- a/tests/acceptance/replay_kernel.py >>>> +++ b/tests/acceptance/replay_kernel.py >>>> @@ -280,6 +280,7 @@ def test_m68k_mcf5208evb(self): >>>> file_path = self.fetch_asset(tar_url, asset_hash=tar_hash) >>>> self.do_test_advcal_2018(file_path, 'sanity-clause.elf') >>>> >>>> + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') >>>> def test_microblaze_s3adsp1800(self): >>>> """ >>>> :avocado: tags=arch:microblaze >>> I think this is a recent regression - it hasn't been failing in the past. We >>> should first try to find out why it is failing now before sending it to the >>> @skipUnless nirvana... could you maybe add it to the "Known issues" at >>> https://wiki.qemu.org/Planning/5.2 instead? >> I agree it looks like a regression. >> >> I disagree we should keep broken tests failing the pipeline, >> even if we are not using a Gating CI. > But what happens if you disable the test at this point in time now? I think > nobody is going to look into this issue anymore since nobody feels > responsible. Thus the bug simply get completely ignored. Please add it at > least the the "Known issues" section. Would make sense do the following? 1. Introduce a new tag (e.g. "regression") to mark tests which are currently broken due regression bugs 2. Filter those tests out of the regular acceptance jobs 3. Add a new job to run only those tests 3.1. Use the allow_failure [1] property so that this job won't change the overall testing status Then if a bug is fixed and the test start to pass again, remove the "regression" tag. Otherwise if the bug is still present on occasion of QEMU release then it is definitively marked to skip. [1] https://docs.gitlab.com/ee/ci/yaml/#allow_failure - Wainer > > Thomas
diff --git a/tests/acceptance/boot_linux_console.py b/tests/acceptance/boot_linux_console.py index 8f433a67f84..010e8790c0f 100644 --- a/tests/acceptance/boot_linux_console.py +++ b/tests/acceptance/boot_linux_console.py @@ -1025,6 +1025,7 @@ def test_m68k_mcf5208evb(self): tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c' self.do_test_advcal_2018('07', tar_hash, 'sanity-clause.elf') + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') def test_microblaze_s3adsp1800(self): """ :avocado: tags=arch:microblaze diff --git a/tests/acceptance/replay_kernel.py b/tests/acceptance/replay_kernel.py index 00c228382bd..c1f5fa4de71 100644 --- a/tests/acceptance/replay_kernel.py +++ b/tests/acceptance/replay_kernel.py @@ -280,6 +280,7 @@ def test_m68k_mcf5208evb(self): file_path = self.fetch_asset(tar_url, asset_hash=tar_hash) self.do_test_advcal_2018(file_path, 'sanity-clause.elf') + @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') def test_microblaze_s3adsp1800(self): """ :avocado: tags=arch:microblaze
This test is regularly failing on CI :( Do not run it automatically. Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org> --- tests/acceptance/boot_linux_console.py | 1 + tests/acceptance/replay_kernel.py | 1 + 2 files changed, 2 insertions(+)