diff mbox series

[v2] automation: upgrade Yocto to scarthgap

Message ID alpine.DEB.2.22.394.2407261639190.4857@ubuntu-linux-20-04-desktop (mailing list archive)
State New
Headers show
Series [v2] automation: upgrade Yocto to scarthgap | expand

Commit Message

Stefano Stabellini July 27, 2024, 12:19 a.m. UTC
Upgrade Yocto to a newer version. Use ext4 as image format for testing
with QEMU on ARM and ARM64 as the default is WIC and it is not available
for our xen-image-minimal target.

Also update the tar.bz2 filename for the rootfs.

Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
---

all yocto tests pass:
https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173

Changes in v2:
- s/EXT4/IMAGE_FMT/
- set IMAGE_FMT before the call to project_build
- also update the filename xen-image-minimal-qemuarm.rootfs.tar.bz2
---
 automation/build/yocto/build-yocto.sh       | 15 ++++++++++++---
 automation/build/yocto/yocto.inc            |  4 ++--
 automation/gitlab-ci/build.yaml             |  2 +-
 automation/scripts/qemu-smoke-dom0-arm32.sh |  2 +-
 4 files changed, 16 insertions(+), 7 deletions(-)

Comments

Marek Marczykowski-Górecki July 30, 2024, 1:46 p.m. UTC | #1
On Fri, Jul 26, 2024 at 05:19:42PM -0700, Stefano Stabellini wrote:
> Upgrade Yocto to a newer version. Use ext4 as image format for testing
> with QEMU on ARM and ARM64 as the default is WIC and it is not available
> for our xen-image-minimal target.
> 
> Also update the tar.bz2 filename for the rootfs.
> 
> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>

Reviewed-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>

> ---
> 
> all yocto tests pass:
> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173
> 
> Changes in v2:
> - s/EXT4/IMAGE_FMT/
> - set IMAGE_FMT before the call to project_build
> - also update the filename xen-image-minimal-qemuarm.rootfs.tar.bz2
> ---
>  automation/build/yocto/build-yocto.sh       | 15 ++++++++++++---
>  automation/build/yocto/yocto.inc            |  4 ++--
>  automation/gitlab-ci/build.yaml             |  2 +-
>  automation/scripts/qemu-smoke-dom0-arm32.sh |  2 +-
>  4 files changed, 16 insertions(+), 7 deletions(-)
> 
> diff --git a/automation/build/yocto/build-yocto.sh b/automation/build/yocto/build-yocto.sh
> index 93ce81ce82..e1e69f2bb5 100755
> --- a/automation/build/yocto/build-yocto.sh
> +++ b/automation/build/yocto/build-yocto.sh
> @@ -25,6 +25,7 @@ TARGET_SUPPORTED="qemuarm qemuarm64 qemux86-64"
>  VERBOSE="n"
>  TARGETLIST=""
>  BUILDJOBS="8"
> +IMAGE_FMT=""
>  
>  # actions to do
>  do_clean="n"
> @@ -38,8 +39,9 @@ build_result=0
>  # layers to include in the project
>  build_layerlist="poky/meta poky/meta-poky poky/meta-yocto-bsp \
>                   meta-openembedded/meta-oe meta-openembedded/meta-python \
> +                 meta-openembedded/meta-networking \
>                   meta-openembedded/meta-filesystems \
> -                 meta-openembedded/meta-networking meta-virtualization"
> +                 meta-virtualization"
>  
>  # yocto image to build
>  build_image="xen-image-minimal"
> @@ -175,7 +177,7 @@ function project_build() {
>                  mkdir -p $OUTPUTDIR
>                  cp $BUILDDIR/tmp/deploy/images/qemuarm/zImage $OUTPUTDIR
>                  cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-qemuarm $OUTPUTDIR
> -                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-image-minimal-qemuarm.tar.bz2 $OUTPUTDIR
> +                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-image-minimal-qemuarm.rootfs.tar.bz2 $OUTPUTDIR
>              fi
>          fi
>      ) || return 1
> @@ -196,7 +198,7 @@ function project_run() {
>  
>          /usr/bin/expect <<EOF
>  set timeout 1000
> -spawn bash -c "runqemu serialstdio nographic slirp"
> +spawn bash -c "runqemu serialstdio nographic slirp ${IMAGE_FMT}"
>  
>  expect_after {
>      -re "(.*)\r" {
> @@ -356,6 +358,13 @@ for f in ${TARGETLIST}; do
>          run_task project_create "${f}"
>      fi
>      if [ -f "${BUILDDIR}/${f}/conf/local.conf" ]; then
> +        # Set the right image target
> +        if [ "$f" = "qemux86-64" ]; then
> +            IMAGE_FMT=""
> +        else
> +            IMAGE_FMT="ext4"
> +        fi
> +
>          if [ "${do_build}" = "y" ]; then
>              run_task project_build "${f}"
>          fi
> diff --git a/automation/build/yocto/yocto.inc b/automation/build/yocto/yocto.inc
> index 2f3b1a5b2a..209df7dde9 100644
> --- a/automation/build/yocto/yocto.inc
> +++ b/automation/build/yocto/yocto.inc
> @@ -6,10 +6,10 @@
>  # YOCTOVERSION-TARGET for x86_64 hosts
>  # YOCTOVERSION-TARGET-arm64v8 for arm64 hosts
>  # For example you can build an arm64 container with the following command:
> -# make yocto/kirkstone-qemuarm64-arm64v8
> +# make yocto/scarthgap-qemuarm64-arm64v8
>  
>  # Yocto versions we are currently using.
> -YOCTO_VERSION = kirkstone
> +YOCTO_VERSION = scarthgap
>  
>  # Yocto BSPs we want to build for.
>  YOCTO_TARGETS = qemuarm64 qemuarm qemux86-64
> diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
> index 7ce88d38e7..32045cef0c 100644
> --- a/automation/gitlab-ci/build.yaml
> +++ b/automation/gitlab-ci/build.yaml
> @@ -212,7 +212,7 @@
>    script:
>      - ./automation/build/yocto/build-yocto.sh -v --log-dir=./logs --xen-dir=`pwd` ${YOCTO_BOARD} ${YOCTO_OUTPUT}
>    variables:
> -    YOCTO_VERSION: kirkstone
> +    YOCTO_VERSION: scarthgap
>      CONTAINER: yocto:${YOCTO_VERSION}-${YOCTO_BOARD}${YOCTO_HOST}
>    artifacts:
>      paths:
> diff --git a/automation/scripts/qemu-smoke-dom0-arm32.sh b/automation/scripts/qemu-smoke-dom0-arm32.sh
> index 31c05cc840..eaaea5a982 100755
> --- a/automation/scripts/qemu-smoke-dom0-arm32.sh
> +++ b/automation/scripts/qemu-smoke-dom0-arm32.sh
> @@ -8,7 +8,7 @@ cd binaries
>  
>  mkdir rootfs
>  cd rootfs
> -tar xvf ../xen-image-minimal-qemuarm.tar.bz2
> +tar xvf ../xen-image-minimal-qemuarm.rootfs.tar.bz2
>  mkdir -p ./root
>  echo "name=\"test\"
>  memory=400
> -- 
> 2.25.1
>
Andrew Cooper July 30, 2024, 2:01 p.m. UTC | #2
On 30/07/2024 2:46 pm, Marek Marczykowski-Górecki wrote:
> On Fri, Jul 26, 2024 at 05:19:42PM -0700, Stefano Stabellini wrote:
>> Upgrade Yocto to a newer version. Use ext4 as image format for testing
>> with QEMU on ARM and ARM64 as the default is WIC and it is not available
>> for our xen-image-minimal target.
>>
>> Also update the tar.bz2 filename for the rootfs.
>>
>> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
> Reviewed-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
>
>> ---
>>
>> all yocto tests pass:
>> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173

That test ran on gitlab-docker-pug, not qubes-ambrosia, so doesn't
confirm the fix to the xattr issue.

Seeing as I'm going to need to rebuild the container anyway, I'll see
about forcing this and double checking.

~Andrew
Marek Marczykowski-Górecki July 30, 2024, 2:07 p.m. UTC | #3
On Tue, Jul 30, 2024 at 03:01:52PM +0100, Andrew Cooper wrote:
> On 30/07/2024 2:46 pm, Marek Marczykowski-Górecki wrote:
> > On Fri, Jul 26, 2024 at 05:19:42PM -0700, Stefano Stabellini wrote:
> >> Upgrade Yocto to a newer version. Use ext4 as image format for testing
> >> with QEMU on ARM and ARM64 as the default is WIC and it is not available
> >> for our xen-image-minimal target.
> >>
> >> Also update the tar.bz2 filename for the rootfs.
> >>
> >> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
> > Reviewed-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> >
> >> ---
> >>
> >> all yocto tests pass:
> >> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173
> 
> That test ran on gitlab-docker-pug, not qubes-ambrosia, so doesn't
> confirm the fix to the xattr issue.

There is one on ambrosia too:
https://gitlab.com/xen-project/people/sstabellini/xen/-/jobs/7423043016

> Seeing as I'm going to need to rebuild the container anyway, I'll see
> about forcing this and double checking.

But double-checking is a good idea anyway.
Stefano Stabellini July 30, 2024, 3:55 p.m. UTC | #4
On Tue, 30 Jul 2024, Marek Marczykowski-Górecki wrote:
> On Tue, Jul 30, 2024 at 03:01:52PM +0100, Andrew Cooper wrote:
> > On 30/07/2024 2:46 pm, Marek Marczykowski-Górecki wrote:
> > > On Fri, Jul 26, 2024 at 05:19:42PM -0700, Stefano Stabellini wrote:
> > >> Upgrade Yocto to a newer version. Use ext4 as image format for testing
> > >> with QEMU on ARM and ARM64 as the default is WIC and it is not available
> > >> for our xen-image-minimal target.
> > >>
> > >> Also update the tar.bz2 filename for the rootfs.
> > >>
> > >> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
> > > Reviewed-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> > >
> > >> ---
> > >>
> > >> all yocto tests pass:
> > >> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173
> > 
> > That test ran on gitlab-docker-pug, not qubes-ambrosia, so doesn't
> > confirm the fix to the xattr issue.
> 
> There is one on ambrosia too:
> https://gitlab.com/xen-project/people/sstabellini/xen/-/jobs/7423043016

Yes indeed: I tested it in a previous run. The last test I let it run
without additional constraints. But as far as I can tell the patch is
100% tested.


> > Seeing as I'm going to need to rebuild the container anyway, I'll see
> > about forcing this and double checking.
> 
> But double-checking is a good idea anyway.

I did update the x86, arm64 and arm32 containers. Feel free to rebuild
any of them.

I'll commit the patch this afternoon if you don't get to it sooner :-)
Andrew Cooper July 30, 2024, 4:11 p.m. UTC | #5
On 30/07/2024 4:55 pm, Stefano Stabellini wrote:
> On Tue, 30 Jul 2024, Marek Marczykowski-Górecki wrote:
>> On Tue, Jul 30, 2024 at 03:01:52PM +0100, Andrew Cooper wrote:
>>> On 30/07/2024 2:46 pm, Marek Marczykowski-Górecki wrote:
>>>> On Fri, Jul 26, 2024 at 05:19:42PM -0700, Stefano Stabellini wrote:
>>>>> Upgrade Yocto to a newer version. Use ext4 as image format for testing
>>>>> with QEMU on ARM and ARM64 as the default is WIC and it is not available
>>>>> for our xen-image-minimal target.
>>>>>
>>>>> Also update the tar.bz2 filename for the rootfs.
>>>>>
>>>>> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
>>>> Reviewed-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
>>>>
>>>>> ---
>>>>>
>>>>> all yocto tests pass:
>>>>> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173
>>> That test ran on gitlab-docker-pug, not qubes-ambrosia, so doesn't
>>> confirm the fix to the xattr issue.
>> There is one on ambrosia too:
>> https://gitlab.com/xen-project/people/sstabellini/xen/-/jobs/7423043016
> Yes indeed: I tested it in a previous run. The last test I let it run
> without additional constraints. But as far as I can tell the patch is
> 100% tested.
>
>
>>> Seeing as I'm going to need to rebuild the container anyway, I'll see
>>> about forcing this and double checking.
>> But double-checking is a good idea anyway.
> I did update the x86, arm64 and arm32 containers. Feel free to rebuild
> any of them.
>
> I'll commit the patch this afternoon if you don't get to it sooner :-)

Just the x86 container rebuild has been going on for the entirety of the
MISRA meeting...

If you've got one to hand already, please deploy it.

~Andrew
Stefano Stabellini July 30, 2024, 4:41 p.m. UTC | #6
On Tue, 30 Jul 2024, Andrew Cooper wrote:
> On 30/07/2024 4:55 pm, Stefano Stabellini wrote:
> > On Tue, 30 Jul 2024, Marek Marczykowski-Górecki wrote:
> >> On Tue, Jul 30, 2024 at 03:01:52PM +0100, Andrew Cooper wrote:
> >>> On 30/07/2024 2:46 pm, Marek Marczykowski-Górecki wrote:
> >>>> On Fri, Jul 26, 2024 at 05:19:42PM -0700, Stefano Stabellini wrote:
> >>>>> Upgrade Yocto to a newer version. Use ext4 as image format for testing
> >>>>> with QEMU on ARM and ARM64 as the default is WIC and it is not available
> >>>>> for our xen-image-minimal target.
> >>>>>
> >>>>> Also update the tar.bz2 filename for the rootfs.
> >>>>>
> >>>>> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
> >>>> Reviewed-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
> >>>>
> >>>>> ---
> >>>>>
> >>>>> all yocto tests pass:
> >>>>> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173
> >>> That test ran on gitlab-docker-pug, not qubes-ambrosia, so doesn't
> >>> confirm the fix to the xattr issue.
> >> There is one on ambrosia too:
> >> https://gitlab.com/xen-project/people/sstabellini/xen/-/jobs/7423043016
> > Yes indeed: I tested it in a previous run. The last test I let it run
> > without additional constraints. But as far as I can tell the patch is
> > 100% tested.
> >
> >
> >>> Seeing as I'm going to need to rebuild the container anyway, I'll see
> >>> about forcing this and double checking.
> >> But double-checking is a good idea anyway.
> > I did update the x86, arm64 and arm32 containers. Feel free to rebuild
> > any of them.
> >
> > I'll commit the patch this afternoon if you don't get to it sooner :-)
> 
> Just the x86 container rebuild has been going on for the entirety of the
> MISRA meeting...
> 
> If you've got one to hand already, please deploy it.

I deployed the three yocto containers
Andrew Cooper July 30, 2024, 4:44 p.m. UTC | #7
On 30/07/2024 5:41 pm, Stefano Stabellini wrote:
> On Tue, 30 Jul 2024, Andrew Cooper wrote:
>> On 30/07/2024 4:55 pm, Stefano Stabellini wrote:
>>> On Tue, 30 Jul 2024, Marek Marczykowski-Górecki wrote:
>>>> On Tue, Jul 30, 2024 at 03:01:52PM +0100, Andrew Cooper wrote:
>>>>> On 30/07/2024 2:46 pm, Marek Marczykowski-Górecki wrote:
>>>>>> On Fri, Jul 26, 2024 at 05:19:42PM -0700, Stefano Stabellini wrote:
>>>>>>> Upgrade Yocto to a newer version. Use ext4 as image format for testing
>>>>>>> with QEMU on ARM and ARM64 as the default is WIC and it is not available
>>>>>>> for our xen-image-minimal target.
>>>>>>>
>>>>>>> Also update the tar.bz2 filename for the rootfs.
>>>>>>>
>>>>>>> Signed-off-by: Stefano Stabellini <stefano.stabellini@amd.com>
>>>>>> Reviewed-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
>>>>>>
>>>>>>> ---
>>>>>>>
>>>>>>> all yocto tests pass:
>>>>>>> https://gitlab.com/xen-project/people/sstabellini/xen/-/pipelines/1390081173
>>>>> That test ran on gitlab-docker-pug, not qubes-ambrosia, so doesn't
>>>>> confirm the fix to the xattr issue.
>>>> There is one on ambrosia too:
>>>> https://gitlab.com/xen-project/people/sstabellini/xen/-/jobs/7423043016
>>> Yes indeed: I tested it in a previous run. The last test I let it run
>>> without additional constraints. But as far as I can tell the patch is
>>> 100% tested.
>>>
>>>
>>>>> Seeing as I'm going to need to rebuild the container anyway, I'll see
>>>>> about forcing this and double checking.
>>>> But double-checking is a good idea anyway.
>>> I did update the x86, arm64 and arm32 containers. Feel free to rebuild
>>> any of them.
>>>
>>> I'll commit the patch this afternoon if you don't get to it sooner :-)
>> Just the x86 container rebuild has been going on for the entirety of the
>> MISRA meeting...
>>
>> If you've got one to hand already, please deploy it.
> I deployed the three yocto containers

Ok, I'll include this patch in my commit sweep

~Andrew
diff mbox series

Patch

diff --git a/automation/build/yocto/build-yocto.sh b/automation/build/yocto/build-yocto.sh
index 93ce81ce82..e1e69f2bb5 100755
--- a/automation/build/yocto/build-yocto.sh
+++ b/automation/build/yocto/build-yocto.sh
@@ -25,6 +25,7 @@  TARGET_SUPPORTED="qemuarm qemuarm64 qemux86-64"
 VERBOSE="n"
 TARGETLIST=""
 BUILDJOBS="8"
+IMAGE_FMT=""
 
 # actions to do
 do_clean="n"
@@ -38,8 +39,9 @@  build_result=0
 # layers to include in the project
 build_layerlist="poky/meta poky/meta-poky poky/meta-yocto-bsp \
                  meta-openembedded/meta-oe meta-openembedded/meta-python \
+                 meta-openembedded/meta-networking \
                  meta-openembedded/meta-filesystems \
-                 meta-openembedded/meta-networking meta-virtualization"
+                 meta-virtualization"
 
 # yocto image to build
 build_image="xen-image-minimal"
@@ -175,7 +177,7 @@  function project_build() {
                 mkdir -p $OUTPUTDIR
                 cp $BUILDDIR/tmp/deploy/images/qemuarm/zImage $OUTPUTDIR
                 cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-qemuarm $OUTPUTDIR
-                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-image-minimal-qemuarm.tar.bz2 $OUTPUTDIR
+                cp $BUILDDIR/tmp/deploy/images/qemuarm/xen-image-minimal-qemuarm.rootfs.tar.bz2 $OUTPUTDIR
             fi
         fi
     ) || return 1
@@ -196,7 +198,7 @@  function project_run() {
 
         /usr/bin/expect <<EOF
 set timeout 1000
-spawn bash -c "runqemu serialstdio nographic slirp"
+spawn bash -c "runqemu serialstdio nographic slirp ${IMAGE_FMT}"
 
 expect_after {
     -re "(.*)\r" {
@@ -356,6 +358,13 @@  for f in ${TARGETLIST}; do
         run_task project_create "${f}"
     fi
     if [ -f "${BUILDDIR}/${f}/conf/local.conf" ]; then
+        # Set the right image target
+        if [ "$f" = "qemux86-64" ]; then
+            IMAGE_FMT=""
+        else
+            IMAGE_FMT="ext4"
+        fi
+
         if [ "${do_build}" = "y" ]; then
             run_task project_build "${f}"
         fi
diff --git a/automation/build/yocto/yocto.inc b/automation/build/yocto/yocto.inc
index 2f3b1a5b2a..209df7dde9 100644
--- a/automation/build/yocto/yocto.inc
+++ b/automation/build/yocto/yocto.inc
@@ -6,10 +6,10 @@ 
 # YOCTOVERSION-TARGET for x86_64 hosts
 # YOCTOVERSION-TARGET-arm64v8 for arm64 hosts
 # For example you can build an arm64 container with the following command:
-# make yocto/kirkstone-qemuarm64-arm64v8
+# make yocto/scarthgap-qemuarm64-arm64v8
 
 # Yocto versions we are currently using.
-YOCTO_VERSION = kirkstone
+YOCTO_VERSION = scarthgap
 
 # Yocto BSPs we want to build for.
 YOCTO_TARGETS = qemuarm64 qemuarm qemux86-64
diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
index 7ce88d38e7..32045cef0c 100644
--- a/automation/gitlab-ci/build.yaml
+++ b/automation/gitlab-ci/build.yaml
@@ -212,7 +212,7 @@ 
   script:
     - ./automation/build/yocto/build-yocto.sh -v --log-dir=./logs --xen-dir=`pwd` ${YOCTO_BOARD} ${YOCTO_OUTPUT}
   variables:
-    YOCTO_VERSION: kirkstone
+    YOCTO_VERSION: scarthgap
     CONTAINER: yocto:${YOCTO_VERSION}-${YOCTO_BOARD}${YOCTO_HOST}
   artifacts:
     paths:
diff --git a/automation/scripts/qemu-smoke-dom0-arm32.sh b/automation/scripts/qemu-smoke-dom0-arm32.sh
index 31c05cc840..eaaea5a982 100755
--- a/automation/scripts/qemu-smoke-dom0-arm32.sh
+++ b/automation/scripts/qemu-smoke-dom0-arm32.sh
@@ -8,7 +8,7 @@  cd binaries
 
 mkdir rootfs
 cd rootfs
-tar xvf ../xen-image-minimal-qemuarm.tar.bz2
+tar xvf ../xen-image-minimal-qemuarm.rootfs.tar.bz2
 mkdir -p ./root
 echo "name=\"test\"
 memory=400