diff mbox series

[isar-cip-core,1/2] ci: Rewrite using extends

Message ID 20200820082434.1302460-1-nobuhiro1.iwamatsu@toshiba.co.jp
State Rejected
Headers show
Series [isar-cip-core,1/2] ci: Rewrite using extends | expand

Commit Message

Nobuhiro Iwamatsu Aug. 20, 2020, 8:24 a.m. UTC
Signed-off-by: Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp>
---
 .gitlab-ci.yml             | 70 +++++++++++++++++++++++++++++---------
 scripts/deploy-cip-core.sh |  8 +++--
 2 files changed, 60 insertions(+), 18 deletions(-)

Comments

Jan Kiszka Aug. 21, 2020, 3:04 p.m. UTC | #1
On 20.08.20 10:24, Nobuhiro Iwamatsu wrote:
> Signed-off-by: Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp>
> ---
>  .gitlab-ci.yml             | 70 +++++++++++++++++++++++++++++---------
>  scripts/deploy-cip-core.sh |  8 +++--
>  2 files changed, 60 insertions(+), 18 deletions(-)
> 
> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
> index 3fe7af2..e23345f 100644
> --- a/.gitlab-ci.yml
> +++ b/.gitlab-ci.yml
> @@ -2,10 +2,17 @@ image: kasproject/kas-isar:1.1
>  
>  variables:
>    GIT_STRATEGY: clone
> +  release: buster
> +  extention: base
> +  use_rt: enable
> +  targz: enable
> +  dtb: none
>  
> -all:
> -  stage: build
> -  script:
> +stages:
> +  - build
> +
> +default:
> +  before_script:
>      - export http_proxy=$HTTP_PROXY
>      - export https_proxy=$HTTPS_PROXY
>      - export ftp_proxy=$FTP_PROXY
> @@ -13,20 +20,51 @@ all:
>      - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
>      - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
>  
> -    - kas build kas-cip.yml:kas/board/simatic-ipc227e.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
> -    - scripts/deploy-cip-core.sh buster simatic-ipc227e
> -
> +.build_base:
> +  stage: build
> +  variables:
> +    base_yaml: "kas-cip.yml:kas/board/${target}.yml"
> +  script:
>      - sudo rm -rf build/tmp
> -    - kas build kas-cip.yml:kas/board/bbb.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
> -    - scripts/deploy-cip-core.sh buster bbb am335x-boneblack.dtb
> +    - if [ "${use_rt}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/rt.yml"; fi;
> +    - if [ "${extention}" != "base" ]; then base_yaml="${base_yaml}:kas/opt/${extention}.yml"; fi;
> +    - if [ "${targz}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/targz-img.yml"; fi;
> +    - kas build ${base_yaml}
> +    - scripts/deploy-cip-core.sh ${release} ${target} ${extention} ${dtb}
>  
> -    - sudo rm -rf build/tmp
> -    - kas build kas-cip.yml:kas/board/iwg20m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
> -    - scripts/deploy-cip-core.sh buster iwg20m r8a7743-iwg20d-q7-dbcm-ca.dtb
> +# base image
> +build:simatic-ipc227e-base:
> +  extends:
> +    - .build_base
> +  variables:
> +    target: simatic-ipc227e
>  
> -    - sudo rm -rf build/tmp
> -    - kas build kas-cip.yml:kas/board/rzg2m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
> -    - scripts/deploy-cip-core.sh buster hihope-rzg2m renesas/r8a774a1-hihope-rzg2m-ex.dtb
> +build:bbb-base:
> +  extends:
> +    - .build_base
> +  variables:
> +    target: bbb
> +    dtb: am335x-boneblack.dtb
>  
> -    - sudo rm -rf build/tmp
> -    - kas build kas-cip.yml:kas/board/qemu-amd64.yml:kas/opt/security.yml
> +build:iwg20m-base:
> +  extends:
> +    - .build_base
> +  variables:
> +    target: iwg20m
> +    dtb: r8a7743-iwg20d-q7-dbcm-ca.dtb
> +
> +build:hihope-rzg2m-base:
> +  extends:
> +    - .build_base
> +  variables:
> +    target: rzg2m
> +    dtb: renesas/r8a774a1-hihope-rzg2m-ex.dtb
> +
> +build:qemu-amd64-base:
> +  extends:
> +    - .build_base
> +  variables:
> +    target: qemu-amd64
> +    extention: security
> +    use_rt: disable
> +    targz: disable
> diff --git a/scripts/deploy-cip-core.sh b/scripts/deploy-cip-core.sh
> index 4c8d4c9..5b7eab9 100755
> --- a/scripts/deploy-cip-core.sh
> +++ b/scripts/deploy-cip-core.sh
> @@ -16,9 +16,13 @@ fi
>  
>  RELEASE=$1
>  TARGET=$2
> -DTB=$3
> +EXTENSION=$3
> +DTB=$4
>  
>  BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET
> +if [ "${EXTENSION}" != "base" ] ; then
> +	BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET-$EXTENSION
> +fi
>  
>  echo "Compressing cip-core-image-cip-core-$RELEASE-$TARGET.wic.img..."
>  xz -9 -k $BASE_PATH.wic.img
> @@ -38,6 +42,6 @@ fi
>  aws s3 cp --no-progress $KERNEL_IMAGE s3://download.cip-project.org/cip-core/$TARGET/
>  aws s3 cp --no-progress $BASE_PATH-initrd.img s3://download.cip-project.org/cip-core/$TARGET/
>  
> -if [ -n "$DTB" ]; then
> +if [ "$DTB" != "none" ]; then
>  	aws s3 cp --no-progress build/tmp/work/cip-core-*/linux-cip*/*/linux-cip-*/debian/linux-image-cip*/usr/lib/linux-image-*/$DTB s3://download.cip-project.org/cip-core/$TARGET/
>  fi
> 

Unfortunately, this scale out to multiple jobs seem to cause download
issues, see e.g.

- https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/697844276
- https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/696946649

First thought it was a sporadic hic-up, but it reoccurs, now with
master. Is kernel.org throttling us here?

Jan

--
Siemens AG, Corporate Technology, CT RDA IOT SES-DE
Corporate Competence Center Embedded Linux
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.

View/Reply Online (#5223): https://lists.cip-project.org/g/cip-dev/message/5223
Mute This Topic: https://lists.cip-project.org/mt/76304024/4520428
Group Owner: cip-dev+owner@lists.cip-project.org
Unsubscribe: https://lists.cip-project.org/g/cip-dev/leave/8129116/1171672734/xyzzy  [patchwork-cip-dev@patchwork.kernel.org]
-=-=-=-=-=-=-=-=-=-=-=-
Jan Kiszka Aug. 21, 2020, 4:44 p.m. UTC | #2
On 21.08.20 17:04, Jan Kiszka wrote:
> On 20.08.20 10:24, Nobuhiro Iwamatsu wrote:
>> Signed-off-by: Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp>
>> ---
>>  .gitlab-ci.yml             | 70 +++++++++++++++++++++++++++++---------
>>  scripts/deploy-cip-core.sh |  8 +++--
>>  2 files changed, 60 insertions(+), 18 deletions(-)
>>
>> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
>> index 3fe7af2..e23345f 100644
>> --- a/.gitlab-ci.yml
>> +++ b/.gitlab-ci.yml
>> @@ -2,10 +2,17 @@ image: kasproject/kas-isar:1.1
>>  
>>  variables:
>>    GIT_STRATEGY: clone
>> +  release: buster
>> +  extention: base
>> +  use_rt: enable
>> +  targz: enable
>> +  dtb: none
>>  
>> -all:
>> -  stage: build
>> -  script:
>> +stages:
>> +  - build
>> +
>> +default:
>> +  before_script:
>>      - export http_proxy=$HTTP_PROXY
>>      - export https_proxy=$HTTPS_PROXY
>>      - export ftp_proxy=$FTP_PROXY
>> @@ -13,20 +20,51 @@ all:
>>      - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
>>      - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
>>  
>> -    - kas build kas-cip.yml:kas/board/simatic-ipc227e.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>> -    - scripts/deploy-cip-core.sh buster simatic-ipc227e
>> -
>> +.build_base:
>> +  stage: build
>> +  variables:
>> +    base_yaml: "kas-cip.yml:kas/board/${target}.yml"
>> +  script:
>>      - sudo rm -rf build/tmp
>> -    - kas build kas-cip.yml:kas/board/bbb.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>> -    - scripts/deploy-cip-core.sh buster bbb am335x-boneblack.dtb
>> +    - if [ "${use_rt}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/rt.yml"; fi;
>> +    - if [ "${extention}" != "base" ]; then base_yaml="${base_yaml}:kas/opt/${extention}.yml"; fi;
>> +    - if [ "${targz}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/targz-img.yml"; fi;
>> +    - kas build ${base_yaml}
>> +    - scripts/deploy-cip-core.sh ${release} ${target} ${extention} ${dtb}
>>  
>> -    - sudo rm -rf build/tmp
>> -    - kas build kas-cip.yml:kas/board/iwg20m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>> -    - scripts/deploy-cip-core.sh buster iwg20m r8a7743-iwg20d-q7-dbcm-ca.dtb
>> +# base image
>> +build:simatic-ipc227e-base:
>> +  extends:
>> +    - .build_base
>> +  variables:
>> +    target: simatic-ipc227e
>>  
>> -    - sudo rm -rf build/tmp
>> -    - kas build kas-cip.yml:kas/board/rzg2m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>> -    - scripts/deploy-cip-core.sh buster hihope-rzg2m renesas/r8a774a1-hihope-rzg2m-ex.dtb
>> +build:bbb-base:
>> +  extends:
>> +    - .build_base
>> +  variables:
>> +    target: bbb
>> +    dtb: am335x-boneblack.dtb
>>  
>> -    - sudo rm -rf build/tmp
>> -    - kas build kas-cip.yml:kas/board/qemu-amd64.yml:kas/opt/security.yml
>> +build:iwg20m-base:
>> +  extends:
>> +    - .build_base
>> +  variables:
>> +    target: iwg20m
>> +    dtb: r8a7743-iwg20d-q7-dbcm-ca.dtb
>> +
>> +build:hihope-rzg2m-base:
>> +  extends:
>> +    - .build_base
>> +  variables:
>> +    target: rzg2m
>> +    dtb: renesas/r8a774a1-hihope-rzg2m-ex.dtb
>> +
>> +build:qemu-amd64-base:
>> +  extends:
>> +    - .build_base
>> +  variables:
>> +    target: qemu-amd64
>> +    extention: security
>> +    use_rt: disable
>> +    targz: disable
>> diff --git a/scripts/deploy-cip-core.sh b/scripts/deploy-cip-core.sh
>> index 4c8d4c9..5b7eab9 100755
>> --- a/scripts/deploy-cip-core.sh
>> +++ b/scripts/deploy-cip-core.sh
>> @@ -16,9 +16,13 @@ fi
>>  
>>  RELEASE=$1
>>  TARGET=$2
>> -DTB=$3
>> +EXTENSION=$3
>> +DTB=$4
>>  
>>  BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET
>> +if [ "${EXTENSION}" != "base" ] ; then
>> +	BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET-$EXTENSION
>> +fi
>>  
>>  echo "Compressing cip-core-image-cip-core-$RELEASE-$TARGET.wic.img..."
>>  xz -9 -k $BASE_PATH.wic.img
>> @@ -38,6 +42,6 @@ fi
>>  aws s3 cp --no-progress $KERNEL_IMAGE s3://download.cip-project.org/cip-core/$TARGET/
>>  aws s3 cp --no-progress $BASE_PATH-initrd.img s3://download.cip-project.org/cip-core/$TARGET/
>>  
>> -if [ -n "$DTB" ]; then
>> +if [ "$DTB" != "none" ]; then
>>  	aws s3 cp --no-progress build/tmp/work/cip-core-*/linux-cip*/*/linux-cip-*/debian/linux-image-cip*/usr/lib/linux-image-*/$DTB s3://download.cip-project.org/cip-core/$TARGET/
>>  fi
>>
> 
> Unfortunately, this scale out to multiple jobs seem to cause download
> issues, see e.g.
> 
> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/697844276
> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/696946649
> 
> First thought it was a sporadic hic-up, but it reoccurs, now with
> master. Is kernel.org throttling us here?
> 

And I'm afraid there is more broken, namely in the deployment that is
only triggered over master. Please have a look at the failing jobs.

Thanks,
Jan
Jan Kiszka Aug. 25, 2020, 5:17 a.m. UTC | #3
On 21.08.20 18:44, Jan Kiszka wrote:
> On 21.08.20 17:04, Jan Kiszka wrote:
>> On 20.08.20 10:24, Nobuhiro Iwamatsu wrote:
>>> Signed-off-by: Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp>
>>> ---
>>>  .gitlab-ci.yml             | 70 +++++++++++++++++++++++++++++---------
>>>  scripts/deploy-cip-core.sh |  8 +++--
>>>  2 files changed, 60 insertions(+), 18 deletions(-)
>>>
>>> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
>>> index 3fe7af2..e23345f 100644
>>> --- a/.gitlab-ci.yml
>>> +++ b/.gitlab-ci.yml
>>> @@ -2,10 +2,17 @@ image: kasproject/kas-isar:1.1
>>>  
>>>  variables:
>>>    GIT_STRATEGY: clone
>>> +  release: buster
>>> +  extention: base
>>> +  use_rt: enable
>>> +  targz: enable
>>> +  dtb: none
>>>  
>>> -all:
>>> -  stage: build
>>> -  script:
>>> +stages:
>>> +  - build
>>> +
>>> +default:
>>> +  before_script:
>>>      - export http_proxy=$HTTP_PROXY
>>>      - export https_proxy=$HTTPS_PROXY
>>>      - export ftp_proxy=$FTP_PROXY
>>> @@ -13,20 +20,51 @@ all:
>>>      - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
>>>      - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
>>>  
>>> -    - kas build kas-cip.yml:kas/board/simatic-ipc227e.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>>> -    - scripts/deploy-cip-core.sh buster simatic-ipc227e
>>> -
>>> +.build_base:
>>> +  stage: build
>>> +  variables:
>>> +    base_yaml: "kas-cip.yml:kas/board/${target}.yml"
>>> +  script:
>>>      - sudo rm -rf build/tmp
>>> -    - kas build kas-cip.yml:kas/board/bbb.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>>> -    - scripts/deploy-cip-core.sh buster bbb am335x-boneblack.dtb
>>> +    - if [ "${use_rt}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/rt.yml"; fi;
>>> +    - if [ "${extention}" != "base" ]; then base_yaml="${base_yaml}:kas/opt/${extention}.yml"; fi;
>>> +    - if [ "${targz}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/targz-img.yml"; fi;
>>> +    - kas build ${base_yaml}
>>> +    - scripts/deploy-cip-core.sh ${release} ${target} ${extention} ${dtb}
>>>  
>>> -    - sudo rm -rf build/tmp
>>> -    - kas build kas-cip.yml:kas/board/iwg20m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>>> -    - scripts/deploy-cip-core.sh buster iwg20m r8a7743-iwg20d-q7-dbcm-ca.dtb
>>> +# base image
>>> +build:simatic-ipc227e-base:
>>> +  extends:
>>> +    - .build_base
>>> +  variables:
>>> +    target: simatic-ipc227e
>>>  
>>> -    - sudo rm -rf build/tmp
>>> -    - kas build kas-cip.yml:kas/board/rzg2m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
>>> -    - scripts/deploy-cip-core.sh buster hihope-rzg2m renesas/r8a774a1-hihope-rzg2m-ex.dtb
>>> +build:bbb-base:
>>> +  extends:
>>> +    - .build_base
>>> +  variables:
>>> +    target: bbb
>>> +    dtb: am335x-boneblack.dtb
>>>  
>>> -    - sudo rm -rf build/tmp
>>> -    - kas build kas-cip.yml:kas/board/qemu-amd64.yml:kas/opt/security.yml
>>> +build:iwg20m-base:
>>> +  extends:
>>> +    - .build_base
>>> +  variables:
>>> +    target: iwg20m
>>> +    dtb: r8a7743-iwg20d-q7-dbcm-ca.dtb
>>> +
>>> +build:hihope-rzg2m-base:
>>> +  extends:
>>> +    - .build_base
>>> +  variables:
>>> +    target: rzg2m
>>> +    dtb: renesas/r8a774a1-hihope-rzg2m-ex.dtb
>>> +
>>> +build:qemu-amd64-base:
>>> +  extends:
>>> +    - .build_base
>>> +  variables:
>>> +    target: qemu-amd64
>>> +    extention: security
>>> +    use_rt: disable
>>> +    targz: disable
>>> diff --git a/scripts/deploy-cip-core.sh b/scripts/deploy-cip-core.sh
>>> index 4c8d4c9..5b7eab9 100755
>>> --- a/scripts/deploy-cip-core.sh
>>> +++ b/scripts/deploy-cip-core.sh
>>> @@ -16,9 +16,13 @@ fi
>>>  
>>>  RELEASE=$1
>>>  TARGET=$2
>>> -DTB=$3
>>> +EXTENSION=$3
>>> +DTB=$4
>>>  
>>>  BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET
>>> +if [ "${EXTENSION}" != "base" ] ; then
>>> +	BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET-$EXTENSION
>>> +fi
>>>  
>>>  echo "Compressing cip-core-image-cip-core-$RELEASE-$TARGET.wic.img..."
>>>  xz -9 -k $BASE_PATH.wic.img
>>> @@ -38,6 +42,6 @@ fi
>>>  aws s3 cp --no-progress $KERNEL_IMAGE s3://download.cip-project.org/cip-core/$TARGET/
>>>  aws s3 cp --no-progress $BASE_PATH-initrd.img s3://download.cip-project.org/cip-core/$TARGET/
>>>  
>>> -if [ -n "$DTB" ]; then
>>> +if [ "$DTB" != "none" ]; then
>>>  	aws s3 cp --no-progress build/tmp/work/cip-core-*/linux-cip*/*/linux-cip-*/debian/linux-image-cip*/usr/lib/linux-image-*/$DTB s3://download.cip-project.org/cip-core/$TARGET/
>>>  fi
>>>
>>
>> Unfortunately, this scale out to multiple jobs seem to cause download
>> issues, see e.g.
>>
>> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/697844276
>> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/696946649
>>
>> First thought it was a sporadic hic-up, but it reoccurs, now with
>> master. Is kernel.org throttling us here?
>>
> 
> And I'm afraid there is more broken, namely in the deployment that is
> only triggered over master. Please have a look at the failing jobs.
> 

Ping - or should I revert those two for now?

Jan
Nobuhiro Iwamatsu Aug. 25, 2020, 5:33 a.m. UTC | #4
Hi,

> -----Original Message-----
> From: Jan Kiszka [mailto:jan.kiszka@siemens.com]
> Sent: Saturday, August 22, 2020 1:45 AM
> To: iwamatsu nobuhiro(岩松 信洋 □SWC◯ACT) <nobuhiro1.iwamatsu@toshiba.co.jp>; cip-dev@lists.cip-project.org
> Subject: Re: [isar-cip-core][PATCH 1/2] ci: Rewrite using extends
> 
> On 21.08.20 17:04, Jan Kiszka wrote:
> > On 20.08.20 10:24, Nobuhiro Iwamatsu wrote:
> >> Signed-off-by: Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp>
> >> ---
> >>  .gitlab-ci.yml             | 70 +++++++++++++++++++++++++++++---------
> >>  scripts/deploy-cip-core.sh |  8 +++--
> >>  2 files changed, 60 insertions(+), 18 deletions(-)

<snip>

> >> -if [ -n "$DTB" ]; then
> >> +if [ "$DTB" != "none" ]; then
> >>  	aws s3 cp --no-progress
> build/tmp/work/cip-core-*/linux-cip*/*/linux-cip-*/debian/linux-image-cip*/usr/lib/linux-image-*/$DTB
> s3://download.cip-project.org/cip-core/$TARGET/
> >>  fi
> >>
> >
> > Unfortunately, this scale out to multiple jobs seem to cause download
> > issues, see e.g.
> >
> > - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/697844276
> > - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/696946649
> >
> > First thought it was a sporadic hic-up, but it reoccurs, now with
> > master. Is kernel.org throttling us here?
> >

This may be a burden on kernel.org as snapshot reconstructs the image
on the server side. I'm not sure, when downloading images continuously,
access may be controlled by kernel.org.

> 
> And I'm afraid there is more broken, namely in the deployment that is
> only triggered over master. Please have a look at the failing jobs.

I don't have no idea about this.
If we retry the test, it is successful and there may be other causes.


> 
> Thanks,
> Jan
> 

Best regards,
  Nobuhiro

> --
> Siemens AG, Corporate Technology, CT RDA IOT SES-DE
> Corporate Competence Center Embedded Linux
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.

View/Reply Online (#5238): https://lists.cip-project.org/g/cip-dev/message/5238
Mute This Topic: https://lists.cip-project.org/mt/76304024/4520428
Group Owner: cip-dev+owner@lists.cip-project.org
Unsubscribe: https://lists.cip-project.org/g/cip-dev/leave/8129116/1171672734/xyzzy  [patchwork-cip-dev@patchwork.kernel.org]
-=-=-=-=-=-=-=-=-=-=-=-
Jan Kiszka Aug. 25, 2020, 5:59 a.m. UTC | #5
On 25.08.20 07:33, nobuhiro1.iwamatsu@toshiba.co.jp wrote:
> Hi,
> 
>> -----Original Message-----
>> From: Jan Kiszka [mailto:jan.kiszka@siemens.com]
>> Sent: Saturday, August 22, 2020 1:45 AM
>> To: iwamatsu nobuhiro(岩松 信洋 □SWC◯ACT) <nobuhiro1.iwamatsu@toshiba.co.jp>; cip-dev@lists.cip-project.org
>> Subject: Re: [isar-cip-core][PATCH 1/2] ci: Rewrite using extends
>>
>> On 21.08.20 17:04, Jan Kiszka wrote:
>>> On 20.08.20 10:24, Nobuhiro Iwamatsu wrote:
>>>> Signed-off-by: Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp>
>>>> ---
>>>>  .gitlab-ci.yml             | 70 +++++++++++++++++++++++++++++---------
>>>>  scripts/deploy-cip-core.sh |  8 +++--
>>>>  2 files changed, 60 insertions(+), 18 deletions(-)
> 
> <snip>
> 
>>>> -if [ -n "$DTB" ]; then
>>>> +if [ "$DTB" != "none" ]; then
>>>>  	aws s3 cp --no-progress
>> build/tmp/work/cip-core-*/linux-cip*/*/linux-cip-*/debian/linux-image-cip*/usr/lib/linux-image-*/$DTB
>> s3://download.cip-project.org/cip-core/$TARGET/
>>>>  fi
>>>>
>>>
>>> Unfortunately, this scale out to multiple jobs seem to cause download
>>> issues, see e.g.
>>>
>>> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/697844276
>>> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/696946649
>>>
>>> First thought it was a sporadic hic-up, but it reoccurs, now with
>>> master. Is kernel.org throttling us here?
>>>
> 
> This may be a burden on kernel.org as snapshot reconstructs the image
> on the server side. I'm not sure, when downloading images continuously,
> access may be controlled by kernel.org.
> 

So we need to avoid that for now, I guess. Back to single-job build, or
do we have an alternative download source?

>>
>> And I'm afraid there is more broken, namely in the deployment that is
>> only triggered over master. Please have a look at the failing jobs.
> 
> I don't have no idea about this.
> If we retry the test, it is successful and there may be other causes.
> 

The wic image names changed, and that broke compression and uploading.

Jan
Nobuhiro Iwamatsu Aug. 26, 2020, 1:18 a.m. UTC | #6
Hi,

> -----Original Message-----
> From: cip-dev@lists.cip-project.org [mailto:cip-dev@lists.cip-project.org] On Behalf Of Jan Kiszka
> Sent: Tuesday, August 25, 2020 2:59 PM
> To: iwamatsu nobuhiro(岩松 信洋 □SWC◯ACT) <nobuhiro1.iwamatsu@toshiba.co.jp>; cip-dev@lists.cip-project.org
> Subject: Re: [cip-dev] [isar-cip-core][PATCH 1/2] ci: Rewrite using extends
> 
> On 25.08.20 07:33, nobuhiro1.iwamatsu@toshiba.co.jp wrote:
> > Hi,
> >
> >> -----Original Message-----
> >> From: Jan Kiszka [mailto:jan.kiszka@siemens.com]
> >> Sent: Saturday, August 22, 2020 1:45 AM
> >> To: iwamatsu nobuhiro(岩松 信洋 □SWC◯ACT) <nobuhiro1.iwamatsu@toshiba.co.jp>;
> cip-dev@lists.cip-project.org
> >> Subject: Re: [isar-cip-core][PATCH 1/2] ci: Rewrite using extends
> >>
> >> On 21.08.20 17:04, Jan Kiszka wrote:
> >>> On 20.08.20 10:24, Nobuhiro Iwamatsu wrote:
> >>>> Signed-off-by: Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp>
> >>>> ---
> >>>>  .gitlab-ci.yml             | 70 +++++++++++++++++++++++++++++---------
> >>>>  scripts/deploy-cip-core.sh |  8 +++--
> >>>>  2 files changed, 60 insertions(+), 18 deletions(-)
> >
> > <snip>
> >
> >>>> -if [ -n "$DTB" ]; then
> >>>> +if [ "$DTB" != "none" ]; then
> >>>>  	aws s3 cp --no-progress
> >> build/tmp/work/cip-core-*/linux-cip*/*/linux-cip-*/debian/linux-image-cip*/usr/lib/linux-image-*/$DTB
> >> s3://download.cip-project.org/cip-core/$TARGET/
> >>>>  fi
> >>>>
> >>>
> >>> Unfortunately, this scale out to multiple jobs seem to cause download
> >>> issues, see e.g.
> >>>
> >>> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/697844276
> >>> - https://gitlab.com/cip-project/cip-core/isar-cip-core/-/jobs/696946649
> >>>
> >>> First thought it was a sporadic hic-up, but it reoccurs, now with
> >>> master. Is kernel.org throttling us here?
> >>>
> >
> > This may be a burden on kernel.org as snapshot reconstructs the image
> > on the server side. I'm not sure, when downloading images continuously,
> > access may be controlled by kernel.org.
> >
> 
> So we need to avoid that for now, I guess. Back to single-job build, or
> do we have an alternative download source?

OK, because I don't know any other good download source, I will change to single job.

> 
> >>
> >> And I'm afraid there is more broken, namely in the deployment that is
> >> only triggered over master. Please have a look at the failing jobs.
> >
> > I don't have no idea about this.
> > If we retry the test, it is successful and there may be other causes.
> >
> 
> The wic image names changed, and that broke compression and uploading.
> 

OK, I will fix this as well.

> Jan
> 

Best regards,
  Nobuhiro
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.

View/Reply Online (#5252): https://lists.cip-project.org/g/cip-dev/message/5252
Mute This Topic: https://lists.cip-project.org/mt/76304024/4520428
Group Owner: cip-dev+owner@lists.cip-project.org
Unsubscribe: https://lists.cip-project.org/g/cip-dev/leave/8129116/1171672734/xyzzy  [patchwork-cip-dev@patchwork.kernel.org]
-=-=-=-=-=-=-=-=-=-=-=-
diff mbox series

Patch

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3fe7af2..e23345f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,10 +2,17 @@  image: kasproject/kas-isar:1.1
 
 variables:
   GIT_STRATEGY: clone
+  release: buster
+  extention: base
+  use_rt: enable
+  targz: enable
+  dtb: none
 
-all:
-  stage: build
-  script:
+stages:
+  - build
+
+default:
+  before_script:
     - export http_proxy=$HTTP_PROXY
     - export https_proxy=$HTTPS_PROXY
     - export ftp_proxy=$FTP_PROXY
@@ -13,20 +20,51 @@  all:
     - export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
     - export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
 
-    - kas build kas-cip.yml:kas/board/simatic-ipc227e.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
-    - scripts/deploy-cip-core.sh buster simatic-ipc227e
-
+.build_base:
+  stage: build
+  variables:
+    base_yaml: "kas-cip.yml:kas/board/${target}.yml"
+  script:
     - sudo rm -rf build/tmp
-    - kas build kas-cip.yml:kas/board/bbb.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
-    - scripts/deploy-cip-core.sh buster bbb am335x-boneblack.dtb
+    - if [ "${use_rt}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/rt.yml"; fi;
+    - if [ "${extention}" != "base" ]; then base_yaml="${base_yaml}:kas/opt/${extention}.yml"; fi;
+    - if [ "${targz}" = "enable" ]; then base_yaml="${base_yaml}:kas/opt/targz-img.yml"; fi;
+    - kas build ${base_yaml}
+    - scripts/deploy-cip-core.sh ${release} ${target} ${extention} ${dtb}
 
-    - sudo rm -rf build/tmp
-    - kas build kas-cip.yml:kas/board/iwg20m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
-    - scripts/deploy-cip-core.sh buster iwg20m r8a7743-iwg20d-q7-dbcm-ca.dtb
+# base image
+build:simatic-ipc227e-base:
+  extends:
+    - .build_base
+  variables:
+    target: simatic-ipc227e
 
-    - sudo rm -rf build/tmp
-    - kas build kas-cip.yml:kas/board/rzg2m.yml:kas/opt/rt.yml:kas/opt/targz-img.yml
-    - scripts/deploy-cip-core.sh buster hihope-rzg2m renesas/r8a774a1-hihope-rzg2m-ex.dtb
+build:bbb-base:
+  extends:
+    - .build_base
+  variables:
+    target: bbb
+    dtb: am335x-boneblack.dtb
 
-    - sudo rm -rf build/tmp
-    - kas build kas-cip.yml:kas/board/qemu-amd64.yml:kas/opt/security.yml
+build:iwg20m-base:
+  extends:
+    - .build_base
+  variables:
+    target: iwg20m
+    dtb: r8a7743-iwg20d-q7-dbcm-ca.dtb
+
+build:hihope-rzg2m-base:
+  extends:
+    - .build_base
+  variables:
+    target: rzg2m
+    dtb: renesas/r8a774a1-hihope-rzg2m-ex.dtb
+
+build:qemu-amd64-base:
+  extends:
+    - .build_base
+  variables:
+    target: qemu-amd64
+    extention: security
+    use_rt: disable
+    targz: disable
diff --git a/scripts/deploy-cip-core.sh b/scripts/deploy-cip-core.sh
index 4c8d4c9..5b7eab9 100755
--- a/scripts/deploy-cip-core.sh
+++ b/scripts/deploy-cip-core.sh
@@ -16,9 +16,13 @@  fi
 
 RELEASE=$1
 TARGET=$2
-DTB=$3
+EXTENSION=$3
+DTB=$4
 
 BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET
+if [ "${EXTENSION}" != "base" ] ; then
+	BASE_PATH=build/tmp/deploy/images/$TARGET/cip-core-image-cip-core-$RELEASE-$TARGET-$EXTENSION
+fi
 
 echo "Compressing cip-core-image-cip-core-$RELEASE-$TARGET.wic.img..."
 xz -9 -k $BASE_PATH.wic.img
@@ -38,6 +42,6 @@  fi
 aws s3 cp --no-progress $KERNEL_IMAGE s3://download.cip-project.org/cip-core/$TARGET/
 aws s3 cp --no-progress $BASE_PATH-initrd.img s3://download.cip-project.org/cip-core/$TARGET/
 
-if [ -n "$DTB" ]; then
+if [ "$DTB" != "none" ]; then
 	aws s3 cp --no-progress build/tmp/work/cip-core-*/linux-cip*/*/linux-cip-*/debian/linux-image-cip*/usr/lib/linux-image-*/$DTB s3://download.cip-project.org/cip-core/$TARGET/
 fi