| Message ID | 20240320103229.1078738-3-Adithya.Balakumar@toshiba-tsip.com (mailing list archive) |
|---|---|
| State | Changes Requested |
| Headers | show |
| Series | Integrate Delta Update with rdiff_image and delta handler | expand |
On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: > swupdate supports delta updates with rdiff_image and delta(zchunk) > handler. This change adds support to use either of the handler > for creating delta update artifacts. zchunk based updates are supported > only for sid. > > DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables can > be modified based on the update type in the swupdate.yml file. > > Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> > --- > classes/delta-update.bbclass | 107 +++++++++++++++++++++++++++++++++++ > conf/layer.conf | 2 +- > kas/opt/swupdate.yml | 5 ++ > 3 files changed, 113 insertions(+), 1 deletion(-) > create mode 100644 classes/delta-update.bbclass > > diff --git a/classes/delta-update.bbclass b/classes/delta-update.bbclass > new file mode 100644 > index 0000000..8443f35 > --- /dev/null > +++ b/classes/delta-update.bbclass > @@ -0,0 +1,107 @@ > +# > +# CIP Core, generic profile > +# > +# Copyright (c) Toshiba Corporation 2024 > +# > +# Authors: > +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> > +# > +# SPDX-License-Identifier: MIT > +# > + > +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" > + > +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" > +do_image_delta_update[cleandirs] += "${WORKDIR}/delta_interim_artifacts" > + > +DELTA_UPDATE_TYPE ??= "rdiff" > +DELTA_RDIFF_REF_IMAGE ??= "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" > +DELTA_ZCK_URL ??= "" > + > +def disable_delta_update_tasks(d): > + d.appendVarFlag("do_image_delta_update", "noexec", "1") > + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") > + d.setVar("DELTA_UPDATE_TYPE", "") > + > +python () { > + import os > + path = d.getVar("LAYERDIR_cip-core") + "/" + d.getVar("DELTA_RDIFF_REF_IMAGE") I would suggest to remove LAYERDIR_cip-core or replace it with a variable as it makes it unusable in downstream layers. This class should not contain any cip-core specific elements. > + if d.getVar("DELTA_UPDATE_TYPE") == "rdiff": > + if not os.path.isfile(path): > + disable_delta_update_tasks(d) > + elif d.getVar("DELTA_UPDATE_TYPE") == "zchunk": > + if d.getVar("DISTRO") != "cip-core-sid": > + disable_delta_update_tasks(d) > + else: > + disable_delta_update_tasks(d) > +} > + > +python do_delta_rdiff_ref_image_fetch () { > + import bb > + import os > + path = d.getVar("LAYERDIR_cip-core") + "/" + d.getVar("DELTA_RDIFF_REF_IMAGE") Same here remove the LAYERDIR_cip-core > + if d.getVar("DELTA_UPDATE_TYPE") == "rdiff": > + if os.path.isfile(path): > + artifact_uri = "file://" + (d.getVar('DELTA_RDIFF_REF_IMAGE') or "") > + artifact_uri_list = artifact_uri.split() > + fetcher = bb.fetch2.Fetch(artifact_uri_list, d) > + fetcher.download() > + fetcher.unpack(d.getVar('WORKDIR')) > +} > +addtask do_delta_rdiff_ref_image_fetch before do_transform_template > + > +create_rdiff_delta_artifact() { > + if [ -z "${DELTA_RDIFF_REF_IMAGE}" ]; then > + bbfatal "You must set DELTA_RDIFF_REF_IMAGE and provide the required files as artifacts to this recipe" > + fi > + > + rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta > + # create signature file with rdiff > + ${SUDO_CHROOT} /usr/bin/rdiff signature ${WORKDIR}/${DELTA_RDIFF_REF_IMAGE} \ > + ${WORKDIR}/delta_interim_artifacts/old-image-rootfs.sig > + > + # create delta file with the signature file > + ${SUDO_CHROOT} /usr/bin/rdiff delta ${WORKDIR}/delta_interim_artifacts/old-image-rootfs.sig \ > + ${PP_DEPLOY}/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE} ${PP_DEPLOY}/${IMAGE_FULLNAME}.delta > + > + DELTA_ARTIFACT_SWU=${IMAGE_FULLNAME}.delta > + > + # create a symbolic link as IMAGE_CMD expects a *.delta_update file in deploy image directory > + ln -sf ${DELTA_ARTIFACT_SWU} ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta_update > +} > + > +create_zchunk_delta_artifact() { > + # Create .zck file > + ${SUDO_CHROOT} /bin/zck \ > + --output ${PP_DEPLOY}/${IMAGE_FULLNAME}.zck \ > + -u --chunk-hash-type sha256 \ > + ${PP_DEPLOY}/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE} > + > + # Calculate size of zck header > + HSIZE="$(${SUDO_CHROOT} /bin/zck_read_header -v ${PP_DEPLOY}/${IMAGE_FULLNAME}.zck | grep "Header size" | cut -d ':' -f 2)" > + > + # Extract the zck header > + ${SUDO_CHROOT} /bin/dd if="${PP_DEPLOY}/${IMAGE_FULLNAME}".zck of="${PP_DEPLOY}/${IMAGE_FULLNAME}".zck.header bs=1 count="$HSIZE" status=none > + > + DELTA_ARTIFACT_SWU=${IMAGE_FULLNAME}.zck.header > + > + # create a symbolic link as IMAGE_CMD expects a *.delta_update file in deploy image directory > + ln -sf ${DELTA_ARTIFACT_SWU} ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta_update > +} > + > +do_image_delta_update[depends] += "${PN}:do_transform_template" > +IMAGE_CMD:delta_update() { > + case "${DELTA_UPDATE_TYPE}" in > + "rdiff") > + create_rdiff_delta_artifact > + ;; > + "zchunk") > + create_zchunk_delta_artifact > + ;; > + *) > + bbfatal "You must set a valid DELTA_UPDATE_TYPE (rdiff/zchunk)" > + ;; > + esac > +} > + > +addtask do_image_delta_update before do_image_swu after do_image_wic > diff --git a/conf/layer.conf b/conf/layer.conf > index 82b4084..d4c87a6 100644 > --- a/conf/layer.conf > +++ b/conf/layer.conf > @@ -23,6 +23,6 @@ LAYERSERIES_COMPAT_cip-core = "next" > LAYERDIR_cip-core = "${LAYERDIR}" > LAYERDIR_cip-core[vardepvalue] = "isar-cip-core" > > -IMAGE_CLASSES += "squashfs verity swupdate" > +IMAGE_CLASSES += "squashfs verity swupdate delta-update" > > SOURCE_DATE_EPOCH := "${@bb.process.run("git -C ${LAYERDIR_cip-core} log -1 --pretty=%ct | tr -d '\n'")[0]}" > diff --git a/kas/opt/swupdate.yml b/kas/opt/swupdate.yml > index ef61e4e..50ded36 100644 > --- a/kas/opt/swupdate.yml > +++ b/kas/opt/swupdate.yml > @@ -30,3 +30,8 @@ local_conf_header: > ABROOTFS_PART_UUID_B ?= "fedcba98-7654-3210-cafe-5e0710000002" > PREFERRED_PROVIDER_swupdate-certificates-key ??= "swupdate-certificates-key-snakeoil" > PREFERRED_PROVIDER_swupdate-certificates ??= "swupdate-certificates-snakeoil" > + # delta update specific variables" > + IMAGE_FSTYPES:append = " delta_update" > + DELTA_UPDATE_TYPE = "rdiff" > + DELTA_RDIFF_REF_IMAGE = "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" > + DELTA_ZCK_URL = "" > > > > -=-=-=-=-=-=-=-=-=-=-=- > Links: You receive all messages sent to this group. > View/Reply Online (#15381): https://lists.cip-project.org/g/cip-dev/message/15381 > Mute This Topic: https://lists.cip-project.org/mt/105041876/1753640 > Group Owner: cip-dev+owner@lists.cip-project.org > Unsubscribe: https://lists.cip-project.org/g/cip-dev/leave/8129121/1753640/1405269326/xyzzy [quirin.gylstorff@siemens.com] > -=-=-=-=-=-=-=-=-=-=-=- >
-----Original Message----- From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On Behalf Of Quirin Gylstorff via lists.cip-project.org Sent: Monday, April 8, 2024 6:25 PM To: cip-dev@lists.cip-project.org; jan.kiszka@siemens.com Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) <dinesh.kumar@toshiba-tsip.com>; balakumar adithya(ＴＳＩＰ TEUR) <Adithya.Balakumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update support with rdiff_image and delta handler On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: > swupdate supports delta updates with rdiff_image and delta(zchunk) > handler. This change adds support to use either of the handler for > creating delta update artifacts. zchunk based updates are supported > only for sid. > > DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables > can be modified based on the update type in the swupdate.yml file. > > Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> > --- > classes/delta-update.bbclass | 107 +++++++++++++++++++++++++++++++++++ > conf/layer.conf | 2 +- > kas/opt/swupdate.yml | 5 ++ > 3 files changed, 113 insertions(+), 1 deletion(-) > create mode 100644 classes/delta-update.bbclass > > diff --git a/classes/delta-update.bbclass > b/classes/delta-update.bbclass new file mode 100644 index > 0000000..8443f35 > --- /dev/null > +++ b/classes/delta-update.bbclass > @@ -0,0 +1,107 @@ > +# > +# CIP Core, generic profile > +# > +# Copyright (c) Toshiba Corporation 2024 # # Authors: > +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> > +# > +# SPDX-License-Identifier: MIT > +# > + > +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" > + > +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" > +do_image_delta_update[cleandirs] += "${WORKDIR}/delta_interim_artifacts" > + > +DELTA_UPDATE_TYPE ??= "rdiff" > +DELTA_RDIFF_REF_IMAGE ??= "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" > +DELTA_ZCK_URL ??= "" > + > +def disable_delta_update_tasks(d): > + d.appendVarFlag("do_image_delta_update", "noexec", "1") > + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") > + d.setVar("DELTA_UPDATE_TYPE", "") > + > +python () { > + import os > + path = d.getVar("LAYERDIR_cip-core") + "/" + > +d.getVar("DELTA_RDIFF_REF_IMAGE") I would suggest to remove LAYERDIR_cip-core or replace it with a variable as it makes it unusable in downstream layers. This class should not contain any cip-core specific elements. [Adithya Balakumar] Hi Quirin, I understand the issue you are pointing at. Could we reference "LAYERDIR_cip-core" through a variable defined in this recipe. As in maybe have a variable (lets say DELTA_REF_ARTIFACT_PATH) with a weak default assignment with the value of "LAYERDIR_cip-core" so that any downstream layer could override the variable with a different value. > + if d.getVar("DELTA_UPDATE_TYPE") == "rdiff": > + if not os.path.isfile(path): > + disable_delta_update_tasks(d) > + elif d.getVar("DELTA_UPDATE_TYPE") == "zchunk": > + if d.getVar("DISTRO") != "cip-core-sid": > + disable_delta_update_tasks(d) > + else: > + disable_delta_update_tasks(d) } > + > +python do_delta_rdiff_ref_image_fetch () { > + import bb > + import os > + path = d.getVar("LAYERDIR_cip-core") + "/" + > +d.getVar("DELTA_RDIFF_REF_IMAGE") Same here remove the LAYERDIR_cip-core > + if d.getVar("DELTA_UPDATE_TYPE") == "rdiff": > + if os.path.isfile(path): > + artifact_uri = "file://" + (d.getVar('DELTA_RDIFF_REF_IMAGE') or "") > + artifact_uri_list = artifact_uri.split() > + fetcher = bb.fetch2.Fetch(artifact_uri_list, d) > + fetcher.download() > + fetcher.unpack(d.getVar('WORKDIR')) > +} > +addtask do_delta_rdiff_ref_image_fetch before do_transform_template > + > +create_rdiff_delta_artifact() { > + if [ -z "${DELTA_RDIFF_REF_IMAGE}" ]; then > + bbfatal "You must set DELTA_RDIFF_REF_IMAGE and provide the required files as artifacts to this recipe" > + fi > + > + rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta > + # create signature file with rdiff > + ${SUDO_CHROOT} /usr/bin/rdiff signature ${WORKDIR}/${DELTA_RDIFF_REF_IMAGE} \ > + ${WORKDIR}/delta_interim_artifacts/old-image-rootfs.sig > + > + # create delta file with the signature file > + ${SUDO_CHROOT} /usr/bin/rdiff delta ${WORKDIR}/delta_interim_artifacts/old-image-rootfs.sig \ > + ${PP_DEPLOY}/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE} > + ${PP_DEPLOY}/${IMAGE_FULLNAME}.delta > + > + DELTA_ARTIFACT_SWU=${IMAGE_FULLNAME}.delta > + > + # create a symbolic link as IMAGE_CMD expects a *.delta_update file in deploy image directory > + ln -sf ${DELTA_ARTIFACT_SWU} > +${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta_update > +} > + > +create_zchunk_delta_artifact() { > + # Create .zck file > + ${SUDO_CHROOT} /bin/zck \ > + --output ${PP_DEPLOY}/${IMAGE_FULLNAME}.zck \ > + -u --chunk-hash-type sha256 \ > + ${PP_DEPLOY}/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE} > + > + # Calculate size of zck header > + HSIZE="$(${SUDO_CHROOT} /bin/zck_read_header -v ${PP_DEPLOY}/${IMAGE_FULLNAME}.zck | grep "Header size" | cut -d ':' -f 2)" > + > + # Extract the zck header > + ${SUDO_CHROOT} /bin/dd if="${PP_DEPLOY}/${IMAGE_FULLNAME}".zck > + of="${PP_DEPLOY}/${IMAGE_FULLNAME}".zck.header bs=1 count="$HSIZE" > + status=none > + > + DELTA_ARTIFACT_SWU=${IMAGE_FULLNAME}.zck.header > + > + # create a symbolic link as IMAGE_CMD expects a *.delta_update file in deploy image directory > + ln -sf ${DELTA_ARTIFACT_SWU} > +${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta_update > +} > + > +do_image_delta_update[depends] += "${PN}:do_transform_template" > +IMAGE_CMD:delta_update() { > + case "${DELTA_UPDATE_TYPE}" in > + "rdiff") > + create_rdiff_delta_artifact > + ;; > + "zchunk") > + create_zchunk_delta_artifact > + ;; > + *) > + bbfatal "You must set a valid DELTA_UPDATE_TYPE (rdiff/zchunk)" > + ;; > + esac > +} > + > +addtask do_image_delta_update before do_image_swu after do_image_wic > diff --git a/conf/layer.conf b/conf/layer.conf index 82b4084..d4c87a6 > 100644 > --- a/conf/layer.conf > +++ b/conf/layer.conf > @@ -23,6 +23,6 @@ LAYERSERIES_COMPAT_cip-core = "next" > LAYERDIR_cip-core = "${LAYERDIR}" > LAYERDIR_cip-core[vardepvalue] = "isar-cip-core" > > -IMAGE_CLASSES += "squashfs verity swupdate" > +IMAGE_CLASSES += "squashfs verity swupdate delta-update" > > SOURCE_DATE_EPOCH := "${@bb.process.run("git -C ${LAYERDIR_cip-core} log -1 --pretty=%ct | tr -d '\n'")[0]}" > diff --git a/kas/opt/swupdate.yml b/kas/opt/swupdate.yml index > ef61e4e..50ded36 100644 > --- a/kas/opt/swupdate.yml > +++ b/kas/opt/swupdate.yml > @@ -30,3 +30,8 @@ local_conf_header: > ABROOTFS_PART_UUID_B ?= "fedcba98-7654-3210-cafe-5e0710000002" > PREFERRED_PROVIDER_swupdate-certificates-key ??= "swupdate-certificates-key-snakeoil" > PREFERRED_PROVIDER_swupdate-certificates ??= "swupdate-certificates-snakeoil" > + # delta update specific variables" > + IMAGE_FSTYPES:append = " delta_update" > + DELTA_UPDATE_TYPE = "rdiff" > + DELTA_RDIFF_REF_IMAGE = "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" > + DELTA_ZCK_URL = "" > > > > >
On 10.04.24 10:27, Adithya.Balakumar@toshiba-tsip.com wrote: > > > -----Original Message----- > From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On Behalf Of Quirin Gylstorff via lists.cip-project.org > Sent: Monday, April 8, 2024 6:25 PM > To: cip-dev@lists.cip-project.org; jan.kiszka@siemens.com > Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) <dinesh.kumar@toshiba-tsip.com>; balakumar adithya(ＴＳＩＰ TEUR) <Adithya.Balakumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> > Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update support with rdiff_image and delta handler > > > > On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: >> swupdate supports delta updates with rdiff_image and delta(zchunk) >> handler. This change adds support to use either of the handler for >> creating delta update artifacts. zchunk based updates are supported >> only for sid. >> >> DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables >> can be modified based on the update type in the swupdate.yml file. >> >> Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> >> --- >> classes/delta-update.bbclass | 107 +++++++++++++++++++++++++++++++++++ >> conf/layer.conf | 2 +- >> kas/opt/swupdate.yml | 5 ++ >> 3 files changed, 113 insertions(+), 1 deletion(-) >> create mode 100644 classes/delta-update.bbclass >> >> diff --git a/classes/delta-update.bbclass >> b/classes/delta-update.bbclass new file mode 100644 index >> 0000000..8443f35 >> --- /dev/null >> +++ b/classes/delta-update.bbclass >> @@ -0,0 +1,107 @@ >> +# >> +# CIP Core, generic profile >> +# >> +# Copyright (c) Toshiba Corporation 2024 # # Authors: >> +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> >> +# >> +# SPDX-License-Identifier: MIT >> +# >> + >> +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" >> + >> +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" >> +do_image_delta_update[cleandirs] += "${WORKDIR}/delta_interim_artifacts" >> + >> +DELTA_UPDATE_TYPE ??= "rdiff" >> +DELTA_RDIFF_REF_IMAGE ??= "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" >> +DELTA_ZCK_URL ??= "" >> + >> +def disable_delta_update_tasks(d): >> + d.appendVarFlag("do_image_delta_update", "noexec", "1") >> + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") >> + d.setVar("DELTA_UPDATE_TYPE", "") >> + >> +python () { >> + import os >> + path = d.getVar("LAYERDIR_cip-core") + "/" + >> +d.getVar("DELTA_RDIFF_REF_IMAGE") > I would suggest to remove LAYERDIR_cip-core or replace it with a variable as it makes it unusable in downstream layers. This class should not contain any cip-core specific elements. > > [Adithya Balakumar] > Hi Quirin, > > I understand the issue you are pointing at. Could we reference "LAYERDIR_cip-core" through a variable defined in this recipe. As in maybe have a variable (lets say DELTA_REF_ARTIFACT_PATH) with a weak default assignment with the value of "LAYERDIR_cip-core" so that any downstream layer could override the variable with a different value. > Why do we need to reference a layer at all? Can't we just define a generic location, maybe somewhere under DEPLOY_DIR_IMAGE? Jan
-----Original Message----- From: Jan Kiszka <jan.kiszka@siemens.com> Sent: Wednesday, April 10, 2024 2:14 PM To: balakumar adithya(ＴＳＩＰ TEUR) <Adithya.Balakumar@toshiba-tsip.com>; cip-dev@lists.cip-project.org; quirin.gylstorff@siemens.com Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) <dinesh.kumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update support with rdiff_image and delta handler On 10.04.24 10:27, Adithya.Balakumar@toshiba-tsip.com wrote: > > > -----Original Message----- > From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On > Behalf Of Quirin Gylstorff via lists.cip-project.org > Sent: Monday, April 8, 2024 6:25 PM > To: cip-dev@lists.cip-project.org; jan.kiszka@siemens.com > Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) > <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) > <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) > <dinesh.kumar@toshiba-tsip.com>; balakumar adithya(ＴＳＩＰ TEUR) > <Adithya.Balakumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ > ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> > Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update > support with rdiff_image and delta handler > > > > On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: >> swupdate supports delta updates with rdiff_image and delta(zchunk) >> handler. This change adds support to use either of the handler for >> creating delta update artifacts. zchunk based updates are supported >> only for sid. >> >> DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables >> can be modified based on the update type in the swupdate.yml file. >> >> Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> >> --- >> classes/delta-update.bbclass | 107 +++++++++++++++++++++++++++++++++++ >> conf/layer.conf | 2 +- >> kas/opt/swupdate.yml | 5 ++ >> 3 files changed, 113 insertions(+), 1 deletion(-) >> create mode 100644 classes/delta-update.bbclass >> >> diff --git a/classes/delta-update.bbclass >> b/classes/delta-update.bbclass new file mode 100644 index >> 0000000..8443f35 >> --- /dev/null >> +++ b/classes/delta-update.bbclass >> @@ -0,0 +1,107 @@ >> +# >> +# CIP Core, generic profile >> +# >> +# Copyright (c) Toshiba Corporation 2024 # # Authors: >> +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> >> +# >> +# SPDX-License-Identifier: MIT >> +# >> + >> +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" >> + >> +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" >> +do_image_delta_update[cleandirs] += "${WORKDIR}/delta_interim_artifacts" >> + >> +DELTA_UPDATE_TYPE ??= "rdiff" >> +DELTA_RDIFF_REF_IMAGE ??= "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" >> +DELTA_ZCK_URL ??= "" >> + >> +def disable_delta_update_tasks(d): >> + d.appendVarFlag("do_image_delta_update", "noexec", "1") >> + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") >> + d.setVar("DELTA_UPDATE_TYPE", "") >> + >> +python () { >> + import os >> + path = d.getVar("LAYERDIR_cip-core") + "/" + >> +d.getVar("DELTA_RDIFF_REF_IMAGE") > I would suggest to remove LAYERDIR_cip-core or replace it with a variable as it makes it unusable in downstream layers. This class should not contain any cip-core specific elements. > > [Adithya Balakumar] > Hi Quirin, > > I understand the issue you are pointing at. Could we reference "LAYERDIR_cip-core" through a variable defined in this recipe. As in maybe have a variable (lets say DELTA_REF_ARTIFACT_PATH) with a weak default assignment with the value of "LAYERDIR_cip-core" so that any downstream layer could override the variable with a different value. > Why do we need to reference a layer at all? Can't we just define a generic location, maybe somewhere under DEPLOY_DIR_IMAGE? [Adithya Balakumar] Hi Jan, Do you mean to store the previous image build artifact (.squash / .verity) in different folder in DEPLOY_DIR_IMAGE itself and pick up previous image artifact (if available) for the next build to generate the delta update? Jan -- Siemens AG, Technology Linux Expert Center
On 10.04.24 11:23, Adithya.Balakumar@toshiba-tsip.com wrote: > > > -----Original Message----- > From: Jan Kiszka <jan.kiszka@siemens.com> > Sent: Wednesday, April 10, 2024 2:14 PM > To: balakumar adithya(ＴＳＩＰ TEUR) <Adithya.Balakumar@toshiba-tsip.com>; cip-dev@lists.cip-project.org; quirin.gylstorff@siemens.com > Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) <dinesh.kumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> > Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update support with rdiff_image and delta handler > > On 10.04.24 10:27, Adithya.Balakumar@toshiba-tsip.com wrote: >> >> >> -----Original Message----- >> From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On >> Behalf Of Quirin Gylstorff via lists.cip-project.org >> Sent: Monday, April 8, 2024 6:25 PM >> To: cip-dev@lists.cip-project.org; jan.kiszka@siemens.com >> Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) >> <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) >> <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) >> <dinesh.kumar@toshiba-tsip.com>; balakumar adithya(ＴＳＩＰ TEUR) >> <Adithya.Balakumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ >> ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> >> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update >> support with rdiff_image and delta handler >> >> >> >> On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: >>> swupdate supports delta updates with rdiff_image and delta(zchunk) >>> handler. This change adds support to use either of the handler for >>> creating delta update artifacts. zchunk based updates are supported >>> only for sid. >>> >>> DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables >>> can be modified based on the update type in the swupdate.yml file. >>> >>> Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> >>> --- >>> classes/delta-update.bbclass | 107 +++++++++++++++++++++++++++++++++++ >>> conf/layer.conf | 2 +- >>> kas/opt/swupdate.yml | 5 ++ >>> 3 files changed, 113 insertions(+), 1 deletion(-) >>> create mode 100644 classes/delta-update.bbclass >>> >>> diff --git a/classes/delta-update.bbclass >>> b/classes/delta-update.bbclass new file mode 100644 index >>> 0000000..8443f35 >>> --- /dev/null >>> +++ b/classes/delta-update.bbclass >>> @@ -0,0 +1,107 @@ >>> +# >>> +# CIP Core, generic profile >>> +# >>> +# Copyright (c) Toshiba Corporation 2024 # # Authors: >>> +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> >>> +# >>> +# SPDX-License-Identifier: MIT >>> +# >>> + >>> +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" >>> + >>> +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" >>> +do_image_delta_update[cleandirs] += "${WORKDIR}/delta_interim_artifacts" >>> + >>> +DELTA_UPDATE_TYPE ??= "rdiff" >>> +DELTA_RDIFF_REF_IMAGE ??= "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" >>> +DELTA_ZCK_URL ??= "" >>> + >>> +def disable_delta_update_tasks(d): >>> + d.appendVarFlag("do_image_delta_update", "noexec", "1") >>> + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") >>> + d.setVar("DELTA_UPDATE_TYPE", "") >>> + >>> +python () { >>> + import os >>> + path = d.getVar("LAYERDIR_cip-core") + "/" + >>> +d.getVar("DELTA_RDIFF_REF_IMAGE") >> I would suggest to remove LAYERDIR_cip-core or replace it with a variable as it makes it unusable in downstream layers. This class should not contain any cip-core specific elements. >> >> [Adithya Balakumar] >> Hi Quirin, >> >> I understand the issue you are pointing at. Could we reference "LAYERDIR_cip-core" through a variable defined in this recipe. As in maybe have a variable (lets say DELTA_REF_ARTIFACT_PATH) with a weak default assignment with the value of "LAYERDIR_cip-core" so that any downstream layer could override the variable with a different value. >> > > Why do we need to reference a layer at all? Can't we just define a generic location, maybe somewhere under DEPLOY_DIR_IMAGE? > > [Adithya Balakumar] > Hi Jan, > > Do you mean to store the previous image build artifact (.squash / .verity) in different folder in DEPLOY_DIR_IMAGE itself and pick up previous image artifact (if available) for the next build to generate the delta update? > I didn't think this through in details yet, but the basic idea would be to look in some build folder (rather than in layer sources) for the last version. I would not simply use DEPLOY_DIR_IMAGE itself, rather request the user to move the old image explicitly under a path that signals "build delta update as well". Jan
On 4/10/24 12:40 PM, Jan Kiszka wrote: > On 10.04.24 11:23, Adithya.Balakumar@toshiba-tsip.com wrote: >> >> >> -----Original Message----- >> From: Jan Kiszka <jan.kiszka@siemens.com> >> Sent: Wednesday, April 10, 2024 2:14 PM >> To: balakumar adithya(ＴＳＩＰ TEUR) <Adithya.Balakumar@toshiba-tsip.com>; cip-dev@lists.cip-project.org; quirin.gylstorff@siemens.com >> Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) <dinesh.kumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> >> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update support with rdiff_image and delta handler >> >> On 10.04.24 10:27, Adithya.Balakumar@toshiba-tsip.com wrote: >>> >>> >>> -----Original Message----- >>> From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On >>> Behalf Of Quirin Gylstorff via lists.cip-project.org >>> Sent: Monday, April 8, 2024 6:25 PM >>> To: cip-dev@lists.cip-project.org; jan.kiszka@siemens.com >>> Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) >>> <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) >>> <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) >>> <dinesh.kumar@toshiba-tsip.com>; balakumar adithya(ＴＳＩＰ TEUR) >>> <Adithya.Balakumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ >>> ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> >>> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update >>> support with rdiff_image and delta handler >>> >>> >>> >>> On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: >>>> swupdate supports delta updates with rdiff_image and delta(zchunk) >>>> handler. This change adds support to use either of the handler for >>>> creating delta update artifacts. zchunk based updates are supported >>>> only for sid. >>>> >>>> DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables >>>> can be modified based on the update type in the swupdate.yml file. >>>> >>>> Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> >>>> --- >>>> classes/delta-update.bbclass | 107 +++++++++++++++++++++++++++++++++++ >>>> conf/layer.conf | 2 +- >>>> kas/opt/swupdate.yml | 5 ++ >>>> 3 files changed, 113 insertions(+), 1 deletion(-) >>>> create mode 100644 classes/delta-update.bbclass >>>> >>>> diff --git a/classes/delta-update.bbclass >>>> b/classes/delta-update.bbclass new file mode 100644 index >>>> 0000000..8443f35 >>>> --- /dev/null >>>> +++ b/classes/delta-update.bbclass >>>> @@ -0,0 +1,107 @@ >>>> +# >>>> +# CIP Core, generic profile >>>> +# >>>> +# Copyright (c) Toshiba Corporation 2024 # # Authors: >>>> +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> >>>> +# >>>> +# SPDX-License-Identifier: MIT >>>> +# >>>> + >>>> +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" >>>> + >>>> +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" >>>> +do_image_delta_update[cleandirs] += "${WORKDIR}/delta_interim_artifacts" >>>> + >>>> +DELTA_UPDATE_TYPE ??= "rdiff" >>>> +DELTA_RDIFF_REF_IMAGE ??= "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" >>>> +DELTA_ZCK_URL ??= "" >>>> + >>>> +def disable_delta_update_tasks(d): >>>> + d.appendVarFlag("do_image_delta_update", "noexec", "1") >>>> + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") >>>> + d.setVar("DELTA_UPDATE_TYPE", "") >>>> + >>>> +python () { >>>> + import os >>>> + path = d.getVar("LAYERDIR_cip-core") + "/" + >>>> +d.getVar("DELTA_RDIFF_REF_IMAGE") >>> I would suggest to remove LAYERDIR_cip-core or replace it with a variable as it makes it unusable in downstream layers. This class should not contain any cip-core specific elements. >>> >>> [Adithya Balakumar] >>> Hi Quirin, >>> >>> I understand the issue you are pointing at. Could we reference "LAYERDIR_cip-core" through a variable defined in this recipe. As in maybe have a variable (lets say DELTA_REF_ARTIFACT_PATH) with a weak default assignment with the value of "LAYERDIR_cip-core" so that any downstream layer could override the variable with a different value. >>> >> >> Why do we need to reference a layer at all? Can't we just define a generic location, maybe somewhere under DEPLOY_DIR_IMAGE? >> >> [Adithya Balakumar] >> Hi Jan, >> >> Do you mean to store the previous image build artifact (.squash / .verity) in different folder in DEPLOY_DIR_IMAGE itself and pick up previous image artifact (if available) for the next build to generate the delta update? >> > > I didn't think this through in details yet, but the basic idea would be > to look in some build folder (rather than in layer sources) for the last > version. I would not simply use DEPLOY_DIR_IMAGE itself, rather request > the user to move the old image explicitly under a path that signals > "build delta update as well". You can use ${TOPDIR}/<some directory>, e.g. `${TOPDIR}/delta-image/`, to search for updates. TOPDIR is the current work directory, normally `./build` Quirin > > Jan >
On 10.04.24 15:24, Gylstorff Quirin wrote: > > > On 4/10/24 12:40 PM, Jan Kiszka wrote: >> On 10.04.24 11:23, Adithya.Balakumar@toshiba-tsip.com wrote: >>> >>> >>> -----Original Message----- >>> From: Jan Kiszka <jan.kiszka@siemens.com> >>> Sent: Wednesday, April 10, 2024 2:14 PM >>> To: balakumar adithya(ＴＳＩＰ TEUR) >>> <Adithya.Balakumar@toshiba-tsip.com>; cip-dev@lists.cip-project.org; >>> quirin.gylstorff@siemens.com >>> Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) >>> <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) >>> <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG >>> Porting) <dinesh.kumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 Ｄ >>> ＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> >>> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update >>> support with rdiff_image and delta handler >>> >>> On 10.04.24 10:27, Adithya.Balakumar@toshiba-tsip.com wrote: >>>> >>>> >>>> -----Original Message----- >>>> From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On >>>> Behalf Of Quirin Gylstorff via lists.cip-project.org >>>> Sent: Monday, April 8, 2024 6:25 PM >>>> To: cip-dev@lists.cip-project.org; jan.kiszka@siemens.com >>>> Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) >>>> <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) >>>> <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG >>>> Porting) >>>> <dinesh.kumar@toshiba-tsip.com>; balakumar adithya(ＴＳＩＰ TEUR) >>>> <Adithya.Balakumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ >>>> ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> >>>> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update >>>> support with rdiff_image and delta handler >>>> >>>> >>>> >>>> On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: >>>>> swupdate supports delta updates with rdiff_image and delta(zchunk) >>>>> handler. This change adds support to use either of the handler for >>>>> creating delta update artifacts. zchunk based updates are supported >>>>> only for sid. >>>>> >>>>> DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables >>>>> can be modified based on the update type in the swupdate.yml file. >>>>> >>>>> Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> >>>>> --- >>>>> classes/delta-update.bbclass | 107 >>>>> +++++++++++++++++++++++++++++++++++ >>>>> conf/layer.conf | 2 +- >>>>> kas/opt/swupdate.yml | 5 ++ >>>>> 3 files changed, 113 insertions(+), 1 deletion(-) >>>>> create mode 100644 classes/delta-update.bbclass >>>>> >>>>> diff --git a/classes/delta-update.bbclass >>>>> b/classes/delta-update.bbclass new file mode 100644 index >>>>> 0000000..8443f35 >>>>> --- /dev/null >>>>> +++ b/classes/delta-update.bbclass >>>>> @@ -0,0 +1,107 @@ >>>>> +# >>>>> +# CIP Core, generic profile >>>>> +# >>>>> +# Copyright (c) Toshiba Corporation 2024 # # Authors: >>>>> +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> >>>>> +# >>>>> +# SPDX-License-Identifier: MIT >>>>> +# >>>>> + >>>>> +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" >>>>> + >>>>> +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" >>>>> +do_image_delta_update[cleandirs] += >>>>> "${WORKDIR}/delta_interim_artifacts" >>>>> + >>>>> +DELTA_UPDATE_TYPE ??= "rdiff" >>>>> +DELTA_RDIFF_REF_IMAGE ??= >>>>> "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" >>>>> +DELTA_ZCK_URL ??= "" >>>>> + >>>>> +def disable_delta_update_tasks(d): >>>>> + d.appendVarFlag("do_image_delta_update", "noexec", "1") >>>>> + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") >>>>> + d.setVar("DELTA_UPDATE_TYPE", "") >>>>> + >>>>> +python () { >>>>> + import os >>>>> + path = d.getVar("LAYERDIR_cip-core") + "/" + >>>>> +d.getVar("DELTA_RDIFF_REF_IMAGE") >>>> I would suggest to remove LAYERDIR_cip-core or replace it with a >>>> variable as it makes it unusable in downstream layers. This class >>>> should not contain any cip-core specific elements. >>>> >>>> [Adithya Balakumar] >>>> Hi Quirin, >>>> >>>> I understand the issue you are pointing at. Could we reference >>>> "LAYERDIR_cip-core" through a variable defined in this recipe. As in >>>> maybe have a variable (lets say DELTA_REF_ARTIFACT_PATH) with a weak >>>> default assignment with the value of "LAYERDIR_cip-core" so that any >>>> downstream layer could override the variable with a different value. >>>> >>> >>> Why do we need to reference a layer at all? Can't we just define a >>> generic location, maybe somewhere under DEPLOY_DIR_IMAGE? >>> >>> [Adithya Balakumar] >>> Hi Jan, >>> >>> Do you mean to store the previous image build artifact (.squash / >>> .verity) in different folder in DEPLOY_DIR_IMAGE itself and pick up >>> previous image artifact (if available) for the next build to generate >>> the delta update? >>> >> >> I didn't think this through in details yet, but the basic idea would be >> to look in some build folder (rather than in layer sources) for the last >> version. I would not simply use DEPLOY_DIR_IMAGE itself, rather request >> the user to move the old image explicitly under a path that signals >> "build delta update as well". > > You can use ${TOPDIR}/<some directory>, e.g. `${TOPDIR}/delta-image/`, > to search for updates. > > TOPDIR is the current work directory, normally `./build` > Maybe better ${TOPDIR}/previous-image/. Which artifact(s) need to be copied there again then? Jan
-----Original Message----- From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> On Behalf Of Jan Kiszka via lists.cip-project.org Sent: Wednesday, April 10, 2024 8:05 PM To: Gylstorff Quirin <quirin.gylstorff@siemens.com>; balakumar adithya(ＴＳＩＰ TEUR) <Adithya.Balakumar@toshiba-tsip.com>; cip-dev@lists.cip-project.org Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG Porting) <dinesh.kumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update support with rdiff_image and delta handler On 10.04.24 15:24, Gylstorff Quirin wrote: > > > On 4/10/24 12:40 PM, Jan Kiszka wrote: >> On 10.04.24 11:23, Adithya.Balakumar@toshiba-tsip.com wrote: >>> >>> >>> -----Original Message----- >>> From: Jan Kiszka <jan.kiszka@siemens.com> >>> Sent: Wednesday, April 10, 2024 2:14 PM >>> To: balakumar adithya(ＴＳＩＰ TEUR) >>> <Adithya.Balakumar@toshiba-tsip.com>; cip-dev@lists.cip-project.org; >>> quirin.gylstorff@siemens.com >>> Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) >>> <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) >>> <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG >>> Porting) <dinesh.kumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 Ｄ >>> ＭＥ ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> >>> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update >>> support with rdiff_image and delta handler >>> >>> On 10.04.24 10:27, Adithya.Balakumar@toshiba-tsip.com wrote: >>>> >>>> >>>> -----Original Message----- >>>> From: cip-dev@lists.cip-project.org <cip-dev@lists.cip-project.org> >>>> On Behalf Of Quirin Gylstorff via lists.cip-project.org >>>> Sent: Monday, April 8, 2024 6:25 PM >>>> To: cip-dev@lists.cip-project.org; jan.kiszka@siemens.com >>>> Cc: kunijadar shivanand(ＴＳＩＰ TMIEC ODG Porting) >>>> <Shivanand.Kunijadar@toshiba-tsip.com>; ashrith sai(ＴＳＩＰ) >>>> <Sai.Sathujoda@toshiba-tsip.com>; dinesh kumar(ＴＳＩＰ TMIEC ODG >>>> Porting) >>>> <dinesh.kumar@toshiba-tsip.com>; balakumar adithya(ＴＳＩＰ TEUR) >>>> <Adithya.Balakumar@toshiba-tsip.com>; hayashi kazuhiro(林 和宏 ＤＭＥ >>>> ○ＤＩＧ□ＭＰＳ○ＭＰ４) <kazuhiro3.hayashi@toshiba.co.jp> >>>> Subject: Re: [cip-dev] [isar-cip-core][RFC v2 2/4] Add Delta update >>>> support with rdiff_image and delta handler >>>> >>>> >>>> >>>> On 3/20/24 11:32 AM, Adithya Balakumar via lists.cip-project.org wrote: >>>>> swupdate supports delta updates with rdiff_image and delta(zchunk) >>>>> handler. This change adds support to use either of the handler for >>>>> creating delta update artifacts. zchunk based updates are >>>>> supported only for sid. >>>>> >>>>> DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL >>>>> variables can be modified based on the update type in the swupdate.yml file. >>>>> >>>>> Signed-off-by: Adithya Balakumar >>>>> <Adithya.Balakumar@toshiba-tsip.com> >>>>> --- >>>>> classes/delta-update.bbclass | 107 >>>>> +++++++++++++++++++++++++++++++++++ >>>>> conf/layer.conf | 2 +- >>>>> kas/opt/swupdate.yml | 5 ++ >>>>> 3 files changed, 113 insertions(+), 1 deletion(-) >>>>> create mode 100644 classes/delta-update.bbclass >>>>> >>>>> diff --git a/classes/delta-update.bbclass >>>>> b/classes/delta-update.bbclass new file mode 100644 index >>>>> 0000000..8443f35 >>>>> --- /dev/null >>>>> +++ b/classes/delta-update.bbclass >>>>> @@ -0,0 +1,107 @@ >>>>> +# >>>>> +# CIP Core, generic profile >>>>> +# >>>>> +# Copyright (c) Toshiba Corporation 2024 # # Authors: >>>>> +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> >>>>> +# >>>>> +# SPDX-License-Identifier: MIT >>>>> +# >>>>> + >>>>> +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" >>>>> + >>>>> +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" >>>>> +do_image_delta_update[cleandirs] += >>>>> "${WORKDIR}/delta_interim_artifacts" >>>>> + >>>>> +DELTA_UPDATE_TYPE ??= "rdiff" >>>>> +DELTA_RDIFF_REF_IMAGE ??= >>>>> "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" >>>>> +DELTA_ZCK_URL ??= "" >>>>> + >>>>> +def disable_delta_update_tasks(d): >>>>> + d.appendVarFlag("do_image_delta_update", "noexec", "1") >>>>> + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", >>>>> +"1") >>>>> + d.setVar("DELTA_UPDATE_TYPE", "") >>>>> + >>>>> +python () { >>>>> + import os >>>>> + path = d.getVar("LAYERDIR_cip-core") + "/" + >>>>> +d.getVar("DELTA_RDIFF_REF_IMAGE") >>>> I would suggest to remove LAYERDIR_cip-core or replace it with a >>>> variable as it makes it unusable in downstream layers. This class >>>> should not contain any cip-core specific elements. >>>> >>>> [Adithya Balakumar] >>>> Hi Quirin, >>>> >>>> I understand the issue you are pointing at. Could we reference >>>> "LAYERDIR_cip-core" through a variable defined in this recipe. As >>>> in maybe have a variable (lets say DELTA_REF_ARTIFACT_PATH) with a >>>> weak default assignment with the value of "LAYERDIR_cip-core" so >>>> that any downstream layer could override the variable with a different value. >>>> >>> >>> Why do we need to reference a layer at all? Can't we just define a >>> generic location, maybe somewhere under DEPLOY_DIR_IMAGE? >>> >>> [Adithya Balakumar] >>> Hi Jan, >>> >>> Do you mean to store the previous image build artifact (.squash / >>> .verity) in different folder in DEPLOY_DIR_IMAGE itself and pick up >>> previous image artifact (if available) for the next build to >>> generate the delta update? >>> >> >> I didn't think this through in details yet, but the basic idea would >> be to look in some build folder (rather than in layer sources) for >> the last version. I would not simply use DEPLOY_DIR_IMAGE itself, >> rather request the user to move the old image explicitly under a path >> that signals "build delta update as well". > > You can use ${TOPDIR}/<some directory>, e.g. `${TOPDIR}/delta-image/`, > to search for updates. > > TOPDIR is the current work directory, normally `./build` > Maybe better ${TOPDIR}/previous-image/. Which artifact(s) need to be copied there again then? [Adithya Balakumar] Thanks for the suggestions Jan and Quirin. I think ${TOPDIR}/previous-image/ is a good option. As for which artifacts need to be copied there, for rdiff based delta creation we require the rootfs artifact of the previous image. So we would need to copy the .squashfs file / .verity file (for SB enabled image). Adithya >Jan -- Siemens AG, Technology Linux Expert Center
diff --git a/classes/delta-update.bbclass b/classes/delta-update.bbclass new file mode 100644 index 0000000..8443f35 --- /dev/null +++ b/classes/delta-update.bbclass @@ -0,0 +1,107 @@ +# +# CIP Core, generic profile +# +# Copyright (c) Toshiba Corporation 2024 +# +# Authors: +# Adithya Balakumar <adithya.balakumar@toshiba-tsip.com> +# +# SPDX-License-Identifier: MIT +# + +IMAGER_INSTALL:delta_update .= "fdisk zchunk rdiff" + +FILESEXTRAPATHS:append = ":${LAYERDIR_cip-core}" +do_image_delta_update[cleandirs] += "${WORKDIR}/delta_interim_artifacts" + +DELTA_UPDATE_TYPE ??= "rdiff" +DELTA_RDIFF_REF_IMAGE ??= "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" +DELTA_ZCK_URL ??= "" + +def disable_delta_update_tasks(d): + d.appendVarFlag("do_image_delta_update", "noexec", "1") + d.appendVarFlag("do_delta_rdiff_ref_image_fetch", "noexec", "1") + d.setVar("DELTA_UPDATE_TYPE", "") + +python () { + import os + path = d.getVar("LAYERDIR_cip-core") + "/" + d.getVar("DELTA_RDIFF_REF_IMAGE") + if d.getVar("DELTA_UPDATE_TYPE") == "rdiff": + if not os.path.isfile(path): + disable_delta_update_tasks(d) + elif d.getVar("DELTA_UPDATE_TYPE") == "zchunk": + if d.getVar("DISTRO") != "cip-core-sid": + disable_delta_update_tasks(d) + else: + disable_delta_update_tasks(d) +} + +python do_delta_rdiff_ref_image_fetch () { + import bb + import os + path = d.getVar("LAYERDIR_cip-core") + "/" + d.getVar("DELTA_RDIFF_REF_IMAGE") + if d.getVar("DELTA_UPDATE_TYPE") == "rdiff": + if os.path.isfile(path): + artifact_uri = "file://" + (d.getVar('DELTA_RDIFF_REF_IMAGE') or "") + artifact_uri_list = artifact_uri.split() + fetcher = bb.fetch2.Fetch(artifact_uri_list, d) + fetcher.download() + fetcher.unpack(d.getVar('WORKDIR')) +} +addtask do_delta_rdiff_ref_image_fetch before do_transform_template + +create_rdiff_delta_artifact() { + if [ -z "${DELTA_RDIFF_REF_IMAGE}" ]; then + bbfatal "You must set DELTA_RDIFF_REF_IMAGE and provide the required files as artifacts to this recipe" + fi + + rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta + # create signature file with rdiff + ${SUDO_CHROOT} /usr/bin/rdiff signature ${WORKDIR}/${DELTA_RDIFF_REF_IMAGE} \ + ${WORKDIR}/delta_interim_artifacts/old-image-rootfs.sig + + # create delta file with the signature file + ${SUDO_CHROOT} /usr/bin/rdiff delta ${WORKDIR}/delta_interim_artifacts/old-image-rootfs.sig \ + ${PP_DEPLOY}/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE} ${PP_DEPLOY}/${IMAGE_FULLNAME}.delta + + DELTA_ARTIFACT_SWU=${IMAGE_FULLNAME}.delta + + # create a symbolic link as IMAGE_CMD expects a *.delta_update file in deploy image directory + ln -sf ${DELTA_ARTIFACT_SWU} ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta_update +} + +create_zchunk_delta_artifact() { + # Create .zck file + ${SUDO_CHROOT} /bin/zck \ + --output ${PP_DEPLOY}/${IMAGE_FULLNAME}.zck \ + -u --chunk-hash-type sha256 \ + ${PP_DEPLOY}/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE} + + # Calculate size of zck header + HSIZE="$(${SUDO_CHROOT} /bin/zck_read_header -v ${PP_DEPLOY}/${IMAGE_FULLNAME}.zck | grep "Header size" | cut -d ':' -f 2)" + + # Extract the zck header + ${SUDO_CHROOT} /bin/dd if="${PP_DEPLOY}/${IMAGE_FULLNAME}".zck of="${PP_DEPLOY}/${IMAGE_FULLNAME}".zck.header bs=1 count="$HSIZE" status=none + + DELTA_ARTIFACT_SWU=${IMAGE_FULLNAME}.zck.header + + # create a symbolic link as IMAGE_CMD expects a *.delta_update file in deploy image directory + ln -sf ${DELTA_ARTIFACT_SWU} ${DEPLOY_DIR_IMAGE}/${IMAGE_FULLNAME}.delta_update +} + +do_image_delta_update[depends] += "${PN}:do_transform_template" +IMAGE_CMD:delta_update() { + case "${DELTA_UPDATE_TYPE}" in + "rdiff") + create_rdiff_delta_artifact + ;; + "zchunk") + create_zchunk_delta_artifact + ;; + *) + bbfatal "You must set a valid DELTA_UPDATE_TYPE (rdiff/zchunk)" + ;; + esac +} + +addtask do_image_delta_update before do_image_swu after do_image_wic diff --git a/conf/layer.conf b/conf/layer.conf index 82b4084..d4c87a6 100644 --- a/conf/layer.conf +++ b/conf/layer.conf @@ -23,6 +23,6 @@ LAYERSERIES_COMPAT_cip-core = "next" LAYERDIR_cip-core = "${LAYERDIR}" LAYERDIR_cip-core[vardepvalue] = "isar-cip-core" -IMAGE_CLASSES += "squashfs verity swupdate" +IMAGE_CLASSES += "squashfs verity swupdate delta-update" SOURCE_DATE_EPOCH := "${@bb.process.run("git -C ${LAYERDIR_cip-core} log -1 --pretty=%ct | tr -d '\n'")[0]}" diff --git a/kas/opt/swupdate.yml b/kas/opt/swupdate.yml index ef61e4e..50ded36 100644 --- a/kas/opt/swupdate.yml +++ b/kas/opt/swupdate.yml @@ -30,3 +30,8 @@ local_conf_header: ABROOTFS_PART_UUID_B ?= "fedcba98-7654-3210-cafe-5e0710000002" PREFERRED_PROVIDER_swupdate-certificates-key ??= "swupdate-certificates-key-snakeoil" PREFERRED_PROVIDER_swupdate-certificates ??= "swupdate-certificates-snakeoil" + # delta update specific variables" + IMAGE_FSTYPES:append = " delta_update" + DELTA_UPDATE_TYPE = "rdiff" + DELTA_RDIFF_REF_IMAGE = "image-v1/${IMAGE_FULLNAME}.${SWU_ROOTFS_TYPE}" + DELTA_ZCK_URL = ""
swupdate supports delta updates with rdiff_image and delta(zchunk) handler. This change adds support to use either of the handler for creating delta update artifacts. zchunk based updates are supported only for sid. DELTA_UPDATE_TYPE, DELTA_RDIFF_REF_IMAGE and DELTA_ZCK_URL variables can be modified based on the update type in the swupdate.yml file. Signed-off-by: Adithya Balakumar <Adithya.Balakumar@toshiba-tsip.com> --- classes/delta-update.bbclass | 107 +++++++++++++++++++++++++++++++++++ conf/layer.conf | 2 +- kas/opt/swupdate.yml | 5 ++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 classes/delta-update.bbclass