| Message ID | 20190520124716.30472-2-kraxel@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | tests/vm: serial console autoinstall, misc fixes. | expand |
On 5/20/19 2:47 PM, Gerd Hoffmann wrote: > Use git archive to create tarballs of qemu and submodules instead of > cloning the repository and the submodules. This is a order of magnitude > faster because it doesn't fetch the submodules from the internet each > time the script runs. > > Signed-off-by: Gerd Hoffmann <kraxel@redhat.com> > Tested-by: Thomas Huth <thuth@redhat.com> > --- > scripts/archive-source.sh | 72 +++++++++++++++++---------------------- > 1 file changed, 31 insertions(+), 41 deletions(-) > > diff --git a/scripts/archive-source.sh b/scripts/archive-source.sh > index 8b8994826000..93a48d2063f9 100755 > --- a/scripts/archive-source.sh > +++ b/scripts/archive-source.sh > @@ -19,16 +19,25 @@ if test $# -lt 1; then > fi > > tar_file=$(realpath "$1") > -list_file="${tar_file}.list" > -vroot_dir="${tar_file}.vroot" > +sub_tdir=$(mktemp -d "${tar_file%.tar}.sub.XXXXXXXX") > +sub_file="${sub_tdir}/submodule.tar" > > # We want a predictable list of submodules for builds, that is > # independent of what the developer currently has initialized > # in their checkout, because the build environment is completely > # different to the host OS. > submodules="dtc slirp ui/keycodemapdb tests/fp/berkeley-softfloat-3 tests/fp/berkeley-testfloat-3" > +sub_deinit="" > > -trap "status=$?; rm -rf \"$list_file\" \"$vroot_dir\"; exit \$status" 0 1 2 3 15 > +function cleanup() { > + local status=$? > + rm -rf "$sub_tdir" > + if test "$sub_deinit" != ""; then > + git submodule deinit $sub_deinit > + fi > + exit $status > +} > +trap "cleanup" 0 1 2 3 15 > > if git diff-index --quiet HEAD -- &>/dev/null > then > @@ -36,45 +45,26 @@ then > else > HEAD=$(git stash create) > fi > -git clone --shared . "$vroot_dir" > -test $? -ne 0 && error "failed to clone into '$vroot_dir'" > -for sm in $submodules; do > - if test -d "$sm/.git" > - then > - git clone --shared "$sm" "$vroot_dir/$sm" > - test $? -ne 0 && error "failed to clone submodule $sm" > - fi > -done > - > -cd "$vroot_dir" > -test $? -ne 0 && error "failed to change into '$vroot_dir'" > - > -git checkout $HEAD > -test $? -ne 0 && error "failed to checkout $HEAD revision" > > +git archive --format tar $HEAD > "$tar_file" > +test $? -ne 0 && error "failed to archive qemu" > for sm in $submodules; do > - git submodule update --init $sm > - test $? -ne 0 && error "failed to init submodule $sm" > + status="$(git submodule status "$sm")" > + smhash="${status#[ +-]}" > + smhash="${smhash%% *}" > + case "$status" in > + -*) > + sub_deinit="$sub_deinit $sm" > + git submodule update --init "$sm" > + test $? -ne 0 && error "failed to update submodule $sm" > + ;; > + +*) > + echo "WARNING: submodule $sm is out of sync" > + ;; > + esac > + (cd $sm; git archive --format tar --prefix "$sm/" $smhash) > "$sub_file" > + test $? -ne 0 && error "failed to archive submodule $sm ($smhash)" > + tar --concatenate --file "$tar_file" "$sub_file" > + test $? -ne 0 && error "failed append submodule $sm to $tar_file" > done > - > -if test -n "$submodules"; then > - { > - git ls-files || error "git ls-files failed" > - for sm in $submodules; do > - (cd $sm; git ls-files) | sed "s:^:$sm/:" > - if test "${PIPESTATUS[*]}" != "0 0"; then > - error "git ls-files in submodule $sm failed" > - fi > - done > - } | grep -x -v $(for sm in $submodules; do echo "-e $sm"; done) > "$list_file" > -else > - git ls-files > "$list_file" > -fi > - > -if test $? -ne 0; then > - error "failed to generate list file" > -fi > - > -tar -cf "$tar_file" -T "$list_file" || error "failed to create tar file" > - > exit 0 > Tested-by: Philippe Mathieu-Daudé <philmd@redhat.com>
diff --git a/scripts/archive-source.sh b/scripts/archive-source.sh index 8b8994826000..93a48d2063f9 100755 --- a/scripts/archive-source.sh +++ b/scripts/archive-source.sh @@ -19,16 +19,25 @@ if test $# -lt 1; then fi tar_file=$(realpath "$1") -list_file="${tar_file}.list" -vroot_dir="${tar_file}.vroot" +sub_tdir=$(mktemp -d "${tar_file%.tar}.sub.XXXXXXXX") +sub_file="${sub_tdir}/submodule.tar" # We want a predictable list of submodules for builds, that is # independent of what the developer currently has initialized # in their checkout, because the build environment is completely # different to the host OS. submodules="dtc slirp ui/keycodemapdb tests/fp/berkeley-softfloat-3 tests/fp/berkeley-testfloat-3" +sub_deinit="" -trap "status=$?; rm -rf \"$list_file\" \"$vroot_dir\"; exit \$status" 0 1 2 3 15 +function cleanup() { + local status=$? + rm -rf "$sub_tdir" + if test "$sub_deinit" != ""; then + git submodule deinit $sub_deinit + fi + exit $status +} +trap "cleanup" 0 1 2 3 15 if git diff-index --quiet HEAD -- &>/dev/null then @@ -36,45 +45,26 @@ then else HEAD=$(git stash create) fi -git clone --shared . "$vroot_dir" -test $? -ne 0 && error "failed to clone into '$vroot_dir'" -for sm in $submodules; do - if test -d "$sm/.git" - then - git clone --shared "$sm" "$vroot_dir/$sm" - test $? -ne 0 && error "failed to clone submodule $sm" - fi -done - -cd "$vroot_dir" -test $? -ne 0 && error "failed to change into '$vroot_dir'" - -git checkout $HEAD -test $? -ne 0 && error "failed to checkout $HEAD revision" +git archive --format tar $HEAD > "$tar_file" +test $? -ne 0 && error "failed to archive qemu" for sm in $submodules; do - git submodule update --init $sm - test $? -ne 0 && error "failed to init submodule $sm" + status="$(git submodule status "$sm")" + smhash="${status#[ +-]}" + smhash="${smhash%% *}" + case "$status" in + -*) + sub_deinit="$sub_deinit $sm" + git submodule update --init "$sm" + test $? -ne 0 && error "failed to update submodule $sm" + ;; + +*) + echo "WARNING: submodule $sm is out of sync" + ;; + esac + (cd $sm; git archive --format tar --prefix "$sm/" $smhash) > "$sub_file" + test $? -ne 0 && error "failed to archive submodule $sm ($smhash)" + tar --concatenate --file "$tar_file" "$sub_file" + test $? -ne 0 && error "failed append submodule $sm to $tar_file" done - -if test -n "$submodules"; then - { - git ls-files || error "git ls-files failed" - for sm in $submodules; do - (cd $sm; git ls-files) | sed "s:^:$sm/:" - if test "${PIPESTATUS[*]}" != "0 0"; then - error "git ls-files in submodule $sm failed" - fi - done - } | grep -x -v $(for sm in $submodules; do echo "-e $sm"; done) > "$list_file" -else - git ls-files > "$list_file" -fi - -if test $? -ne 0; then - error "failed to generate list file" -fi - -tar -cf "$tar_file" -T "$list_file" || error "failed to create tar file" - exit 0