diff mbox series

[RFC,1/2] scripts: Add script to do the repetitive bits of the release process

Message ID 20190405171342.10902-1-george.dunlap@citrix.com (mailing list archive)
State New, archived
Headers show
Series [RFC,1/2] scripts: Add script to do the repetitive bits of the release process | expand

Commit Message

George Dunlap April 5, 2019, 5:13 p.m. UTC 
With this script, once the main checks are out of the way, doing a
release (either an RC or the final release) should mostly be a matter
of executing a sequence of 4 commands given by the `help` function in
this script.

Signed-off-by: George Dunlap <george.dunlap@citrix.com>
---
There's one hard-coded "default" path in here that refers to my own
directory structure.  If Ian finds these scripts useful, we should
probably move that to a copy on mail.xenproject.org somewhere instead.

There are also lots of opportunities for this script to be improved,
by (for instance) implementing programmatic checks for the various
checks listed as 'manual' at the moment.

I plan to implement containerize-able tests for the first three steps
(tag, make tarball, push tag), using "dummy" paths and gpg keys.  I've
made revisions to tarball-cvs-checkin-and-post which I haven't had the
opportunity to test yet; ideas for how to keep this "fresh" are
welcome.

CC: Ian Jackson <ian.jackson@citrix.com>
---
 scripts/release | 450 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 450 insertions(+)
 create mode 100755 scripts/release

Comments

George Dunlap May 1, 2019, 2:56 p.m. UTC | #1 
Ping?

On 4/5/19 6:13 PM, George Dunlap wrote:
> With this script, once the main checks are out of the way, doing a
> release (either an RC or the final release) should mostly be a matter
> of executing a sequence of 4 commands given by the `help` function in
> this script.
> 
> Signed-off-by: George Dunlap <george.dunlap@citrix.com>
> ---
> There's one hard-coded "default" path in here that refers to my own
> directory structure.  If Ian finds these scripts useful, we should
> probably move that to a copy on mail.xenproject.org somewhere instead.
> 
> There are also lots of opportunities for this script to be improved,
> by (for instance) implementing programmatic checks for the various
> checks listed as 'manual' at the moment.
> 
> I plan to implement containerize-able tests for the first three steps
> (tag, make tarball, push tag), using "dummy" paths and gpg keys.  I've
> made revisions to tarball-cvs-checkin-and-post which I haven't had the
> opportunity to test yet; ideas for how to keep this "fresh" are
> welcome.
> 
> CC: Ian Jackson <ian.jackson@citrix.com>
> ---
>  scripts/release | 450 ++++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 450 insertions(+)
>  create mode 100755 scripts/release
> 
> diff --git a/scripts/release b/scripts/release
> new file mode 100755
> index 0000000000..0442cd4ef9
> --- /dev/null
> +++ b/scripts/release
> @@ -0,0 +1,450 @@
> +#!/bin/bash
> +
> +###
> +# George's bash library core
> +###
> +
> +# arg-parse debug
> +_apd=false
> +
> +arg_parse_cmd=\
> +"local -a args;
> +local _a;
> +local _vn;
> +local _m;
> +local CLVL=\$((\$CLVL+1))
> +
> +_m=true;
> +
> +for _a in \"\$@\" ; do
> +    $_apd && echo \"Evaluating \${_a} [[ \"\${_a/=}\" = \"\${_a}\" ]]\";
> +    if \$_m && [[ \"\${_a/=}\" != \"\${_a}\" ]] ; then
> +        $_apd && echo Parameter;
> +        _vn=\${_a%%=*};
> +        eval \"local \$_vn\";
> +        eval \"\$_a\";
> +    elif \$_m && [[ \"\${_a}\" == \"--\" ]] ; then
> +        $_apd && echo Separator;
> +        _m=false;
> +    else
> +        $_apd && echo Argument;
> +        _m=false;
> +        args+=(\"\$_a\");
> +    fi;
> +done"
> +
> +arg_parse="eval $arg_parse_cmd"
> +
> +# Pass in either the current function name, or the name of the script
> +requireargs="eval _func=\"\$FUNCNAME\" ; eval [[ -n \\\"\$_func\\\" ]] || _func=\$0 ; eval _require-args \$_func"
> +
> +function _require-args()
> +{
> +    local _arg
> +    local _args
> +
> +    _args=($@)
> +
> +    for _arg in ${_args[@]:1} ; do
> +	eval "[[ -n \"\${$_arg}\" ]] || fail \"${_args[0]}: Missing $_arg\""
> +    done
> +}
> +
> +function default()
> +{
> +    # l0: eval i="5"
> +    # l1: default_post="eval $1=\"$2\""
> +    # l3: eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval \$1=\\\"$2\\\"\" ; fi"
> +    eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval local \$1=\\\"$2\\\"\" ; else unset default_post ; fi"
> +}
> +
> +function fail()
> +{
> +   echo FATAL $@
> +   [[ -n "$fail_cleanup" ]] && $fail_cleanup
> +   exit 1
> +}
> +
> +function info()
> +{
> +   echo INFO $CLVL $@ 1>&2
> +}
> +
> +function error()
> +{
> +   echo ERROR $@ 1>&2
> +}
> +
> +function status()
> +{
> +   echo STATUS $CLVL $@ 1>&2
> +   return 0
> +}
> +
> +function report-result()
> +{
> +    if [[ -n "$var" ]] ; then
> +	eval "${var}=\"$1\""
> +    else
> +	if [[ -n "$1" ]] ; then
> +	    echo "$1"
> +	else
> +	    echo "(empty)"
> +	fi
> +    fi
> +}
> +
> +function cmdline()
> +{
> +    local cmd;
> +
> +    if [[ "$#" -eq "0" ]] ; then
> +	help
> +	exit 1
> +    fi
> +
> +    $arg_parse
> +    info Running "${args[0]}"
> +    "${args[0]}" "${args[@]:1}" || exit 1
> +
> +    if ! [[ -z "$RET" ]] ; then
> +	echo $RET
> +    fi
> +}
> +
> +###
> +# release-specific code
> +###
> +
> +# Global / meta variables:
> +#
> +# tdir: "root" directory to do tarball work.
> +# rdir: Directory where tarball & sig will be put (==$tdir/$v)
> +# rtgz: Base filename for tarball ($rdir/xen-$v.tar.gz)
> +#
> +# v: Full release version (e.g., 4.12.0-rc5, 4.10.3)
> +# x: Major+minor xen release version (e.g., 4.12, 4.10)
> +# p: point release (e.g., 0 in 4.12.0; 3 in 4.10.3)
> +# #r: Numbers-only release (e.g., 4.12.0, 4.10.3) # PROBABLY NOT NEEDED
> +# rc: -rcN
> +#
> +# s: branch name (e.g., master, stable-4.12, stable-4.10)
> +# t: Tag from a given release (e.g,. 4.12.0-rc5, RELEASE-4.10.3)
> +# isrc: Boolean indicating whether the version is an rc (e.g., true for 4.12.0-rc5, false for 4.10.3)
> +
> +
> +
> +function xen-make-prefix-config() {
> +    $arg_parse
> +
> +    # TODO: Ping drall.uk.xensource.com to see if we can reach it?
> +    
> +    default cache_prefix "git://drall.uk.xensource.com:9419/" ; $default_post
> +
> +    perl -ne "if(/^([A-Z_]+_URL) \?= (git.*)/) { print \"\$1 ?= ${cache_prefix}\$2\n\"; }" Config.mk >> .config || fail "Generating .config"
> +    cat .config
> +}
> +
> +function set-tdir() {
> +    if [[ -z "$tdir" || ! -e "$tdir" ]] ; then
> +	info "$tdir doesn't exist, using /tmp"
> +	tdir="/tmp"
> +    fi
> +}
> +
> +# Take `v` and generate the appropriate metavariables variables.
> +function parse-version() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    if [[ -n "$x" && -n "$p" ]] ; then
> +	echo "Version already parsed"
> +	return
> +    fi
> +
> +    if [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9])(-rc[0-9]) ]] ; then
> +	x=${BASH_REMATCH[1]}
> +	p=${BASH_REMATCH[2]}
> +	rc=${BASH_REMATCH[3]}
> +	isrc=true
> +    elif [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9]) ]] ; then
> +	x=${BASH_REMATCH[1]}
> +	p=${BASH_REMATCH[2]}
> +	isrc=false
> +    else
> +	fail "Bad version"
> +    fi
> +
> +    if $isrc ; then
> +	t=$v
> +    else
> +	t=RELEASE-$v
> +    fi
> +}
> +
> +function check() {
> +    # TODO: Automate some of these
> +    info "Please perform manually: All XSAs  have been applied"
> +    info "Please perform manually: Check http://logs.test-lab.xenproject.org/osstest/results/all-branch-statuses.txt"
> +    info "Please perform manually: Check version in README"
> +    info "Please perform manually: Check version in SUPPORT.md"
> +    info "Please perform manually: Tags for appropriate *_REVISION's in Config.mk"
> +    info "Please perform manually: xen/Makefile:XEN_EXTRAVERSION set to 0"
> +    info "Please perform manually: tools/Rules.mk: debug ?= n"
> +    info "Please perform manually: xen/Kconfig.debug:config DEBUG should default to `n`"
> +}
> +
> +# Usage:
> +#   tag v=[version you want to release] [c=commithash]
> +# eg.
> +#   tag v=4.12.0-rc6
> +# Other arguments:
> +#  key:  Name of key to sign the commit with
> +#  tdir: Name of top-level tarball directory
> +function tag() {
> +    $arg_parse
> +
> +    default key "23E3222C145F4475FA8060A783FE14C957E82BD9"; $default_post
> +
> +    $requireargs v
> +
> +    set-tdir
> +
> +    $requireargs tdir
> +
> +    parse-version
> +
> +    $requireargs t
> +
> +    git fetch origin
> +
> +    if [[ -n "$c" ]] ; then
> +	info "Checking out commit $c"
> +	git checkout $c || fail
> +    else
> +	local q
> +	git checkout stable-$x || fail "Couldn't check out stable branch"
> +	git merge || fail "Merge"
> +	git log -n 10
> +	read -p "Enter to continue, anything else to quit: " q
> +	[[ -z "$q" ]] || return
> +    fi
> +
> +    # FIXME: Add checks:
> +    # - Make sure Config.mk has tags, not hashes
> +    # - sonames?
> +    # - Appropriate version numbers in SUPPORT.md, xen/Makefile, &c
> +
> +    echo git tag -u "$key" -s -m "Xen $v" $t ; sleep 1
> +    git tag -u "$key" -s -m "Xen $v" $t || fail "Creating signed tag"
> +
> +    info "Release tagged.  Now run release make-tarball v=$v"
> +}
> +
> +function push-tag() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    parse-version
> +
> +    git push origin $t || fail "Pushing tag"
> +    # FIXME: This is in the release checklist, but I'm not sure why
> +    # git push origin staging-$x || fail "Pushing tag commit"
> +
> +    info "Tag pushed.  Now run release tarball-cvs-checkin-and-post v=$v"
> +}
> +
> +function make-tarball-only()
> +{
> +    $arg_parse
> +
> +    $requireargs v tdir
> +
> +    parse-version
> +
> +    git fetch || fail "git fetch"
> +    
> +    git checkout $t || fail "Checking out tag $t"
> +
> +    git clean -ffdx
> +
> +    xen-make-prefix-config
> +
> +    ./configure || fail "Configuring"
> +    
> +    if $isrc ; then
> +	make src-tarball || fail "Making src-tarball"
> +    else
> +	make src-tarball-release || fail "Making src-tarball"
> +    fi
> +
> +    rm -rf $tdir/$v
> +
> +    mkdir -p $tdir/$v || fail "Couldn't make target directory"
> +
> +    cp dist/xen-$v.tar.gz $tdir/$v || fail "Couldn't copy tarball"
> +}
> +
> +function buildtest-tarball() {
> +    $arg_parse
> +
> +    default bdir "/tmp" ; $default_post
> +
> +    $requireargs tdir v
> +    
> +    cd $bdir || fail "cd $bdir"
> +
> +    rm -rf build-$v
> +    mkdir build-$v || fail "mkdir"
> +
> +    cd build-$v
> +
> +    tar xfz $tdir/$v/xen-$v.tar.gz || fail "Untar"
> +
> +    cd xen-$v || fail "cd"
> +
> +    xen-make-prefix-config
> +    info "Testing build (tail -f $bdir/build-$v/log.$v)..."
> +    (./configure && make -j4 && touch $tdir/$v/build-tested && echo OK) 2>&1 > ../log.$v
> +
> +    [[ -e $tdir/$v/build-tested ]] || fail "Build failed; log at $bdir/build-$v/log.$v"
> +}
> +
> +function sign-tarball() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    if [[ -z "$rtgz" ]] ; then
> +	set-tdir
> +	rtgz=$tdir/$v/xen-$v.tar.gz
> +    fi
> +
> +    default key "23E3222C145F4475FA8060A783FE14C957E82BD9" ; $default_post
> +
> +    if ! gpg --list-secret-keys | grep $key ; then
> +	info "Signature required; please run the following command with the public key available"
> +	info " gpg --detach-sign -u 'xen tree' $rtgz"
> +	exit 0
> +    fi
> +
> +    gpg --detach-sign -u $key $rtgz || fail "Signing $rtgz"
> +}
> +
> +function tarball-checksig() {
> +    gpg --verify $rtgz.sig || fail "Signature failed"
> +}
> +
> +function make-tarball() {
> +    local rdir
> +    local rtgz
> +    
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    set-tdir
> +
> +    $requireargs tdir
> +
> +    parse-version
> +    
> +    info "Using tag $t"
> +
> +    rdir=$tdir/$v
> +
> +    rtgz=$rdir/xen-$v.tar.gz
> +
> +    if [[ ! -e $rtgz ]] ; then
> +	info "$rtgz not present, generating"
> +	make-tarball-only
> +    fi
> +
> +    info "Tarball created"
> +
> +    if [[ ! -e $rdir/build-tested ]] ; then
> +	buildtest-tarball
> +    fi
> +
> +    info "Build tested"
> +
> +    if [[ ! -e $rtgz.sig ]] ; then
> +	sign-tarball
> +    else
> +	tarball-checksig
> +    fi
> +
> +    info "Tarball made, signed, and build-tested.  Now run release push-tag v=$v"
> +}
> +
> +function tarball-cvs-checkin-and-post() {
> +    $arg_parse
> +
> +    $requireargs v
> +
> +    # TODO: This tree probably wants to be put somewhere on
> +    # mail.xenproject.org
> +    
> +    default cvsdir "/build/hg/push/xen.org/" ; $default_post
> +
> +    if [[ ! -e $cvsdir ]] ; then
> +	fail "$cvsdir does not exist"
> +    fi
> +
> +    if [[ -z "$rtgz" ]] ; then
> +	set-tdir
> +	rtgz=$tdir/$v/xen-$v.tar.gz
> +    fi
> +
> +    cd $cvsdir || fail "cd"
> +
> +    mkdir -p oss-xen/release/$v || fail "Creating directory in CVS"
> +
> +    cvs add -kb oss-xen/release/$v/ || fail "cvs add release directory"
> +
> +    cd oss-xen/release/$v || fail "cd"
> +
> +    cp $tdir/$v/xen-$v.tar.gz . || fail "Copying tarball"
> +    cp $tdir/$v/xen-$v.tar.gz.sig . || fail "Copying sig"
> +    
> +    cvs add -kb xen-$v.tar.gz || fail "cvs add tarball"
> +    cvs add -kb xen-$v.tar.gz.sig || fail "cvs add sig"
> +    
> +    cd ../../..
> +
> +    cvs ci -m $v || fail "cvs checkin"
> +
> +    ssh mail.xenproject.org "cd /data/downloads.xenproject.org/xen.org && cvs -q up -d" || fail "Deploying tarball"
> +
> +    info "Tarball Uploaded.  Xen version $v released."
> +}
> +
> +function help() {
> +    cat <<EOF
> +General workflow:
> +
> +* Do a number of pre-release sanity checks
> +  release check v=4.12.0-rc5
> +
> +* Tag and sign a Xen commit
> +  release tag v=4.12.0-rc5
> +   or
> +  release tag v=4.12.0-rc5 c=07c181c
> +
> +* Create, test, and sign a release tarball
> +  release make-tarball v=4.12.0-rc5
> +
> +* Push tags
> +  release push-tag v=4.12.0-rc5
> +
> +* Publish tarball
> +  release tarball-cvs-checkin-and-post v=4.12.0-rc5
> +EOF
> +}
> +
> +###
> +# The actual command-line
> +###
> +cmdline "$@"
>
Ian Jackson Aug. 5, 2019, 10:57 a.m. UTC | #2 
Hi.  Thanks for looking into this.  Sorry about the delay to the
review.  I found it, unsent, while recovering a crashed mailreader
session...

George Dunlap writes ("[PATCH RFC 1/2] scripts: Add script to do the repetitive bits of the release process"):
> With this script, once the main checks are out of the way, doing a
> release (either an RC or the final release) should mostly be a matter
> of executing a sequence of 4 commands given by the `help` function in
> this script.

Script is missing set -e.  I'm generally not happy with the || fail
pattern, except for improving error messages.  It makes it far too
easy to accidentally miss an error check.  Your script has at least a
few missing error checks and I don't want to read it and try to spot
them all.  Can you change this ?

> +###
> +# George's bash library core
> +###

You've cloned (and presumably not hacked) some personal library of
your own ?  What's wrong with getopt(1) eg
   eval "$(getopt -s bash ... "$@")"
See /usr/share/doc/util-linux/examples/getopt-parse.bash.

Although, it's not clear to me that a full-blown option parser is
a better approach than
  var1=DEFAULT1
  var2=DEFAULT2
  for x in "$@"; do eval "$x"; done
for this script, since it will have few users.

> +function cmdline()

I think the keyword `function' here is unidiomatic shell.  (I think the
brace place is too but I don't want to quibble about style.)

+    info Running "${args[0]}"
+    "${args[0]}" "${args[@]:1}" || exit 1

You should probably namespace these with an appropriate prefix, rather
than exposing every internal function as a toplevel verb.

> +function xen-make-prefix-config() {
> +    $arg_parse
> +
> +    # TODO: Ping drall.uk.xensource.com to see if we can reach it?
> +    
> +    default cache_prefix "git://drall.uk.xensource.com:9419/" ; $default_post
> +
> +    perl -ne "if(/^([A-Z_]+_URL) \?= (git.*)/) { print \"\$1 ?= ${cache_prefix}\$2\n\"; }" Config.mk >> .config || fail "Generating .config"
> +    cat .config
> +}

Maybe we can expect the caller to have this in their global git
config.  I do this:

mariner:~> git-config -l | grep instead
url.git://git-cache.xs.citrite.net:9419/git://.insteadof=git://
url.git://git-cache.xs.citrite.net:9419/.insteadof=git://git-cache.xs.citrite.net:9419/
url.git://drall:9419/http://.insteadof=git://drall:9419/http://
url.git://drall:9419/https://.insteadof=git://drall:9419/https://
url.git://drall:9419/git://.insteadof=git://drall:9419/git://
mariner:~>

> +function set-tdir() {
> +    if [[ -z "$tdir" || ! -e "$tdir" ]] ; then
> +	info "$tdir doesn't exist, using /tmp"
> +	tdir="/tmp"
> +    fi

It is not OK to use /tmp/$v, because of tmpfile races.  If you don't
want to use somewhere in ~ then you need to mess with mktemp.

> +function tag() {
> +    $arg_parse

Overall I am surprised at how much code there is in this script.  It
seems much longer than the current runes in the release checklist.

> +    if [[ -n "$c" ]] ; then
> +	info "Checking out commit $c"
> +	git checkout $c || fail
> +    else
> +	local q
> +	git checkout stable-$x || fail "Couldn't check out stable branch"
> +	git merge || fail "Merge"

Surely we rarely want to do this, and never automatically.

> +	git log -n 10
> +	read -p "Enter to continue, anything else to quit: " q

Better to ask for a "y".  read might return "" due to eof.

> +    cvs ci -m $v || fail "cvs checkin"
> +
> +    ssh mail.xenproject.org "cd /data/downloads.xenproject.org/xen.org && cvs -q up -d" || fail "Deploying tarball"

Should surely read   ssh downloads.xenproject.org   and then it should
be a variable.  Also the scriptlet could be formatted across
multiple lines (and start with set -e, rather than using &&).

Thanks,
Ian.
George Dunlap Aug. 6, 2019, 1:37 p.m. UTC | #3 
On 8/5/19 11:57 AM, Ian Jackson wrote:
> Hi.  Thanks for looking into this.  Sorry about the delay to the
> review.  I found it, unsent, while recovering a crashed mailreader
> session...

Thanks for the review -- I was going to come back and ping this at some
point; it would be nice to have it in tree before we actually need it
again. :-)

So just in general: I'm willing to make fixes and small tweaks, but I'm
not going to invest a lot of time rewriting this script.  It was useful
for me; if you don't want it in tree, then I'll just use it myself when
I find it useful.

> George Dunlap writes ("[PATCH RFC 1/2] scripts: Add script to do the repetitive bits of the release process"):
>> With this script, once the main checks are out of the way, doing a
>> release (either an RC or the final release) should mostly be a matter
>> of executing a sequence of 4 commands given by the `help` function in
>> this script.
> 
> Script is missing set -e.  I'm generally not happy with the || fail
> pattern, except for improving error messages.  It makes it far too
> easy to accidentally miss an error check.  Your script has at least a
> few missing error checks and I don't want to read it and try to spot
> them all.  Can you change this ?
I've never run with `set -e` before, but let me give it a try and see
how it goes.

>> +###
>> +# George's bash library core
>> +###
> 
> You've cloned (and presumably not hacked) some personal library of
> your own ?

I C&P my personal library, yes.  (Not sure what "hacked" means in this
context.)  I've got a reasonably extensive set of personal scripts that
use this calling convention as their core.  It's also the basis for a
bunch of scripting I did around the CentOS package maintenanace:

https://github.com/CentOS-virt7/xen/tree/xen-412/lib

(I've C&P'd here a subset of 'core.sh' in that directory.)

>  What's wrong with getopt(1) eg
>    eval "$(getopt -s bash ... "$@")"

* It's ugly and hard to read naturally

* You have to think about short names for every option you want; if you
want more than 26, then you start to run out of meaningful letters.

* You have to encode everything in a single line of runes.  "$arg_parse"
sets local variables for all your arguments; "$requireargs" is a more
natual way of saying what's required, and "$default" allows line-by-line
specification of defaults if something isn't specified.

* You don't get "inheritance".  One of the explicit reasons I developed
this type of framework was so that I could do something like:

```
tgt=c6_01;

vm-start

tgt-ssh "<some command>"

vm-shutdown
```

(In case it isn't clear, `vm-start`, `tgt-ssh`, and `vm-shutdown` all
take `tgt` as an argument.)

Obviously this makes some things easier by making other things more
dangerous; but bash already has inherited variables anyway.  If I wanted
real scoping I'd switch to a locally-scoped programming language.

In any case, in line with what I said above -- I'm used to writing bash
scripts in this way; I find it useful.  And although the core
"arg_parse" machinery is somewhat ugly, it's been tested and refined
over several years.  I'm not going to spend a lot of time rewriting this
script to do something else.

>> +function cmdline()
> 
> I think the keyword `function' here is unidiomatic shell.  (I think the
> brace place is too but I don't want to quibble about style.)

What would you prefer instead?

> 
> +    info Running "${args[0]}"
> +    "${args[0]}" "${args[@]:1}" || exit 1
> 
> You should probably namespace these with an appropriate prefix, rather
> than exposing every internal function as a toplevel verb.

You mean namespace the internal "library" functions like `fail`?  Or
helper functions like `tag`?

Either way, I don't see a big issue from allowing them to be called from
the command-line: sometimes it's helpful (if only for unit testing); and
although calling `release fail "blah"` seems a bit pointless, it's not
harmful, and I'd prefer to keep names of the "library" functions short.

>> +function xen-make-prefix-config() {
>> +    $arg_parse
>> +
>> +    # TODO: Ping drall.uk.xensource.com to see if we can reach it?
>> +    
>> +    default cache_prefix "git://drall.uk.xensource.com:9419/" ; $default_post
>> +
>> +    perl -ne "if(/^([A-Z_]+_URL) \?= (git.*)/) { print \"\$1 ?= ${cache_prefix}\$2\n\"; }" Config.mk >> .config || fail "Generating .config"
>> +    cat .config
>> +}
> 
> Maybe we can expect the caller to have this in their global git
> config.  I do this:
> 
> mariner:~> git-config -l | grep instead
> url.git://git-cache.xs.citrite.net:9419/git://.insteadof=git://
> url.git://git-cache.xs.citrite.net:9419/.insteadof=git://git-cache.xs.citrite.net:9419/
> url.git://drall:9419/http://.insteadof=git://drall:9419/http://
> url.git://drall:9419/https://.insteadof=git://drall:9419/https://
> url.git://drall:9419/git://.insteadof=git://drall:9419/git://
> mariner:~>

It looks like only the first one of those does anything?  Or am I
misunderstanding the syntax?

At any rate, yes, that's probably a cleaner solution; we can add that to
the instructions for the script.  (I didn't know about `insteadof`.)

>> +function set-tdir() {
>> +    if [[ -z "$tdir" || ! -e "$tdir" ]] ; then
>> +	info "$tdir doesn't exist, using /tmp"
>> +	tdir="/tmp"
>> +    fi
> 
> It is not OK to use /tmp/$v, because of tmpfile races.  If you don't
> want to use somewhere in ~ then you need to mess with mktemp.
> 
>> +function tag() {
>> +    $arg_parse
> 
> Overall I am surprised at how much code there is in this script.  It
> seems much longer than the current runes in the release checklist.

Some of it is boilerplate to be able to make functions; a lot of it is
encoding error and exception handling which is implicit in a checklist
(because expert humans know what an error looks like and how to fix
things up).

>> +    if [[ -n "$c" ]] ; then
>> +	info "Checking out commit $c"
>> +	git checkout $c || fail
>> +    else
>> +	local q
>> +	git checkout stable-$x || fail "Couldn't check out stable branch"
>> +	git merge || fail "Merge"
> 
> Surely we rarely want to do this, and never automatically.

Oh, this should have an `--ff-only`.  (I've got --ff-only enabled by
default in my .gitconfig).  So for me this will just update to the
newest commit on the origin/stable-$x branch (and fail if an actual
merge commit would be needed).

If we don't want to auto-ff, then we should check to see how many
commits we are away from upstream and fail out if that number is 0.

>> +	git log -n 10
>> +	read -p "Enter to continue, anything else to quit: " q
> 
> Better to ask for a "y".  read might return "" due to eof.

Sure.

>> +    cvs ci -m $v || fail "cvs checkin"
>> +
>> +    ssh mail.xenproject.org "cd /data/downloads.xenproject.org/xen.org && cvs -q up -d" || fail "Deploying tarball"
> 
> Should surely read   ssh downloads.xenproject.org   and then it should
> be a variable.  Also the scriptlet could be formatted across
> multiple lines (and start with set -e, rather than using &&).

Can you 'set -e' in an ssh command like this?

In any case -- before I invest a lot of time cleaning this up further,
are you willing to take it with my quirky calling conventions rather
than getopt?  If not I'll probably just keep this locally.

 -George
diff mbox series

Patch

diff --git a/scripts/release b/scripts/release
new file mode 100755
index 0000000000..0442cd4ef9
--- /dev/null
+++ b/scripts/release
@@ -0,0 +1,450 @@ 
+#!/bin/bash
+
+###
+# George's bash library core
+###
+
+# arg-parse debug
+_apd=false
+
+arg_parse_cmd=\
+"local -a args;
+local _a;
+local _vn;
+local _m;
+local CLVL=\$((\$CLVL+1))
+
+_m=true;
+
+for _a in \"\$@\" ; do
+    $_apd && echo \"Evaluating \${_a} [[ \"\${_a/=}\" = \"\${_a}\" ]]\";
+    if \$_m && [[ \"\${_a/=}\" != \"\${_a}\" ]] ; then
+        $_apd && echo Parameter;
+        _vn=\${_a%%=*};
+        eval \"local \$_vn\";
+        eval \"\$_a\";
+    elif \$_m && [[ \"\${_a}\" == \"--\" ]] ; then
+        $_apd && echo Separator;
+        _m=false;
+    else
+        $_apd && echo Argument;
+        _m=false;
+        args+=(\"\$_a\");
+    fi;
+done"
+
+arg_parse="eval $arg_parse_cmd"
+
+# Pass in either the current function name, or the name of the script
+requireargs="eval _func=\"\$FUNCNAME\" ; eval [[ -n \\\"\$_func\\\" ]] || _func=\$0 ; eval _require-args \$_func"
+
+function _require-args()
+{
+    local _arg
+    local _args
+
+    _args=($@)
+
+    for _arg in ${_args[@]:1} ; do
+	eval "[[ -n \"\${$_arg}\" ]] || fail \"${_args[0]}: Missing $_arg\""
+    done
+}
+
+function default()
+{
+    # l0: eval i="5"
+    # l1: default_post="eval $1=\"$2\""
+    # l3: eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval \$1=\\\"$2\\\"\" ; fi"
+    eval "if [[ -z \"\$$1\" ]] ; then default_post=\"eval local \$1=\\\"$2\\\"\" ; else unset default_post ; fi"
+}
+
+function fail()
+{
+   echo FATAL $@
+   [[ -n "$fail_cleanup" ]] && $fail_cleanup
+   exit 1
+}
+
+function info()
+{
+   echo INFO $CLVL $@ 1>&2
+}
+
+function error()
+{
+   echo ERROR $@ 1>&2
+}
+
+function status()
+{
+   echo STATUS $CLVL $@ 1>&2
+   return 0
+}
+
+function report-result()
+{
+    if [[ -n "$var" ]] ; then
+	eval "${var}=\"$1\""
+    else
+	if [[ -n "$1" ]] ; then
+	    echo "$1"
+	else
+	    echo "(empty)"
+	fi
+    fi
+}
+
+function cmdline()
+{
+    local cmd;
+
+    if [[ "$#" -eq "0" ]] ; then
+	help
+	exit 1
+    fi
+
+    $arg_parse
+    info Running "${args[0]}"
+    "${args[0]}" "${args[@]:1}" || exit 1
+
+    if ! [[ -z "$RET" ]] ; then
+	echo $RET
+    fi
+}
+
+###
+# release-specific code
+###
+
+# Global / meta variables:
+#
+# tdir: "root" directory to do tarball work.
+# rdir: Directory where tarball & sig will be put (==$tdir/$v)
+# rtgz: Base filename for tarball ($rdir/xen-$v.tar.gz)
+#
+# v: Full release version (e.g., 4.12.0-rc5, 4.10.3)
+# x: Major+minor xen release version (e.g., 4.12, 4.10)
+# p: point release (e.g., 0 in 4.12.0; 3 in 4.10.3)
+# #r: Numbers-only release (e.g., 4.12.0, 4.10.3) # PROBABLY NOT NEEDED
+# rc: -rcN
+#
+# s: branch name (e.g., master, stable-4.12, stable-4.10)
+# t: Tag from a given release (e.g,. 4.12.0-rc5, RELEASE-4.10.3)
+# isrc: Boolean indicating whether the version is an rc (e.g., true for 4.12.0-rc5, false for 4.10.3)
+
+
+
+function xen-make-prefix-config() {
+    $arg_parse
+
+    # TODO: Ping drall.uk.xensource.com to see if we can reach it?
+    
+    default cache_prefix "git://drall.uk.xensource.com:9419/" ; $default_post
+
+    perl -ne "if(/^([A-Z_]+_URL) \?= (git.*)/) { print \"\$1 ?= ${cache_prefix}\$2\n\"; }" Config.mk >> .config || fail "Generating .config"
+    cat .config
+}
+
+function set-tdir() {
+    if [[ -z "$tdir" || ! -e "$tdir" ]] ; then
+	info "$tdir doesn't exist, using /tmp"
+	tdir="/tmp"
+    fi
+}
+
+# Take `v` and generate the appropriate metavariables variables.
+function parse-version() {
+    $arg_parse
+
+    $requireargs v
+
+    if [[ -n "$x" && -n "$p" ]] ; then
+	echo "Version already parsed"
+	return
+    fi
+
+    if [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9])(-rc[0-9]) ]] ; then
+	x=${BASH_REMATCH[1]}
+	p=${BASH_REMATCH[2]}
+	rc=${BASH_REMATCH[3]}
+	isrc=true
+    elif [[ $v =~ ([0-9]+\.[0-9]+)\.([0-9]) ]] ; then
+	x=${BASH_REMATCH[1]}
+	p=${BASH_REMATCH[2]}
+	isrc=false
+    else
+	fail "Bad version"
+    fi
+
+    if $isrc ; then
+	t=$v
+    else
+	t=RELEASE-$v
+    fi
+}
+
+function check() {
+    # TODO: Automate some of these
+    info "Please perform manually: All XSAs  have been applied"
+    info "Please perform manually: Check http://logs.test-lab.xenproject.org/osstest/results/all-branch-statuses.txt"
+    info "Please perform manually: Check version in README"
+    info "Please perform manually: Check version in SUPPORT.md"
+    info "Please perform manually: Tags for appropriate *_REVISION's in Config.mk"
+    info "Please perform manually: xen/Makefile:XEN_EXTRAVERSION set to 0"
+    info "Please perform manually: tools/Rules.mk: debug ?= n"
+    info "Please perform manually: xen/Kconfig.debug:config DEBUG should default to `n`"
+}
+
+# Usage:
+#   tag v=[version you want to release] [c=commithash]
+# eg.
+#   tag v=4.12.0-rc6
+# Other arguments:
+#  key:  Name of key to sign the commit with
+#  tdir: Name of top-level tarball directory
+function tag() {
+    $arg_parse
+
+    default key "23E3222C145F4475FA8060A783FE14C957E82BD9"; $default_post
+
+    $requireargs v
+
+    set-tdir
+
+    $requireargs tdir
+
+    parse-version
+
+    $requireargs t
+
+    git fetch origin
+
+    if [[ -n "$c" ]] ; then
+	info "Checking out commit $c"
+	git checkout $c || fail
+    else
+	local q
+	git checkout stable-$x || fail "Couldn't check out stable branch"
+	git merge || fail "Merge"
+	git log -n 10
+	read -p "Enter to continue, anything else to quit: " q
+	[[ -z "$q" ]] || return
+    fi
+
+    # FIXME: Add checks:
+    # - Make sure Config.mk has tags, not hashes
+    # - sonames?
+    # - Appropriate version numbers in SUPPORT.md, xen/Makefile, &c
+
+    echo git tag -u "$key" -s -m "Xen $v" $t ; sleep 1
+    git tag -u "$key" -s -m "Xen $v" $t || fail "Creating signed tag"
+
+    info "Release tagged.  Now run release make-tarball v=$v"
+}
+
+function push-tag() {
+    $arg_parse
+
+    $requireargs v
+
+    parse-version
+
+    git push origin $t || fail "Pushing tag"
+    # FIXME: This is in the release checklist, but I'm not sure why
+    # git push origin staging-$x || fail "Pushing tag commit"
+
+    info "Tag pushed.  Now run release tarball-cvs-checkin-and-post v=$v"
+}
+
+function make-tarball-only()
+{
+    $arg_parse
+
+    $requireargs v tdir
+
+    parse-version
+
+    git fetch || fail "git fetch"
+    
+    git checkout $t || fail "Checking out tag $t"
+
+    git clean -ffdx
+
+    xen-make-prefix-config
+
+    ./configure || fail "Configuring"
+    
+    if $isrc ; then
+	make src-tarball || fail "Making src-tarball"
+    else
+	make src-tarball-release || fail "Making src-tarball"
+    fi
+
+    rm -rf $tdir/$v
+
+    mkdir -p $tdir/$v || fail "Couldn't make target directory"
+
+    cp dist/xen-$v.tar.gz $tdir/$v || fail "Couldn't copy tarball"
+}
+
+function buildtest-tarball() {
+    $arg_parse
+
+    default bdir "/tmp" ; $default_post
+
+    $requireargs tdir v
+    
+    cd $bdir || fail "cd $bdir"
+
+    rm -rf build-$v
+    mkdir build-$v || fail "mkdir"
+
+    cd build-$v
+
+    tar xfz $tdir/$v/xen-$v.tar.gz || fail "Untar"
+
+    cd xen-$v || fail "cd"
+
+    xen-make-prefix-config
+    info "Testing build (tail -f $bdir/build-$v/log.$v)..."
+    (./configure && make -j4 && touch $tdir/$v/build-tested && echo OK) 2>&1 > ../log.$v
+
+    [[ -e $tdir/$v/build-tested ]] || fail "Build failed; log at $bdir/build-$v/log.$v"
+}
+
+function sign-tarball() {
+    $arg_parse
+
+    $requireargs v
+
+    if [[ -z "$rtgz" ]] ; then
+	set-tdir
+	rtgz=$tdir/$v/xen-$v.tar.gz
+    fi
+
+    default key "23E3222C145F4475FA8060A783FE14C957E82BD9" ; $default_post
+
+    if ! gpg --list-secret-keys | grep $key ; then
+	info "Signature required; please run the following command with the public key available"
+	info " gpg --detach-sign -u 'xen tree' $rtgz"
+	exit 0
+    fi
+
+    gpg --detach-sign -u $key $rtgz || fail "Signing $rtgz"
+}
+
+function tarball-checksig() {
+    gpg --verify $rtgz.sig || fail "Signature failed"
+}
+
+function make-tarball() {
+    local rdir
+    local rtgz
+    
+    $arg_parse
+
+    $requireargs v
+
+    set-tdir
+
+    $requireargs tdir
+
+    parse-version
+    
+    info "Using tag $t"
+
+    rdir=$tdir/$v
+
+    rtgz=$rdir/xen-$v.tar.gz
+
+    if [[ ! -e $rtgz ]] ; then
+	info "$rtgz not present, generating"
+	make-tarball-only
+    fi
+
+    info "Tarball created"
+
+    if [[ ! -e $rdir/build-tested ]] ; then
+	buildtest-tarball
+    fi
+
+    info "Build tested"
+
+    if [[ ! -e $rtgz.sig ]] ; then
+	sign-tarball
+    else
+	tarball-checksig
+    fi
+
+    info "Tarball made, signed, and build-tested.  Now run release push-tag v=$v"
+}
+
+function tarball-cvs-checkin-and-post() {
+    $arg_parse
+
+    $requireargs v
+
+    # TODO: This tree probably wants to be put somewhere on
+    # mail.xenproject.org
+    
+    default cvsdir "/build/hg/push/xen.org/" ; $default_post
+
+    if [[ ! -e $cvsdir ]] ; then
+	fail "$cvsdir does not exist"
+    fi
+
+    if [[ -z "$rtgz" ]] ; then
+	set-tdir
+	rtgz=$tdir/$v/xen-$v.tar.gz
+    fi
+
+    cd $cvsdir || fail "cd"
+
+    mkdir -p oss-xen/release/$v || fail "Creating directory in CVS"
+
+    cvs add -kb oss-xen/release/$v/ || fail "cvs add release directory"
+
+    cd oss-xen/release/$v || fail "cd"
+
+    cp $tdir/$v/xen-$v.tar.gz . || fail "Copying tarball"
+    cp $tdir/$v/xen-$v.tar.gz.sig . || fail "Copying sig"
+    
+    cvs add -kb xen-$v.tar.gz || fail "cvs add tarball"
+    cvs add -kb xen-$v.tar.gz.sig || fail "cvs add sig"
+    
+    cd ../../..
+
+    cvs ci -m $v || fail "cvs checkin"
+
+    ssh mail.xenproject.org "cd /data/downloads.xenproject.org/xen.org && cvs -q up -d" || fail "Deploying tarball"
+
+    info "Tarball Uploaded.  Xen version $v released."
+}
+
+function help() {
+    cat <<EOF
+General workflow:
+
+* Do a number of pre-release sanity checks
+  release check v=4.12.0-rc5
+
+* Tag and sign a Xen commit
+  release tag v=4.12.0-rc5
+   or
+  release tag v=4.12.0-rc5 c=07c181c
+
+* Create, test, and sign a release tarball
+  release make-tarball v=4.12.0-rc5
+
+* Push tags
+  release push-tag v=4.12.0-rc5
+
+* Publish tarball
+  release tarball-cvs-checkin-and-post v=4.12.0-rc5
+EOF
+}
+
+###
+# The actual command-line
+###
+cmdline "$@"