[v4,01/13] tests: Add utilities for docker testing

Commit Message

Fam Zheng March 17, 2016, 6:34 a.m. UTC

docker.py is added with a number of useful subcommands to manager docker
images and instances for QEMU docker testing. Subcommands are:

run: A wrapper of "docker run" (or "sudo -n docker run" if necessary),
which takes care of killing and removing the running container at
SIGINT.

clean: Tear down all the containers including inactive ones that are
started by docker_run.

build: Compare an image from given dockerfile and rebuild it if they're
different.

Signed-off-by: Fam Zheng <[e-mail]>
---
 tests/docker/docker.py | 191 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 191 insertions(+)
 create mode 100755 tests/docker/docker.py

Comments

Alex Bennée March 31, 2016, 9:47 a.m. UTC | #1

Fam Zheng <[e-mail]> writes:

> docker.py is added with a number of useful subcommands to manager docker
> images and instances for QEMU docker testing. Subcommands are:
>
> run: A wrapper of "docker run" (or "sudo -n docker run" if necessary),
> which takes care of killing and removing the running container at
> SIGINT.
>
> clean: Tear down all the containers including inactive ones that are
> started by docker_run.

I wonder if at some point we need a delete/remove sub-command to remove
all traces of the images? I dropped to plain docker and did:

  docker rmi -f qemu:fedora

while debugging.

Anyway:

Reviewed-by: Alex Bennée <[e-mail]>


>
> build: Compare an image from given dockerfile and rebuild it if they're
> different.
>
> Signed-off-by: Fam Zheng <[e-mail]>
> ---
>  tests/docker/docker.py | 191 +++++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 191 insertions(+)
>  create mode 100755 tests/docker/docker.py
>
> diff --git a/tests/docker/docker.py b/tests/docker/docker.py
> new file mode 100755
> index 0000000..fe73de7
> --- /dev/null
> +++ b/tests/docker/docker.py
> @@ -0,0 +1,191 @@
> +#!/usr/bin/env python2
> +#
> +# Docker controlling module
> +#
> +# Copyright (c) 2016 Red Hat Inc.
> +#
> +# Authors:
> +#  Fam Zheng <[e-mail]>
> +#
> +# This work is licensed under the terms of the GNU GPL, version 2
> +# or (at your option) any later version. See the COPYING file in
> +# the top-level directory.
> +
> +import os
> +import sys
> +import subprocess
> +import json
> +import hashlib
> +import atexit
> +import uuid
> +import argparse
> +import tempfile
> +
> +def _text_checksum(text):
> +    """Calculate a digest string unique to the text content"""
> +    return hashlib.sha1(text).hexdigest()
> +
> +def _guess_docker_command():
> +    """ Guess a working docker command or raise exception if not found"""
> +    commands = [["docker"], ["sudo", "-n", "docker"]]
> +    for cmd in commands:
> +        if subprocess.call(cmd + ["images"],
> +                           stdout=subprocess.PIPE,
> +                           stderr=subprocess.PIPE) == 0:
> +            return cmd
> +    commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
> +    raise Exception("Cannot find working docker command. Tried:\n%s" % \
> +                    commands_txt)
> +
> +class Docker(object):
> +    """ Running Docker commands """
> +    def __init__(self):
> +        self._command = _guess_docker_command()
> +        self._instances = []
> +        atexit.register(self._kill_instances)
> +
> +    def _do(self, cmd, quiet=True, **kwargs):
> +        if quiet:
> +            kwargs["stdout"] = subprocess.PIPE
> +        return subprocess.call(self._command + cmd, **kwargs)
> +
> +    def _do_kill_instances(self, only_known, only_active=True):
> +        cmd = ["ps", "-q"]
> +        if not only_active:
> +            cmd.append("-a")
> +        for i in self._output(cmd).split():
> +            resp = self._output(["inspect", i])
> +            labels = json.loads(resp)[0]["Config"]["Labels"]
> +            active = json.loads(resp)[0]["State"]["Running"]
> +            if not labels:
> +                continue
> +            instance_uuid = labels.get("com.qemu.instance.uuid", None)
> +            if not instance_uuid:
> +                continue
> +            if only_known and instance_uuid not in self._instances:
> +                continue
> +            print "Terminating", i
> +            if active:
> +                self._do(["kill", i])
> +            self._do(["rm", i])
> +
> +    def clean(self):
> +        self._do_kill_instances(False, False)
> +        return 0
> +
> +    def _kill_instances(self):
> +        return self._do_kill_instances(True)
> +
> +    def _output(self, cmd, **kwargs):
> +        return subprocess.check_output(self._command + cmd,
> +                                       stderr=subprocess.STDOUT,
> +                                       **kwargs)
> +
> +    def get_image_dockerfile_checksum(self, tag):
> +        resp = self._output(["inspect", tag])
> +        labels = json.loads(resp)[0]["Config"].get("Labels", {})
> +        return labels.get("com.qemu.dockerfile-checksum", "")
> +
> +    def build_image(self, tag, dockerfile, df_path, quiet=True, argv=None):
> +        if argv == None:
> +            argv = []
> +        tmp = dockerfile + "\n" + \
> +              "LABEL com.qemu.dockerfile-checksum=%s" % \
> +              _text_checksum(dockerfile)
> +        dirname = os.path.dirname(df_path)
> +        tmp_df = tempfile.NamedTemporaryFile(dir=dirname)
> +        tmp_df.write(tmp)
> +        tmp_df.flush()
> +        self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
> +                 [dirname],
> +                 quiet=quiet)
> +
> +    def image_matches_dockerfile(self, tag, dockerfile):
> +        try:
> +            checksum = self.get_image_dockerfile_checksum(tag)
> +        except Exception:
> +            return False
> +        return checksum == _text_checksum(dockerfile)
> +
> +    def run(self, cmd, keep, quiet):
> +        label = uuid.uuid1().hex
> +        if not keep:
> +            self._instances.append(label)
> +        ret = self._do(["run", "--label",
> +                        "com.qemu.instance.uuid=" + label] + cmd,
> +                       quiet=quiet)
> +        if not keep:
> +            self._instances.remove(label)
> +        return ret
> +
> +class SubCommand(object):
> +    """A SubCommand template base class"""
> +    name = None # Subcommand name
> +    def shared_args(self, parser):
> +        parser.add_argument("--quiet", action="store_true",
> +                            help="Run quietly unless an error occured")
> +
> +    def args(self, parser):
> +        """Setup argument parser"""
> +        pass
> +    def run(self, args, argv):
> +        """Run command.
> +        args: parsed argument by argument parser.
> +        argv: remaining arguments from sys.argv.
> +        """
> +        pass
> +
> +class RunCommand(SubCommand):
> +    """Invoke docker run and take care of cleaning up"""
> +    name = "run"
> +    def args(self, parser):
> +        parser.add_argument("--keep", action="store_true",
> +                            help="Don't remove image when command completes")
> +    def run(self, args, argv):
> +        return Docker().run(argv, args.keep, quiet=args.quiet)
> +
> +class BuildCommand(SubCommand):
> +    """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
> +    name = "build"
> +    def args(self, parser):
> +        parser.add_argument("tag",
> +                            help="Image Tag")
> +        parser.add_argument("dockerfile",
> +                            help="Dockerfile name")
> +
> +    def run(self, args, argv):
> +        dockerfile = open(args.dockerfile, "rb").read()
> +        tag = args.tag
> +
> +        dkr = Docker()
> +        if dkr.image_matches_dockerfile(tag, dockerfile):
> +            if not args.quiet:
> +                print "Image is up to date."
> +            return 0
> +
> +        dkr.build_image(tag, dockerfile, args.dockerfile,
> +                        quiet=args.quiet, argv=argv)
> +        return 0
> +
> +class CleanCommand(SubCommand):
> +    """Clean up docker instances"""
> +    name = "clean"
> +    def run(self, args, argv):
> +        Docker().clean()
> +        return 0
> +
> +def main():
> +    parser = argparse.ArgumentParser(description="A Docker helper",
> +            usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
> +    subparsers = parser.add_subparsers(title="subcommands", help=None)
> +    for cls in SubCommand.__subclasses__():
> +        cmd = cls()
> +        subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
> +        cmd.shared_args(subp)
> +        cmd.args(subp)
> +        subp.set_defaults(cmdobj=cmd)
> +    args, argv = parser.parse_known_args()
> +    return args.cmdobj.run(args, argv)
> +
> +if __name__ == "__main__":
> +    sys.exit(main())


--
Alex Bennée

Patch

diff --git a/tests/docker/docker.py b/tests/docker/docker.py
new file mode 100755
index 0000000..fe73de7
--- /dev/null
+++ b/tests/docker/docker.py
@@ -0,0 +1,191 @@ 
+#!/usr/bin/env python2
+#
+# Docker controlling module
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <[e-mail]>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+import os
+import sys
+import subprocess
+import json
+import hashlib
+import atexit
+import uuid
+import argparse
+import tempfile
+
+def _text_checksum(text):
+    """Calculate a digest string unique to the text content"""
+    return hashlib.sha1(text).hexdigest()
+
+def _guess_docker_command():
+    """ Guess a working docker command or raise exception if not found"""
+    commands = [["docker"], ["sudo", "-n", "docker"]]
+    for cmd in commands:
+        if subprocess.call(cmd + ["images"],
+                           stdout=subprocess.PIPE,
+                           stderr=subprocess.PIPE) == 0:
+            return cmd
+    commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
+    raise Exception("Cannot find working docker command. Tried:\n%s" % \
+                    commands_txt)
+
+class Docker(object):
+    """ Running Docker commands """
+    def __init__(self):
+        self._command = _guess_docker_command()
+        self._instances = []
+        atexit.register(self._kill_instances)
+
+    def _do(self, cmd, quiet=True, **kwargs):
+        if quiet:
+            kwargs["stdout"] = subprocess.PIPE
+        return subprocess.call(self._command + cmd, **kwargs)
+
+    def _do_kill_instances(self, only_known, only_active=True):
+        cmd = ["ps", "-q"]
+        if not only_active:
+            cmd.append("-a")
+        for i in self._output(cmd).split():
+            resp = self._output(["inspect", i])
+            labels = json.loads(resp)[0]["Config"]["Labels"]
+            active = json.loads(resp)[0]["State"]["Running"]
+            if not labels:
+                continue
+            instance_uuid = labels.get("com.qemu.instance.uuid", None)
+            if not instance_uuid:
+                continue
+            if only_known and instance_uuid not in self._instances:
+                continue
+            print "Terminating", i
+            if active:
+                self._do(["kill", i])
+            self._do(["rm", i])
+
+    def clean(self):
+        self._do_kill_instances(False, False)
+        return 0
+
+    def _kill_instances(self):
+        return self._do_kill_instances(True)
+
+    def _output(self, cmd, **kwargs):
+        return subprocess.check_output(self._command + cmd,
+                                       stderr=subprocess.STDOUT,
+                                       **kwargs)
+
+    def get_image_dockerfile_checksum(self, tag):
+        resp = self._output(["inspect", tag])
+        labels = json.loads(resp)[0]["Config"].get("Labels", {})
+        return labels.get("com.qemu.dockerfile-checksum", "")
+
+    def build_image(self, tag, dockerfile, df_path, quiet=True, argv=None):
+        if argv == None:
+            argv = []
+        tmp = dockerfile + "\n" + \
+              "LABEL com.qemu.dockerfile-checksum=%s" % \
+              _text_checksum(dockerfile)
+        dirname = os.path.dirname(df_path)
+        tmp_df = tempfile.NamedTemporaryFile(dir=dirname)
+        tmp_df.write(tmp)
+        tmp_df.flush()
+        self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
+                 [dirname],
+                 quiet=quiet)
+
+    def image_matches_dockerfile(self, tag, dockerfile):
+        try:
+            checksum = self.get_image_dockerfile_checksum(tag)
+        except Exception:
+            return False
+        return checksum == _text_checksum(dockerfile)
+
+    def run(self, cmd, keep, quiet):
+        label = uuid.uuid1().hex
+        if not keep:
+            self._instances.append(label)
+        ret = self._do(["run", "--label",
+                        "com.qemu.instance.uuid=" + label] + cmd,
+                       quiet=quiet)
+        if not keep:
+            self._instances.remove(label)
+        return ret
+
+class SubCommand(object):
+    """A SubCommand template base class"""
+    name = None # Subcommand name
+    def shared_args(self, parser):
+        parser.add_argument("--quiet", action="store_true",
+                            help="Run quietly unless an error occured")
+
+    def args(self, parser):
+        """Setup argument parser"""
+        pass
+    def run(self, args, argv):
+        """Run command.
+        args: parsed argument by argument parser.
+        argv: remaining arguments from sys.argv.
+        """
+        pass
+
+class RunCommand(SubCommand):
+    """Invoke docker run and take care of cleaning up"""
+    name = "run"
+    def args(self, parser):
+        parser.add_argument("--keep", action="store_true",
+                            help="Don't remove image when command completes")
+    def run(self, args, argv):
+        return Docker().run(argv, args.keep, quiet=args.quiet)
+
+class BuildCommand(SubCommand):
+    """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
+    name = "build"
+    def args(self, parser):
+        parser.add_argument("tag",
+                            help="Image Tag")
+        parser.add_argument("dockerfile",
+                            help="Dockerfile name")
+
+    def run(self, args, argv):
+        dockerfile = open(args.dockerfile, "rb").read()
+        tag = args.tag
+
+        dkr = Docker()
+        if dkr.image_matches_dockerfile(tag, dockerfile):
+            if not args.quiet:
+                print "Image is up to date."
+            return 0
+
+        dkr.build_image(tag, dockerfile, args.dockerfile,
+                        quiet=args.quiet, argv=argv)
+        return 0
+
+class CleanCommand(SubCommand):
+    """Clean up docker instances"""
+    name = "clean"
+    def run(self, args, argv):
+        Docker().clean()
+        return 0
+
+def main():
+    parser = argparse.ArgumentParser(description="A Docker helper",
+            usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
+    subparsers = parser.add_subparsers(title="subcommands", help=None)
+    for cls in SubCommand.__subclasses__():
+        cmd = cls()
+        subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
+        cmd.shared_args(subp)
+        cmd.args(subp)
+        subp.set_defaults(cmdobj=cmd)
+    args, argv = parser.parse_known_args()
+    return args.cmdobj.run(args, argv)
+
+if __name__ == "__main__":
+    sys.exit(main())