diff mbox

[v3,02/13] Makefile: Rules for docker testing

Message ID 1457086720-30391-3-git-send-email-famz@redhat.com (mailing list archive)
State New, archived
Headers show

Commit Message

Fam Zheng March 4, 2016, 10:18 a.m. UTC 
This adds a group of make targets to run docker tests, all are available
in source tree without running ./configure.

The usage is shown by "make docker".

Besides the fixed ones, dynamic targets for building each image and
running each test in each image are generated automatically by make,
scanning $(SRC_PATH)/tests/docker/ files with specific patterns.

Alternative to manually list particular targets (docker-run-FOO@BAR)
set, you can control which tests/images to run by filtering variables,
TESTS= and IMAGES=, which are expressed in Makefile pattern syntax,
"foo% %bar ...". For example:

    $ make docker-run IMAGES="ubuntu fedora"

Unfortunately, it's impossible to propagate "-j $JOBS" into make in
containers, however since each combination is made a first class target
is the top Makefile, "make -j$N docker-run" still parallels the tests
coarsely.

Instead of providing a live version of the source tree to the docker
container we snapshot it with git-archive. This ensure the tree is in a
pristine state for whatever operations the container is going to run on
them.

Uncommitted changes known to files known by the git index will be
included in the snapshot if there are any.

Signed-off-by: Fam Zheng <famz@redhat.com>
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
---
 Makefile                      |   4 +-
 tests/docker/Makefile.include | 121 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 124 insertions(+), 1 deletion(-)
 create mode 100644 tests/docker/Makefile.include

Comments

Alex Bennée March 11, 2016, 3:11 p.m. UTC | #1 
Fam Zheng <famz@redhat.com> writes:

> This adds a group of make targets to run docker tests, all are available
> in source tree without running ./configure.
>
> The usage is shown by "make docker".
>
> Besides the fixed ones, dynamic targets for building each image and
> running each test in each image are generated automatically by make,
> scanning $(SRC_PATH)/tests/docker/ files with specific patterns.
>
> Alternative to manually list particular targets (docker-run-FOO@BAR)
> set, you can control which tests/images to run by filtering variables,
> TESTS= and IMAGES=, which are expressed in Makefile pattern syntax,
> "foo% %bar ...". For example:
>
>     $ make docker-run IMAGES="ubuntu fedora"

I thought I mentioned this last time, it needs fixing in the commit
message:

15:05 alex@zen/x86_64  [qemu.git/review/docker-v3] >make docker-run
IMAGES="ubuntu fedora"
make: *** No rule to make target `docker-run'. Stop.

>
> Unfortunately, it's impossible to propagate "-j $JOBS" into make in
> containers, however since each combination is made a first class target
> is the top Makefile, "make -j$N docker-run" still parallels the tests
> coarsely.
>
> Instead of providing a live version of the source tree to the docker
> container we snapshot it with git-archive. This ensure the tree is in a
> pristine state for whatever operations the container is going to run on
> them.
>
> Uncommitted changes known to files known by the git index will be
> included in the snapshot if there are any.
>
> Signed-off-by: Fam Zheng <famz@redhat.com>
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

fix the commit message and have a:

Reviewed-by: Alex Bennée <alex.bennee@linaro.org>


> ---
>  Makefile                      |   4 +-
>  tests/docker/Makefile.include | 121 ++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 124 insertions(+), 1 deletion(-)
>  create mode 100644 tests/docker/Makefile.include
>
> diff --git a/Makefile b/Makefile
> index 70e3ebc..b0dccc5 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -6,7 +6,7 @@ BUILD_DIR=$(CURDIR)
>  # Before including a proper config-host.mak, assume we are in the source tree
>  SRC_PATH=.
>
> -UNCHECKED_GOALS := %clean TAGS cscope ctags
> +UNCHECKED_GOALS := %clean TAGS cscope ctags docker docker-%
>
>  # All following code might depend on configuration variables
>  ifneq ($(wildcard config-host.mak),)
> @@ -651,3 +651,5 @@ endif
>  # Include automatically generated dependency files
>  # Dependencies in Makefile.objs files come from our recursive subdir rules
>  -include $(wildcard *.d tests/*.d)
> +
> +include $(SRC_PATH)/tests/docker/Makefile.include
> diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
> new file mode 100644
> index 0000000..4ca84c9
> --- /dev/null
> +++ b/tests/docker/Makefile.include
> @@ -0,0 +1,121 @@
> +# Makefile for Docker tests
> +
> +$(if $(quiet-command),,$(eval include $(SRC_PATH)/rules.mak))
> +
> +.PHONY: docker docker-test docker-clean docker-image docker-qemu-src
> +
> +DOCKER_SUFFIX := .docker
> +DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles
> +DOCKER_IMAGES := $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker)))
> +DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))
> +# Use a global constant ccache directory to speed up repetitive builds
> +DOCKER_CCACHE_DIR := /var/tmp/qemu-docker-ccache
> +
> +DOCKER_TESTS := $(notdir $(shell \
> +	find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f -executable))
> +
> +DOCKER_TOOLS := travis
> +
> +TESTS ?= %
> +IMAGES ?= %
> +SRC_COPY := $(shell mktemp -u /tmp/qemu-src.XXXXX)
> +
> +# Make archive from git repo $1 to tar.gz $2
> +make-archive-maybe = $(if $(wildcard $1/*), \
> +	$(call quiet-command, \
> +		(cd $1; if git diff-index --quiet HEAD -- &>/dev/null; then \
> +			git archive -1 HEAD --format=tar.gz -o $2; \
> +		else \
> +			git archive -1 $$(git stash create) --format=tar.gz -o $2; \
> +		fi), \
> +		"  ARCHIVE $(notdir $2)"))
> +
> +$(SRC_COPY):
> +	@mkdir -p $@
> +	$(call make-archive-maybe, $(SRC_PATH), $(SRC_COPY)/qemu.tgz)
> +	$(call make-archive-maybe, $(SRC_PATH)/dtc, $(SRC_COPY)/dtc.tgz)
> +	$(call make-archive-maybe, $(SRC_PATH)/pixman, $(SRC_COPY)/pixman.tgz)
> +	$(call quiet-command, cp "$(SRC_PATH)/tests/docker/run" "$(SRC_COPY)/run", \
> +		"  COPY RUNNER")
> +
> +docker-qemu-src: $(SRC_COPY)
> +
> +docker-image: ${DOCKER_TARGETS}
> +
> +# General rule for building docker images
> +docker-image-%: $(DOCKER_FILES_DIR)/%.docker
> +	$(call quiet-command,\
> +		$(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \
> +		$(if $V,-v,) $(if $(NOCACHE),--no-cache),\
> +		"  BUILD $*")
> +
> +# Expand all the pre-requistes for each docker image and test combination
> +$(foreach i,$(DOCKER_IMAGES), \
> +	$(foreach t,$(DOCKER_TESTS) $(DOCKER_TOOLS), \
> +		$(eval .PHONY: docker-$t@$i) \
> +		$(eval docker-$t@$i: docker-image-$i docker-run-$t@$i) \
> +	) \
> +	$(foreach t,$(DOCKER_TESTS), \
> +		$(eval docker-test: docker-$t@$i) \
> +	) \
> +)
> +
> +docker:
> +	@echo 'Build QEMU and run tests inside Docker containers'
> +	@echo
> +	@echo 'Available targets:'
> +	@echo
> +	@echo '    docker:              Print this help.'
> +	@echo '    docker-test:         Run all image/test combinations.'
> +	@echo '    docker-clean:        Kill and remove residual docker testing containers.'
> +	@echo '    docker-TEST@IMAGE:   Run "TEST" in container "IMAGE".'
> +	@echo '                         Note: "TEST" is one of the listed test name,'
> +	@echo '                         or a script name under $$QEMU_SRC/tests/docker/;'
> +	@echo '                         "IMAGE" is one of the listed container name."'
> +	@echo '    docker-image:        Build all images.'
> +	@echo '    docker-image-IMAGE:  Build image "IMG".'
> +	@echo
> +	@echo 'Available container images:'
> +	@echo '    $(DOCKER_IMAGES)'
> +	@echo
> +	@echo 'Available tests:'
> +	@echo '    $(DOCKER_TESTS)'
> +	@echo
> +	@echo 'Available tools:'
> +	@echo '    $(DOCKER_TOOLS)'
> +	@echo
> +	@echo 'Special variables:'
> +	@echo '    TARGET_LIST=a,b,c    Override target list in builds.'
> +	@echo '    IMAGES="a b c ..":   Filters which images to build or run.'
> +	@echo '    TESTS="x y z .."     Filters which tests to run (for docker-test).'
> +	@echo '    J=[0..9]*            Overrides the -jN parameter for make commands'
> +	@echo '                         (default is 1)'
> +	@echo '    DEBUG=1              Stop and drop to shell in the created container'
> +	@echo '                         before running the command.'
> +	@echo '    NOCACHE=1            Ignore cache when build images.'
> +
> +docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/')
> +docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/')
> +docker-run-%: docker-qemu-src
> +	@if test -z "$(IMAGE)" || test -z "$(CMD)"; \
> +		then echo "Invalid target"; exit 1; \
> +	fi
> +	$(if $(filter $(TESTS),$(CMD)),$(if $(filter $(IMAGES),$(IMAGE)), \
> +		$(call quiet-command,\
> +			$(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
> +				--privileged -t \
> +				$(if $(DEBUG),-i,--net=none) \
> +				-e TARGET_LIST=$(TARGET_LIST) \
> +				-e V=$V -e J=$J -e DEBUG=$(DEBUG)\
> +				-e CCACHE_DIR=/var/tmp/ccache \
> +				-v $$(realpath $(SRC_COPY)):/var/tmp/qemu:ro \
> +				-v $(DOCKER_CCACHE_DIR):/var/tmp/ccache \
> +				-w /var/tmp/qemu \
> +				qemu:$(IMAGE) \
> +				$(if $V,/bin/bash -x ,) \
> +				./run \
> +				$(CMD); \
> +			, "  RUN $(CMD) in $(IMAGE)")))
> +
> +docker-clean:
> +	$(call quiet-command, $(SRC_PATH)/tests/docker/docker.py clean)


--
Alex Bennée
Fam Zheng March 16, 2016, 3:37 a.m. UTC | #2 
On Fri, 03/11 15:11, Alex Bennée wrote:
> 
> Fam Zheng <famz@redhat.com> writes:
> 
> > This adds a group of make targets to run docker tests, all are available
> > in source tree without running ./configure.
> >
> > The usage is shown by "make docker".
> >
> > Besides the fixed ones, dynamic targets for building each image and
> > running each test in each image are generated automatically by make,
> > scanning $(SRC_PATH)/tests/docker/ files with specific patterns.
> >
> > Alternative to manually list particular targets (docker-run-FOO@BAR)
> > set, you can control which tests/images to run by filtering variables,
> > TESTS= and IMAGES=, which are expressed in Makefile pattern syntax,
> > "foo% %bar ...". For example:
> >
> >     $ make docker-run IMAGES="ubuntu fedora"
> 
> I thought I mentioned this last time, it needs fixing in the commit
> message:
> 
> 15:05 alex@zen/x86_64  [qemu.git/review/docker-v3] >make docker-run
> IMAGES="ubuntu fedora"
> make: *** No rule to make target `docker-run'. Stop.

My bad!

> 
> >
> > Unfortunately, it's impossible to propagate "-j $JOBS" into make in
> > containers, however since each combination is made a first class target
> > is the top Makefile, "make -j$N docker-run" still parallels the tests
> > coarsely.
> >
> > Instead of providing a live version of the source tree to the docker
> > container we snapshot it with git-archive. This ensure the tree is in a
> > pristine state for whatever operations the container is going to run on
> > them.
> >
> > Uncommitted changes known to files known by the git index will be
> > included in the snapshot if there are any.
> >
> > Signed-off-by: Fam Zheng <famz@redhat.com>
> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> fix the commit message and have a:
> 
> Reviewed-by: Alex Bennée <alex.bennee@linaro.org>

Thanks!

Fam
diff mbox

Patch

diff --git a/Makefile b/Makefile
index 70e3ebc..b0dccc5 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@  BUILD_DIR=$(CURDIR)
 # Before including a proper config-host.mak, assume we are in the source tree
 SRC_PATH=.
 
-UNCHECKED_GOALS := %clean TAGS cscope ctags
+UNCHECKED_GOALS := %clean TAGS cscope ctags docker docker-%
 
 # All following code might depend on configuration variables
 ifneq ($(wildcard config-host.mak),)
@@ -651,3 +651,5 @@  endif
 # Include automatically generated dependency files
 # Dependencies in Makefile.objs files come from our recursive subdir rules
 -include $(wildcard *.d tests/*.d)
+
+include $(SRC_PATH)/tests/docker/Makefile.include
diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
new file mode 100644
index 0000000..4ca84c9
--- /dev/null
+++ b/tests/docker/Makefile.include
@@ -0,0 +1,121 @@ 
+# Makefile for Docker tests
+
+$(if $(quiet-command),,$(eval include $(SRC_PATH)/rules.mak))
+
+.PHONY: docker docker-test docker-clean docker-image docker-qemu-src
+
+DOCKER_SUFFIX := .docker
+DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles
+DOCKER_IMAGES := $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker)))
+DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))
+# Use a global constant ccache directory to speed up repetitive builds
+DOCKER_CCACHE_DIR := /var/tmp/qemu-docker-ccache
+
+DOCKER_TESTS := $(notdir $(shell \
+	find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f -executable))
+
+DOCKER_TOOLS := travis
+
+TESTS ?= %
+IMAGES ?= %
+SRC_COPY := $(shell mktemp -u /tmp/qemu-src.XXXXX)
+
+# Make archive from git repo $1 to tar.gz $2
+make-archive-maybe = $(if $(wildcard $1/*), \
+	$(call quiet-command, \
+		(cd $1; if git diff-index --quiet HEAD -- &>/dev/null; then \
+			git archive -1 HEAD --format=tar.gz -o $2; \
+		else \
+			git archive -1 $$(git stash create) --format=tar.gz -o $2; \
+		fi), \
+		"  ARCHIVE $(notdir $2)"))
+
+$(SRC_COPY):
+	@mkdir -p $@
+	$(call make-archive-maybe, $(SRC_PATH), $(SRC_COPY)/qemu.tgz)
+	$(call make-archive-maybe, $(SRC_PATH)/dtc, $(SRC_COPY)/dtc.tgz)
+	$(call make-archive-maybe, $(SRC_PATH)/pixman, $(SRC_COPY)/pixman.tgz)
+	$(call quiet-command, cp "$(SRC_PATH)/tests/docker/run" "$(SRC_COPY)/run", \
+		"  COPY RUNNER")
+
+docker-qemu-src: $(SRC_COPY)
+
+docker-image: ${DOCKER_TARGETS}
+
+# General rule for building docker images
+docker-image-%: $(DOCKER_FILES_DIR)/%.docker
+	$(call quiet-command,\
+		$(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \
+		$(if $V,-v,) $(if $(NOCACHE),--no-cache),\
+		"  BUILD $*")
+
+# Expand all the pre-requistes for each docker image and test combination
+$(foreach i,$(DOCKER_IMAGES), \
+	$(foreach t,$(DOCKER_TESTS) $(DOCKER_TOOLS), \
+		$(eval .PHONY: docker-$t@$i) \
+		$(eval docker-$t@$i: docker-image-$i docker-run-$t@$i) \
+	) \
+	$(foreach t,$(DOCKER_TESTS), \
+		$(eval docker-test: docker-$t@$i) \
+	) \
+)
+
+docker:
+	@echo 'Build QEMU and run tests inside Docker containers'
+	@echo
+	@echo 'Available targets:'
+	@echo
+	@echo '    docker:              Print this help.'
+	@echo '    docker-test:         Run all image/test combinations.'
+	@echo '    docker-clean:        Kill and remove residual docker testing containers.'
+	@echo '    docker-TEST@IMAGE:   Run "TEST" in container "IMAGE".'
+	@echo '                         Note: "TEST" is one of the listed test name,'
+	@echo '                         or a script name under $$QEMU_SRC/tests/docker/;'
+	@echo '                         "IMAGE" is one of the listed container name."'
+	@echo '    docker-image:        Build all images.'
+	@echo '    docker-image-IMAGE:  Build image "IMG".'
+	@echo
+	@echo 'Available container images:'
+	@echo '    $(DOCKER_IMAGES)'
+	@echo
+	@echo 'Available tests:'
+	@echo '    $(DOCKER_TESTS)'
+	@echo
+	@echo 'Available tools:'
+	@echo '    $(DOCKER_TOOLS)'
+	@echo
+	@echo 'Special variables:'
+	@echo '    TARGET_LIST=a,b,c    Override target list in builds.'
+	@echo '    IMAGES="a b c ..":   Filters which images to build or run.'
+	@echo '    TESTS="x y z .."     Filters which tests to run (for docker-test).'
+	@echo '    J=[0..9]*            Overrides the -jN parameter for make commands'
+	@echo '                         (default is 1)'
+	@echo '    DEBUG=1              Stop and drop to shell in the created container'
+	@echo '                         before running the command.'
+	@echo '    NOCACHE=1            Ignore cache when build images.'
+
+docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/')
+docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/')
+docker-run-%: docker-qemu-src
+	@if test -z "$(IMAGE)" || test -z "$(CMD)"; \
+		then echo "Invalid target"; exit 1; \
+	fi
+	$(if $(filter $(TESTS),$(CMD)),$(if $(filter $(IMAGES),$(IMAGE)), \
+		$(call quiet-command,\
+			$(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
+				--privileged -t \
+				$(if $(DEBUG),-i,--net=none) \
+				-e TARGET_LIST=$(TARGET_LIST) \
+				-e V=$V -e J=$J -e DEBUG=$(DEBUG)\
+				-e CCACHE_DIR=/var/tmp/ccache \
+				-v $$(realpath $(SRC_COPY)):/var/tmp/qemu:ro \
+				-v $(DOCKER_CCACHE_DIR):/var/tmp/ccache \
+				-w /var/tmp/qemu \
+				qemu:$(IMAGE) \
+				$(if $V,/bin/bash -x ,) \
+				./run \
+				$(CMD); \
+			, "  RUN $(CMD) in $(IMAGE)")))
+
+docker-clean:
+	$(call quiet-command, $(SRC_PATH)/tests/docker/docker.py clean)