diff mbox series

[v6,02/12] kbuild: deb-pkg: create source package without cleaning

Message ID 20230215012034.403356-2-masahiroy@kernel.org (mailing list archive)
State New, archived
Headers show
Series [v6,01/12] kbuild: add a tool to list files ignored by git | expand

Commit Message

Masahiro Yamada Feb. 15, 2023, 1:20 a.m. UTC
If you run 'make deb-pkg', all objects are lost due to 'make clean',
which makes the incremental builds impossible.

Instead of cleaning, pass the exclude list to tar's --exclude-from
option.

Previously, *.diff.gz contained some check-in files such as
.clang-format, .cocciconfig.

With this commit, *.diff.gz will only contain the .config and debian/.
The other source files will go into the .orig tarball.

linux.tar.gz is rebuilt only when the source files that would go into
the tarball are changed.

Signed-off-by: Masahiro Yamada <masahiroy@kernel.org>
---

Changes in v6:
  - Add more stubs to cmd_tar so I can reuse it for the other rules

Changes in v5:
  - Avoid unneeded rebuild of the tarball when nothing in the source tree
    is changed.

Changes in v4:
  - Fix a typo in comment

Changes in v3:
  - Add --extra-pattern='*.rej'
  - Exclude symlinks at the toplevel
  - Add --sort=name tar option

 scripts/Makefile.package | 48 +++++++++++++++++++++++++++++++++++-----
 scripts/package/mkdebian | 24 ++++++++++++++++++++
 2 files changed, 66 insertions(+), 6 deletions(-)

Comments

Nicolas Schier Feb. 24, 2023, 10:28 p.m. UTC | #1
On Wed, Feb 15, 2023 at 10:20:24AM +0900 Masahiro Yamada wrote:
> If you run 'make deb-pkg', all objects are lost due to 'make clean',
> which makes the incremental builds impossible.
> 
> Instead of cleaning, pass the exclude list to tar's --exclude-from
> option.
> 
> Previously, *.diff.gz contained some check-in files such as
> .clang-format, .cocciconfig.
> 
> With this commit, *.diff.gz will only contain the .config and debian/.
> The other source files will go into the .orig tarball.
> 
> linux.tar.gz is rebuilt only when the source files that would go into
> the tarball are changed.
> 
> Signed-off-by: Masahiro Yamada <masahiroy@kernel.org>
> ---
> 
> Changes in v6:
>   - Add more stubs to cmd_tar so I can reuse it for the other rules
> 
> Changes in v5:
>   - Avoid unneeded rebuild of the tarball when nothing in the source tree
>     is changed.
> 
> Changes in v4:
>   - Fix a typo in comment
> 
> Changes in v3:
>   - Add --extra-pattern='*.rej'
>   - Exclude symlinks at the toplevel
>   - Add --sort=name tar option
> 
>  scripts/Makefile.package | 48 +++++++++++++++++++++++++++++++++++-----
>  scripts/package/mkdebian | 24 ++++++++++++++++++++
>  2 files changed, 66 insertions(+), 6 deletions(-)
> 
> diff --git a/scripts/Makefile.package b/scripts/Makefile.package
> index dfbf40454a99..f0002ace4156 100644
> --- a/scripts/Makefile.package
> +++ b/scripts/Makefile.package
> @@ -43,13 +43,47 @@ if test "$(objtree)" != "$(srctree)"; then \
>  	echo >&2 "  ERROR:"; \
>  	echo >&2 "  Building source tarball is not possible outside the"; \
>  	echo >&2 "  kernel source tree. Don't set KBUILD_OUTPUT, or use the"; \
> -	echo >&2 "  binrpm-pkg or bindeb-pkg target instead."; \
> +	echo >&2 "  binrpm-pkg target instead."; \
>  	echo >&2; \
>  	false; \
>  fi ; \
>  tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
>  	--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3)
>  
> +# .tmp_filelist .tmp_filelist_exclude
> +# ---------------------------------------------------------------------------
> +
> +scripts/list-gitignored: FORCE
> +	$(Q)$(MAKE) -f $(srctree)/Makefile scripts_package
> +
> +# 1f5d3a6b6532e25a5cdf1f311956b2b03d343a48 removed '*.rej' from .gitignore,
> +# but it is definitely a generated file.
> +filechk_filelist = \
> +	$< --exclude='*.rej' --output=$@_exclude --prefix=./ --rootdir=$(srctree) --stat=-
> +
> +.tmp_filelist: scripts/list-gitignored FORCE
> +	$(call filechk,filelist)
> +
> +# tarball
> +# ---------------------------------------------------------------------------
> +
> +quiet_cmd_tar = TAR     $@
> +      cmd_tar = tar -c -f $@ $(tar-compress-opt) $(tar-exclude-opt) \
> +                --owner=0 --group=0 --sort=name \
> +                --transform 's:^\.:$*:S' -C $(tar-rootdir) .
> +
> +tar-rootdir := $(srctree)
> +
> +%.tar.gz: private tar-compress-opt := -I $(KGZIP)
> +%.tar.gz:
> +	$(call cmd,tar)
> +
> +# Linux source tarball
> +# ---------------------------------------------------------------------------
> +
> +linux.tar.gz: tar-exclude-opt = --exclude=./$@ --exclude-from=$<_exclude
> +linux.tar.gz: .tmp_filelist
> +
>  # rpm-pkg
>  # ---------------------------------------------------------------------------
>  PHONY += rpm-pkg
> @@ -80,13 +114,12 @@ binrpm-pkg:
>  		$(UTS_MACHINE)-linux -bb $(objtree)/binkernel.spec
>  
>  PHONY += deb-pkg
> -deb-pkg:
> -	$(MAKE) clean
> +deb-pkg: linux.tar.gz
>  	$(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian
> -	$(call cmd,src_tar,$(KDEB_SOURCENAME))
>  	origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\
> -		mv $(KDEB_SOURCENAME).tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz
> -	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) --source-option=-sP -i.git -us -uc
> +		cp linux.tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz
> +	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) \
> +		--build=source,binary --source-option=-sP -nc -us -uc
>  
>  PHONY += bindeb-pkg
>  bindeb-pkg:
> @@ -174,4 +207,7 @@ help:
>  	@echo '  perf-tarxz-src-pkg  - Build $(perf-tar).tar.xz source tarball'
>  	@echo '  perf-tarzst-src-pkg - Build $(perf-tar).tar.zst source tarball'
>  
> +PHONY += FORCE
> +FORCE:
> +
>  .PHONY: $(PHONY)
> diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian
> index c3bbef7a6754..68b13ef590ba 100755
> --- a/scripts/package/mkdebian
> +++ b/scripts/package/mkdebian
> @@ -84,6 +84,8 @@ set_debarch() {
>  	fi
>  }
>  
> +rm -rf debian
> +
>  # Some variables and settings used throughout the script
>  version=$KERNELRELEASE
>  if [ -n "$KDEB_PKGVERSION" ]; then
> @@ -135,6 +137,28 @@ fi
>  mkdir -p debian/source/
>  echo "1.0" > debian/source/format
>  
> +# Ugly: ignore anything except .config or debian/
> +cat<<'EOF' > debian/source/local-options
> +diff-ignore
> +
> +extend-diff-ignore = ^[^.d]
> +
> +extend-diff-ignore = ^\.[^c]
> +extend-diff-ignore = ^\.c($|[^o])
> +extend-diff-ignore = ^\.co($|[^n])
> +extend-diff-ignore = ^\.con($|[^f])
> +extend-diff-ignore = ^\.conf($|[^i])
> +extend-diff-ignore = ^\.confi($|[^g])
> +extend-diff-ignore = ^\.config.
> +
> +extend-diff-ignore = ^d($|[^e])
> +extend-diff-ignore = ^de($|[^b])
> +extend-diff-ignore = ^deb($|[^i])
> +extend-diff-ignore = ^debi($|[^a])
> +extend-diff-ignore = ^debia($|[^n])
> +extend-diff-ignore = ^debian[^/]
> +EOF
> +
>  echo $debarch > debian/arch
>  extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev:native)"
>  extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)"

ah, nice.  When I attempted to build with current Debian amd64 kernel config,
python3 was missing as build-depencendy (due to some BPF kconfig).

Reviewed-by: Nicolas Schier <nicolas@fjasle.eu>

> -- 
> 2.34.1
diff mbox series

Patch

diff --git a/scripts/Makefile.package b/scripts/Makefile.package
index dfbf40454a99..f0002ace4156 100644
--- a/scripts/Makefile.package
+++ b/scripts/Makefile.package
@@ -43,13 +43,47 @@  if test "$(objtree)" != "$(srctree)"; then \
 	echo >&2 "  ERROR:"; \
 	echo >&2 "  Building source tarball is not possible outside the"; \
 	echo >&2 "  kernel source tree. Don't set KBUILD_OUTPUT, or use the"; \
-	echo >&2 "  binrpm-pkg or bindeb-pkg target instead."; \
+	echo >&2 "  binrpm-pkg target instead."; \
 	echo >&2; \
 	false; \
 fi ; \
 tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
 	--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3)
 
+# .tmp_filelist .tmp_filelist_exclude
+# ---------------------------------------------------------------------------
+
+scripts/list-gitignored: FORCE
+	$(Q)$(MAKE) -f $(srctree)/Makefile scripts_package
+
+# 1f5d3a6b6532e25a5cdf1f311956b2b03d343a48 removed '*.rej' from .gitignore,
+# but it is definitely a generated file.
+filechk_filelist = \
+	$< --exclude='*.rej' --output=$@_exclude --prefix=./ --rootdir=$(srctree) --stat=-
+
+.tmp_filelist: scripts/list-gitignored FORCE
+	$(call filechk,filelist)
+
+# tarball
+# ---------------------------------------------------------------------------
+
+quiet_cmd_tar = TAR     $@
+      cmd_tar = tar -c -f $@ $(tar-compress-opt) $(tar-exclude-opt) \
+                --owner=0 --group=0 --sort=name \
+                --transform 's:^\.:$*:S' -C $(tar-rootdir) .
+
+tar-rootdir := $(srctree)
+
+%.tar.gz: private tar-compress-opt := -I $(KGZIP)
+%.tar.gz:
+	$(call cmd,tar)
+
+# Linux source tarball
+# ---------------------------------------------------------------------------
+
+linux.tar.gz: tar-exclude-opt = --exclude=./$@ --exclude-from=$<_exclude
+linux.tar.gz: .tmp_filelist
+
 # rpm-pkg
 # ---------------------------------------------------------------------------
 PHONY += rpm-pkg
@@ -80,13 +114,12 @@  binrpm-pkg:
 		$(UTS_MACHINE)-linux -bb $(objtree)/binkernel.spec
 
 PHONY += deb-pkg
-deb-pkg:
-	$(MAKE) clean
+deb-pkg: linux.tar.gz
 	$(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian
-	$(call cmd,src_tar,$(KDEB_SOURCENAME))
 	origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\
-		mv $(KDEB_SOURCENAME).tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz
-	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) --source-option=-sP -i.git -us -uc
+		cp linux.tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz
+	+dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) \
+		--build=source,binary --source-option=-sP -nc -us -uc
 
 PHONY += bindeb-pkg
 bindeb-pkg:
@@ -174,4 +207,7 @@  help:
 	@echo '  perf-tarxz-src-pkg  - Build $(perf-tar).tar.xz source tarball'
 	@echo '  perf-tarzst-src-pkg - Build $(perf-tar).tar.zst source tarball'
 
+PHONY += FORCE
+FORCE:
+
 .PHONY: $(PHONY)
diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian
index c3bbef7a6754..68b13ef590ba 100755
--- a/scripts/package/mkdebian
+++ b/scripts/package/mkdebian
@@ -84,6 +84,8 @@  set_debarch() {
 	fi
 }
 
+rm -rf debian
+
 # Some variables and settings used throughout the script
 version=$KERNELRELEASE
 if [ -n "$KDEB_PKGVERSION" ]; then
@@ -135,6 +137,28 @@  fi
 mkdir -p debian/source/
 echo "1.0" > debian/source/format
 
+# Ugly: ignore anything except .config or debian/
+cat<<'EOF' > debian/source/local-options
+diff-ignore
+
+extend-diff-ignore = ^[^.d]
+
+extend-diff-ignore = ^\.[^c]
+extend-diff-ignore = ^\.c($|[^o])
+extend-diff-ignore = ^\.co($|[^n])
+extend-diff-ignore = ^\.con($|[^f])
+extend-diff-ignore = ^\.conf($|[^i])
+extend-diff-ignore = ^\.confi($|[^g])
+extend-diff-ignore = ^\.config.
+
+extend-diff-ignore = ^d($|[^e])
+extend-diff-ignore = ^de($|[^b])
+extend-diff-ignore = ^deb($|[^i])
+extend-diff-ignore = ^debi($|[^a])
+extend-diff-ignore = ^debia($|[^n])
+extend-diff-ignore = ^debian[^/]
+EOF
+
 echo $debarch > debian/arch
 extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev:native)"
 extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)"