diff mbox series

[2/2] MIPS: Add support for ZSTD-compressed kernels

Message ID 20200821162948.146947-2-paul@crapouillou.net (mailing list archive)
State Superseded
Headers show
Series [1/2] lib: decompress_unzstd: Limit output size | expand

Commit Message

Paul Cercueil Aug. 21, 2020, 4:29 p.m. UTC 
Add support for self-extracting kernels with a ZSTD compression.

Tested on a kernel for the GCW-Zero, it allows to reduce the size of the
kernel file from 4.1 MiB with gzip to 3.5 MiB with ZSTD, and boots just
as fast.

Signed-off-by: Paul Cercueil <paul@crapouillou.net>
---
 arch/mips/Kconfig                      |  1 +
 arch/mips/boot/compressed/Makefile     |  1 +
 arch/mips/boot/compressed/decompress.c |  4 ++++
 arch/mips/boot/compressed/string.c     | 16 ++++++++++++++++
 4 files changed, 22 insertions(+)

Comments

Nick Terrell Aug. 24, 2020, 7:51 p.m. UTC | #1 
> On Aug 21, 2020, at 9:29 AM, Paul Cercueil <paul@crapouillou.net> wrote:
> 
> Add support for self-extracting kernels with a ZSTD compression.
> 
> Tested on a kernel for the GCW-Zero, it allows to reduce the size of the
> kernel file from 4.1 MiB with gzip to 3.5 MiB with ZSTD, and boots just
> as fast.
> 
> Signed-off-by: Paul Cercueil <paul@crapouillou.net>
> ---
> arch/mips/Kconfig                      |  1 +
> arch/mips/boot/compressed/Makefile     |  1 +
> arch/mips/boot/compressed/decompress.c |  4 ++++
> arch/mips/boot/compressed/string.c     | 16 ++++++++++++++++
> 4 files changed, 22 insertions(+)
> 
> diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
> index c95fa3a2484c..b9d7c4249dc9 100644
> --- a/arch/mips/Kconfig
> +++ b/arch/mips/Kconfig
> @@ -1890,6 +1890,7 @@ config SYS_SUPPORTS_ZBOOT
> 	select HAVE_KERNEL_LZMA
> 	select HAVE_KERNEL_LZO
> 	select HAVE_KERNEL_XZ
> +	select HAVE_KERNEL_ZSTD
> 
> config SYS_SUPPORTS_ZBOOT_UART16550
> 	bool
> diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile
> index 6e56caef69f0..86ddc6fc16f4 100644
> --- a/arch/mips/boot/compressed/Makefile
> +++ b/arch/mips/boot/compressed/Makefile
> @@ -70,6 +70,7 @@ tool_$(CONFIG_KERNEL_LZ4)     = lz4
> tool_$(CONFIG_KERNEL_LZMA)    = lzma
> tool_$(CONFIG_KERNEL_LZO)     = lzo
> tool_$(CONFIG_KERNEL_XZ)      = xzkern
> +tool_$(CONFIG_KERNEL_ZSTD)    = zstd

You can use zstd22 here. It will give you slightly better compression
without any additional memory usage. Also, you should add
-D__DISABLE_EXPORTS to the KBUILD_CFLAGS like x86 does [1].

[1] https://github.com/torvalds/linux/blob/master/arch/x86/boot/compressed/Makefile

-Nick

> targets += vmlinux.bin.z
> $(obj)/vmlinux.bin.z: $(obj)/vmlinux.bin FORCE
> diff --git a/arch/mips/boot/compressed/decompress.c b/arch/mips/boot/compressed/decompress.c
> index 88f5d637b1c4..c61c641674e6 100644
> --- a/arch/mips/boot/compressed/decompress.c
> +++ b/arch/mips/boot/compressed/decompress.c
> @@ -72,6 +72,10 @@ void error(char *x)
> #include "../../../../lib/decompress_unxz.c"
> #endif
> 
> +#ifdef CONFIG_KERNEL_ZSTD
> +#include "../../../../lib/decompress_unzstd.c"
> +#endif
> +
> const unsigned long __stack_chk_guard = 0x000a0dff;
> 
> void __stack_chk_fail(void)
> diff --git a/arch/mips/boot/compressed/string.c b/arch/mips/boot/compressed/string.c
> index 43beecc3587c..ab95722ec0c9 100644
> --- a/arch/mips/boot/compressed/string.c
> +++ b/arch/mips/boot/compressed/string.c
> @@ -27,3 +27,19 @@ void *memset(void *s, int c, size_t n)
> 		ss[i] = c;
> 	return s;
> }
> +
> +void *memmove(void *dest, const void *src, size_t n)
> +{
> +	unsigned int i;
> +	const char *s = src;
> +	char *d = dest;
> +
> +	if ((uintptr_t)dest < (uintptr_t)src) {
> +		for (i = 0; i < n; i++)
> +			d[i] = s[i];
> +	} else {
> +		for (i = n; i > 0; i--)
> +			d[i - 1] = s[i - 1];
> +	}
> +	return dest;
> +}
> -- 
> 2.28.0
>
Paul Cercueil Aug. 24, 2020, 9:02 p.m. UTC | #2 
Hi Nick,

Le lun. 24 août 2020 à 19:51, Nick Terrell <terrelln@fb.com> a écrit 
:
> 
> 
>>  On Aug 21, 2020, at 9:29 AM, Paul Cercueil <paul@crapouillou.net> 
>> wrote:
>> 
>>  Add support for self-extracting kernels with a ZSTD compression.
>> 
>>  Tested on a kernel for the GCW-Zero, it allows to reduce the size 
>> of the
>>  kernel file from 4.1 MiB with gzip to 3.5 MiB with ZSTD, and boots 
>> just
>>  as fast.
>> 
>>  Signed-off-by: Paul Cercueil <paul@crapouillou.net>
>>  ---
>>  arch/mips/Kconfig                      |  1 +
>>  arch/mips/boot/compressed/Makefile     |  1 +
>>  arch/mips/boot/compressed/decompress.c |  4 ++++
>>  arch/mips/boot/compressed/string.c     | 16 ++++++++++++++++
>>  4 files changed, 22 insertions(+)
>> 
>>  diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
>>  index c95fa3a2484c..b9d7c4249dc9 100644
>>  --- a/arch/mips/Kconfig
>>  +++ b/arch/mips/Kconfig
>>  @@ -1890,6 +1890,7 @@ config SYS_SUPPORTS_ZBOOT
>>  	select HAVE_KERNEL_LZMA
>>  	select HAVE_KERNEL_LZO
>>  	select HAVE_KERNEL_XZ
>>  +	select HAVE_KERNEL_ZSTD
>> 
>>  config SYS_SUPPORTS_ZBOOT_UART16550
>>  	bool
>>  diff --git a/arch/mips/boot/compressed/Makefile 
>> b/arch/mips/boot/compressed/Makefile
>>  index 6e56caef69f0..86ddc6fc16f4 100644
>>  --- a/arch/mips/boot/compressed/Makefile
>>  +++ b/arch/mips/boot/compressed/Makefile
>>  @@ -70,6 +70,7 @@ tool_$(CONFIG_KERNEL_LZ4)     = lz4
>>  tool_$(CONFIG_KERNEL_LZMA)    = lzma
>>  tool_$(CONFIG_KERNEL_LZO)     = lzo
>>  tool_$(CONFIG_KERNEL_XZ)      = xzkern
>>  +tool_$(CONFIG_KERNEL_ZSTD)    = zstd
> 
> You can use zstd22 here. It will give you slightly better compression
> without any additional memory usage. Also, you should add
> -D__DISABLE_EXPORTS to the KBUILD_CFLAGS like x86 does [1].

Indeed, it's 0.01% smaller :)

What is __DISABLE_EXPORTS for?

-Paul

> 
> [1] 
> https://github.com/torvalds/linux/blob/master/arch/x86/boot/compressed/Makefile
> 
> -Nick
> 
>>  targets += vmlinux.bin.z
>>  $(obj)/vmlinux.bin.z: $(obj)/vmlinux.bin FORCE
>>  diff --git a/arch/mips/boot/compressed/decompress.c 
>> b/arch/mips/boot/compressed/decompress.c
>>  index 88f5d637b1c4..c61c641674e6 100644
>>  --- a/arch/mips/boot/compressed/decompress.c
>>  +++ b/arch/mips/boot/compressed/decompress.c
>>  @@ -72,6 +72,10 @@ void error(char *x)
>>  #include "../../../../lib/decompress_unxz.c"
>>  #endif
>> 
>>  +#ifdef CONFIG_KERNEL_ZSTD
>>  +#include "../../../../lib/decompress_unzstd.c"
>>  +#endif
>>  +
>>  const unsigned long __stack_chk_guard = 0x000a0dff;
>> 
>>  void __stack_chk_fail(void)
>>  diff --git a/arch/mips/boot/compressed/string.c 
>> b/arch/mips/boot/compressed/string.c
>>  index 43beecc3587c..ab95722ec0c9 100644
>>  --- a/arch/mips/boot/compressed/string.c
>>  +++ b/arch/mips/boot/compressed/string.c
>>  @@ -27,3 +27,19 @@ void *memset(void *s, int c, size_t n)
>>  		ss[i] = c;
>>  	return s;
>>  }
>>  +
>>  +void *memmove(void *dest, const void *src, size_t n)
>>  +{
>>  +	unsigned int i;
>>  +	const char *s = src;
>>  +	char *d = dest;
>>  +
>>  +	if ((uintptr_t)dest < (uintptr_t)src) {
>>  +		for (i = 0; i < n; i++)
>>  +			d[i] = s[i];
>>  +	} else {
>>  +		for (i = n; i > 0; i--)
>>  +			d[i - 1] = s[i - 1];
>>  +	}
>>  +	return dest;
>>  +}
>>  --
>>  2.28.0
>>
Nick Terrell Aug. 24, 2020, 9:45 p.m. UTC | #3 
> On Aug 24, 2020, at 2:02 PM, Paul Cercueil <paul@crapouillou.net> wrote:
> 
> Hi Nick,
> 
> Le lun. 24 août 2020 à 19:51, Nick Terrell <terrelln@fb.com> a écrit :
>>> On Aug 21, 2020, at 9:29 AM, Paul Cercueil <paul@crapouillou.net> wrote:
>>> Add support for self-extracting kernels with a ZSTD compression.
>>> Tested on a kernel for the GCW-Zero, it allows to reduce the size of the
>>> kernel file from 4.1 MiB with gzip to 3.5 MiB with ZSTD, and boots just
>>> as fast.
>>> Signed-off-by: Paul Cercueil <paul@crapouillou.net>
>>> ---
>>> arch/mips/Kconfig                      |  1 +
>>> arch/mips/boot/compressed/Makefile     |  1 +
>>> arch/mips/boot/compressed/decompress.c |  4 ++++
>>> arch/mips/boot/compressed/string.c     | 16 ++++++++++++++++
>>> 4 files changed, 22 insertions(+)
>>> diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
>>> index c95fa3a2484c..b9d7c4249dc9 100644
>>> --- a/arch/mips/Kconfig
>>> +++ b/arch/mips/Kconfig
>>> @@ -1890,6 +1890,7 @@ config SYS_SUPPORTS_ZBOOT
>>> 	select HAVE_KERNEL_LZMA
>>> 	select HAVE_KERNEL_LZO
>>> 	select HAVE_KERNEL_XZ
>>> +	select HAVE_KERNEL_ZSTD
>>> config SYS_SUPPORTS_ZBOOT_UART16550
>>> 	bool
>>> diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile
>>> index 6e56caef69f0..86ddc6fc16f4 100644
>>> --- a/arch/mips/boot/compressed/Makefile
>>> +++ b/arch/mips/boot/compressed/Makefile
>>> @@ -70,6 +70,7 @@ tool_$(CONFIG_KERNEL_LZ4)     = lz4
>>> tool_$(CONFIG_KERNEL_LZMA)    = lzma
>>> tool_$(CONFIG_KERNEL_LZO)     = lzo
>>> tool_$(CONFIG_KERNEL_XZ)      = xzkern
>>> +tool_$(CONFIG_KERNEL_ZSTD)    = zstd
>> You can use zstd22 here. It will give you slightly better compression
>> without any additional memory usage. Also, you should add
>> -D__DISABLE_EXPORTS to the KBUILD_CFLAGS like x86 does [1].
> 
> Indeed, it's 0.01% smaller :)
> 
> What is __DISABLE_EXPORTS for?

It disables the EXPORT_SYMBOL() macros inside of lib/zstd/decompress.c.
On x86 the kernel won’t boot with these defined. Other decompressors hide
them if the STATIC macro is defined, but zstd uses this method, which was
added somewhat recently.

-Nick

> -Paul
> 
>> [1] https://github.com/torvalds/linux/blob/master/arch/x86/boot/compressed/Makefile
>> -Nick
>>> targets += vmlinux.bin.z
>>> $(obj)/vmlinux.bin.z: $(obj)/vmlinux.bin FORCE
>>> diff --git a/arch/mips/boot/compressed/decompress.c b/arch/mips/boot/compressed/decompress.c
>>> index 88f5d637b1c4..c61c641674e6 100644
>>> --- a/arch/mips/boot/compressed/decompress.c
>>> +++ b/arch/mips/boot/compressed/decompress.c
>>> @@ -72,6 +72,10 @@ void error(char *x)
>>> #include "../../../../lib/decompress_unxz.c"
>>> #endif
>>> +#ifdef CONFIG_KERNEL_ZSTD
>>> +#include "../../../../lib/decompress_unzstd.c"
>>> +#endif
>>> +
>>> const unsigned long __stack_chk_guard = 0x000a0dff;
>>> void __stack_chk_fail(void)
>>> diff --git a/arch/mips/boot/compressed/string.c b/arch/mips/boot/compressed/string.c
>>> index 43beecc3587c..ab95722ec0c9 100644
>>> --- a/arch/mips/boot/compressed/string.c
>>> +++ b/arch/mips/boot/compressed/string.c
>>> @@ -27,3 +27,19 @@ void *memset(void *s, int c, size_t n)
>>> 		ss[i] = c;
>>> 	return s;
>>> }
>>> +
>>> +void *memmove(void *dest, const void *src, size_t n)
>>> +{
>>> +	unsigned int i;
>>> +	const char *s = src;
>>> +	char *d = dest;
>>> +
>>> +	if ((uintptr_t)dest < (uintptr_t)src) {
>>> +		for (i = 0; i < n; i++)
>>> +			d[i] = s[i];
>>> +	} else {
>>> +		for (i = n; i > 0; i--)
>>> +			d[i - 1] = s[i - 1];
>>> +	}
>>> +	return dest;
>>> +}
>>> --
>>> 2.28.0
diff mbox series

Patch

diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index c95fa3a2484c..b9d7c4249dc9 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -1890,6 +1890,7 @@  config SYS_SUPPORTS_ZBOOT
 	select HAVE_KERNEL_LZMA
 	select HAVE_KERNEL_LZO
 	select HAVE_KERNEL_XZ
+	select HAVE_KERNEL_ZSTD
 
 config SYS_SUPPORTS_ZBOOT_UART16550
 	bool
diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile
index 6e56caef69f0..86ddc6fc16f4 100644
--- a/arch/mips/boot/compressed/Makefile
+++ b/arch/mips/boot/compressed/Makefile
@@ -70,6 +70,7 @@  tool_$(CONFIG_KERNEL_LZ4)     = lz4
 tool_$(CONFIG_KERNEL_LZMA)    = lzma
 tool_$(CONFIG_KERNEL_LZO)     = lzo
 tool_$(CONFIG_KERNEL_XZ)      = xzkern
+tool_$(CONFIG_KERNEL_ZSTD)    = zstd
 
 targets += vmlinux.bin.z
 $(obj)/vmlinux.bin.z: $(obj)/vmlinux.bin FORCE
diff --git a/arch/mips/boot/compressed/decompress.c b/arch/mips/boot/compressed/decompress.c
index 88f5d637b1c4..c61c641674e6 100644
--- a/arch/mips/boot/compressed/decompress.c
+++ b/arch/mips/boot/compressed/decompress.c
@@ -72,6 +72,10 @@  void error(char *x)
 #include "../../../../lib/decompress_unxz.c"
 #endif
 
+#ifdef CONFIG_KERNEL_ZSTD
+#include "../../../../lib/decompress_unzstd.c"
+#endif
+
 const unsigned long __stack_chk_guard = 0x000a0dff;
 
 void __stack_chk_fail(void)
diff --git a/arch/mips/boot/compressed/string.c b/arch/mips/boot/compressed/string.c
index 43beecc3587c..ab95722ec0c9 100644
--- a/arch/mips/boot/compressed/string.c
+++ b/arch/mips/boot/compressed/string.c
@@ -27,3 +27,19 @@  void *memset(void *s, int c, size_t n)
 		ss[i] = c;
 	return s;
 }
+
+void *memmove(void *dest, const void *src, size_t n)
+{
+	unsigned int i;
+	const char *s = src;
+	char *d = dest;
+
+	if ((uintptr_t)dest < (uintptr_t)src) {
+		for (i = 0; i < n; i++)
+			d[i] = s[i];
+	} else {
+		for (i = n; i > 0; i--)
+			d[i - 1] = s[i - 1];
+	}
+	return dest;
+}