[v2,2/2] btrfs: Add zstd support to grub btrfs
diff mbox series

Message ID 20181008230621.2185884-3-terrelln@fb.com
State New
Headers show
Series
  • btrfs: Add zstd support to grub btrfs
Related show

Commit Message

Nick Terrell Oct. 8, 2018, 11:06 p.m. UTC
Adds zstd support to the btrfs module. I'm not sure that my changes to the
Makefiles are correct, please let me know if I need to do something
differently.

Tested on Ubuntu-18.04 with a btrfs /boot partition with and without zstd
compression. A test case was also added to the test suite that fails before
the patch, and passes after.

Signed-off-by: Nick Terrell <terrelln@fb.com>
---
v1 -> v2:
- Fix comments from Daniel Kiper

 Makefile.util.def            |  10 +++-
 grub-core/Makefile.core.def  |  10 +++-
 grub-core/fs/btrfs.c         | 105 ++++++++++++++++++++++++++++++++++-
 grub-core/lib/zstd/up        |   1 -
 tests/btrfs_test.in          |   1 +
 tests/util/grub-fs-tester.in |   2 +-
 6 files changed, 124 insertions(+), 5 deletions(-)
 delete mode 120000 grub-core/lib/zstd/up

--
2.17.1

Comments

Daniel Kiper Oct. 9, 2018, 7:07 p.m. UTC | #1
On Mon, Oct 08, 2018 at 04:06:21PM -0700, Nick Terrell wrote:
> Adds zstd support to the btrfs module. I'm not sure that my changes to the
> Makefiles are correct, please let me know if I need to do something
> differently.
>
> Tested on Ubuntu-18.04 with a btrfs /boot partition with and without zstd
> compression. A test case was also added to the test suite that fails before
> the patch, and passes after.
>
> Signed-off-by: Nick Terrell <terrelln@fb.com>
> ---
> v1 -> v2:
> - Fix comments from Daniel Kiper
>
>  Makefile.util.def            |  10 +++-
>  grub-core/Makefile.core.def  |  10 +++-
>  grub-core/fs/btrfs.c         | 105 ++++++++++++++++++++++++++++++++++-
>  grub-core/lib/zstd/up        |   1 -

Please do not take this file into patch #1.

>  tests/btrfs_test.in          |   1 +
>  tests/util/grub-fs-tester.in |   2 +-
>  6 files changed, 124 insertions(+), 5 deletions(-)
>  delete mode 120000 grub-core/lib/zstd/up
>
> diff --git a/Makefile.util.def b/Makefile.util.def
> index f9caccb97..81898c509 100644
> --- a/Makefile.util.def
> +++ b/Makefile.util.def
> @@ -54,7 +54,7 @@ library = {
>  library = {
>    name = libgrubmods.a;
>    cflags = '-fno-builtin -Wno-undef';
> -  cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -DMINILZO_HAVE_CONFIG_H';
> +  cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -I$(top_srcdir)/grub-core/lib/zstd -DMINILZO_HAVE_CONFIG_H';

s/top_srcdir/srcdir/?

>    common_nodist = grub_script.tab.c;
>    common_nodist = grub_script.yy.c;
> @@ -164,6 +164,14 @@ library = {
>    common = grub-core/lib/xzembed/xz_dec_bcj.c;
>    common = grub-core/lib/xzembed/xz_dec_lzma2.c;
>    common = grub-core/lib/xzembed/xz_dec_stream.c;
> +  common = grub-core/lib/zstd/debug.c;
> +  common = grub-core/lib/zstd/entropy_common.c;
> +  common = grub-core/lib/zstd/error_private.c;
> +  common = grub-core/lib/zstd/fse_decompress.c;
> +  common = grub-core/lib/zstd/xxhash.c;
> +  common = grub-core/lib/zstd/zstd_common.c;
> +  common = grub-core/lib/zstd/huf_decompress.c;
> +  common = grub-core/lib/zstd/zstd_decompress.c;
>  };
>
>  program = {
> diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def
> index 2c1d62cee..2a97bb807 100644
> --- a/grub-core/Makefile.core.def
> +++ b/grub-core/Makefile.core.def
> @@ -1265,8 +1265,16 @@ module = {
>    name = btrfs;
>    common = fs/btrfs.c;
>    common = lib/crc.c;
> +  common = lib/zstd/debug.c;
> +  common = lib/zstd/entropy_common.c;
> +  common = lib/zstd/error_private.c;
> +  common = lib/zstd/fse_decompress.c;
> +  common = lib/zstd/xxhash.c;
> +  common = lib/zstd/zstd_common.c;
> +  common = lib/zstd/huf_decompress.c;
> +  common = lib/zstd/zstd_decompress.c;

Could you sort the names of files here?

>    cflags = '$(CFLAGS_POSIX) -Wno-undef';
> -  cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -DMINILZO_HAVE_CONFIG_H';
> +  cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -I$(srcdir)/lib/zstd -DMINILZO_HAVE_CONFIG_H';
>  };
>
>  module = {
> diff --git a/grub-core/fs/btrfs.c b/grub-core/fs/btrfs.c
> index 4849c1ceb..cfeb7995e 100644
> --- a/grub-core/fs/btrfs.c
> +++ b/grub-core/fs/btrfs.c
> @@ -17,6 +17,8 @@
>   *  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
>   */
>
> +#define ZSTD_STATIC_LINKING_ONLY
> +
>  #include <grub/err.h>
>  #include <grub/file.h>
>  #include <grub/mm.h>
> @@ -27,6 +29,7 @@
>  #include <grub/lib/crc.h>
>  #include <grub/deflate.h>
>  #include <minilzo.h>
> +#include <zstd.h>
>  #include <grub/i18n.h>
>  #include <grub/btrfs.h>
>
> @@ -45,6 +48,9 @@ GRUB_MOD_LICENSE ("GPLv3+");
>  #define GRUB_BTRFS_LZO_BLOCK_MAX_CSIZE (GRUB_BTRFS_LZO_BLOCK_SIZE + \
>  				     (GRUB_BTRFS_LZO_BLOCK_SIZE / 16) + 64 + 3)
>
> +#define ZSTD_BTRFS_MAX_WINDOWLOG 17
> +#define ZSTD_BTRFS_MAX_INPUT     (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
> +
>  typedef grub_uint8_t grub_btrfs_checksum_t[0x20];
>  typedef grub_uint16_t grub_btrfs_uuid_t[8];
>
> @@ -212,6 +218,7 @@ struct grub_btrfs_extent_data
>  #define GRUB_BTRFS_COMPRESSION_NONE 0
>  #define GRUB_BTRFS_COMPRESSION_ZLIB 1
>  #define GRUB_BTRFS_COMPRESSION_LZO  2
> +#define GRUB_BTRFS_COMPRESSION_ZSTD 3
>
>  #define GRUB_BTRFS_OBJECT_ID_CHUNK 0x100
>
> @@ -912,6 +919,88 @@ grub_btrfs_read_inode (struct grub_btrfs_data *data,
>    return grub_btrfs_read_logical (data, elemaddr, inode, sizeof (*inode), 0);
>  }
>
> +static void* grub_zstd_malloc (void* state, size_t size)
> +{
> +	(void)state;

Do we need that?

> +	return grub_malloc (size);
> +}
> +
> +static void grub_zstd_free (void* state, void* address)
> +{
> +	(void)state;

Ditto?

> +	return grub_free (address);
> +}
> +
> +static ZSTD_customMem grub_zstd_allocator (void)
> +{
> +	ZSTD_customMem allocator;

Please add empty line here.

> +	allocator.customAlloc = &grub_zstd_malloc;
> +	allocator.customFree = &grub_zstd_free;
> +	allocator.opaque = NULL;

And here.

> +	return allocator;
> +}
> +
> +static grub_ssize_t
> +grub_btrfs_zstd_decompress (char *ibuf, grub_size_t isize, grub_off_t off,
> +			    char *obuf, grub_size_t osize)
> +{
> +	void *allocated = NULL;
> +	char *otmpbuf = obuf;
> +	grub_size_t otmpsize = osize;
> +	ZSTD_DCtx *dctx = NULL;
> +	grub_size_t zstd_ret;
> +	grub_ssize_t ret = -1;
> +
> +	/* Zstd will fail if it can't fit the entire output in the destination
> +	 * buffer, so if osize isn't large enough, allocate a temporary buffer.
> +	 */

        /*
         * Zstd will fail if it can't fit the entire output in the destination
         * buffer, so if osize isn't large enough, allocate a temporary buffer.
         */

> +	if (otmpsize < ZSTD_BTRFS_MAX_INPUT) {
> +		allocated = grub_malloc (ZSTD_BTRFS_MAX_INPUT);
> +		if (!allocated) {
> +			grub_dprintf ("zstd", "outtmpbuf allocation failed\n");
> +			goto err;
> +		}
> +		otmpbuf = (char*)allocated;
> +		otmpsize = ZSTD_BTRFS_MAX_INPUT;
> +	}
> +
> +	/* Create the ZSTD_DCtx. */
> +	dctx = ZSTD_createDCtx_advanced (grub_zstd_allocator ());
> +	if (!dctx) {
> +		grub_dprintf ("zstd", "failed to allocate the ZSTD_DCtx\n");

Should not you set grub_errno here?

> +		goto err;
> +	}
> +
> +	/* Get the real input size, there may be junk at the
> +	 * end of the frame.
> +	 */

Please reformat the comment like above.

> +	isize = ZSTD_findFrameCompressedSize (ibuf, isize);
> +	if (ZSTD_isError (isize)) {
> +		grub_dprintf ("zstd", "first frame is invalid: %s\n",
> +			      ZSTD_getErrorName (isize));

grub_errno?

> +		goto err;
> +	}
> +
> +	/* Decompress and check for errors */

This can stay as is. You can add a full stop at the end.

> +	zstd_ret = ZSTD_decompressDCtx (dctx, otmpbuf, otmpsize, ibuf, isize);
> +	if (ZSTD_isError (zstd_ret)) {
> +		grub_dprintf ("zstd", "zstd failed with message: %s\n",
> +			      ZSTD_getErrorName (zstd_ret));

grub_errno?

> +		goto err;
> +	}
> +
> +	/* Move the requested data into the obuf.
> +	 * obuf may be equal to otmpbuf, which is why grub_memmove() is required.
> +	 */

Begs for reformat.

Daniel
Nick Terrell Oct. 9, 2018, 8:16 p.m. UTC | #2
> On Oct 9, 2018, at 12:07 PM, Daniel Kiper <dkiper@net-space.pl> wrote:
> On Mon, Oct 08, 2018 at 04:06:21PM -0700, Nick Terrell wrote:
>> Adds zstd support to the btrfs module. I'm not sure that my changes to the
>> Makefiles are correct, please let me know if I need to do something
>> differently.
>> 
>> Tested on Ubuntu-18.04 with a btrfs /boot partition with and without zstd
>> compression. A test case was also added to the test suite that fails before
>> the patch, and passes after.
>> 
>> Signed-off-by: Nick Terrell <terrelln@fb.com>
>> ---
>> v1 -> v2:
>> - Fix comments from Daniel Kiper
>> 
>> Makefile.util.def            |  10 +++-
>> grub-core/Makefile.core.def  |  10 +++-
>> grub-core/fs/btrfs.c         | 105 ++++++++++++++++++++++++++++++++++-
>> grub-core/lib/zstd/up        |   1 -
> 
> Please do not take this file into patch #1.

Oops, sorry about that.

>> 
>> 
>> +static void* grub_zstd_malloc (void* state, size_t size)
>> +{
>> +	(void)state;
> 
> Do we need that?

Yeah, otherwise my compiler (gcc-7.3) gives an unused parameter error.

Thanks for the fast review! I'll have an updated version up later today.

Nick

Patch
diff mbox series

diff --git a/Makefile.util.def b/Makefile.util.def
index f9caccb97..81898c509 100644
--- a/Makefile.util.def
+++ b/Makefile.util.def
@@ -54,7 +54,7 @@  library = {
 library = {
   name = libgrubmods.a;
   cflags = '-fno-builtin -Wno-undef';
-  cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -DMINILZO_HAVE_CONFIG_H';
+  cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -I$(top_srcdir)/grub-core/lib/zstd -DMINILZO_HAVE_CONFIG_H';

   common_nodist = grub_script.tab.c;
   common_nodist = grub_script.yy.c;
@@ -164,6 +164,14 @@  library = {
   common = grub-core/lib/xzembed/xz_dec_bcj.c;
   common = grub-core/lib/xzembed/xz_dec_lzma2.c;
   common = grub-core/lib/xzembed/xz_dec_stream.c;
+  common = grub-core/lib/zstd/debug.c;
+  common = grub-core/lib/zstd/entropy_common.c;
+  common = grub-core/lib/zstd/error_private.c;
+  common = grub-core/lib/zstd/fse_decompress.c;
+  common = grub-core/lib/zstd/xxhash.c;
+  common = grub-core/lib/zstd/zstd_common.c;
+  common = grub-core/lib/zstd/huf_decompress.c;
+  common = grub-core/lib/zstd/zstd_decompress.c;
 };

 program = {
diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def
index 2c1d62cee..2a97bb807 100644
--- a/grub-core/Makefile.core.def
+++ b/grub-core/Makefile.core.def
@@ -1265,8 +1265,16 @@  module = {
   name = btrfs;
   common = fs/btrfs.c;
   common = lib/crc.c;
+  common = lib/zstd/debug.c;
+  common = lib/zstd/entropy_common.c;
+  common = lib/zstd/error_private.c;
+  common = lib/zstd/fse_decompress.c;
+  common = lib/zstd/xxhash.c;
+  common = lib/zstd/zstd_common.c;
+  common = lib/zstd/huf_decompress.c;
+  common = lib/zstd/zstd_decompress.c;
   cflags = '$(CFLAGS_POSIX) -Wno-undef';
-  cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -DMINILZO_HAVE_CONFIG_H';
+  cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -I$(srcdir)/lib/zstd -DMINILZO_HAVE_CONFIG_H';
 };

 module = {
diff --git a/grub-core/fs/btrfs.c b/grub-core/fs/btrfs.c
index 4849c1ceb..cfeb7995e 100644
--- a/grub-core/fs/btrfs.c
+++ b/grub-core/fs/btrfs.c
@@ -17,6 +17,8 @@ 
  *  along with GRUB.  If not, see <http://www.gnu.org/licenses/>.
  */

+#define ZSTD_STATIC_LINKING_ONLY
+
 #include <grub/err.h>
 #include <grub/file.h>
 #include <grub/mm.h>
@@ -27,6 +29,7 @@ 
 #include <grub/lib/crc.h>
 #include <grub/deflate.h>
 #include <minilzo.h>
+#include <zstd.h>
 #include <grub/i18n.h>
 #include <grub/btrfs.h>

@@ -45,6 +48,9 @@  GRUB_MOD_LICENSE ("GPLv3+");
 #define GRUB_BTRFS_LZO_BLOCK_MAX_CSIZE (GRUB_BTRFS_LZO_BLOCK_SIZE + \
 				     (GRUB_BTRFS_LZO_BLOCK_SIZE / 16) + 64 + 3)

+#define ZSTD_BTRFS_MAX_WINDOWLOG 17
+#define ZSTD_BTRFS_MAX_INPUT     (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
+
 typedef grub_uint8_t grub_btrfs_checksum_t[0x20];
 typedef grub_uint16_t grub_btrfs_uuid_t[8];

@@ -212,6 +218,7 @@  struct grub_btrfs_extent_data
 #define GRUB_BTRFS_COMPRESSION_NONE 0
 #define GRUB_BTRFS_COMPRESSION_ZLIB 1
 #define GRUB_BTRFS_COMPRESSION_LZO  2
+#define GRUB_BTRFS_COMPRESSION_ZSTD 3

 #define GRUB_BTRFS_OBJECT_ID_CHUNK 0x100

@@ -912,6 +919,88 @@  grub_btrfs_read_inode (struct grub_btrfs_data *data,
   return grub_btrfs_read_logical (data, elemaddr, inode, sizeof (*inode), 0);
 }

+static void* grub_zstd_malloc (void* state, size_t size)
+{
+	(void)state;
+	return grub_malloc (size);
+}
+
+static void grub_zstd_free (void* state, void* address)
+{
+	(void)state;
+	return grub_free (address);
+}
+
+static ZSTD_customMem grub_zstd_allocator (void)
+{
+	ZSTD_customMem allocator;
+	allocator.customAlloc = &grub_zstd_malloc;
+	allocator.customFree = &grub_zstd_free;
+	allocator.opaque = NULL;
+	return allocator;
+}
+
+static grub_ssize_t
+grub_btrfs_zstd_decompress (char *ibuf, grub_size_t isize, grub_off_t off,
+			    char *obuf, grub_size_t osize)
+{
+	void *allocated = NULL;
+	char *otmpbuf = obuf;
+	grub_size_t otmpsize = osize;
+	ZSTD_DCtx *dctx = NULL;
+	grub_size_t zstd_ret;
+	grub_ssize_t ret = -1;
+
+	/* Zstd will fail if it can't fit the entire output in the destination
+	 * buffer, so if osize isn't large enough, allocate a temporary buffer.
+	 */
+	if (otmpsize < ZSTD_BTRFS_MAX_INPUT) {
+		allocated = grub_malloc (ZSTD_BTRFS_MAX_INPUT);
+		if (!allocated) {
+			grub_dprintf ("zstd", "outtmpbuf allocation failed\n");
+			goto err;
+		}
+		otmpbuf = (char*)allocated;
+		otmpsize = ZSTD_BTRFS_MAX_INPUT;
+	}
+
+	/* Create the ZSTD_DCtx. */
+	dctx = ZSTD_createDCtx_advanced (grub_zstd_allocator ());
+	if (!dctx) {
+		grub_dprintf ("zstd", "failed to allocate the ZSTD_DCtx\n");
+		goto err;
+	}
+
+	/* Get the real input size, there may be junk at the
+	 * end of the frame.
+	 */
+	isize = ZSTD_findFrameCompressedSize (ibuf, isize);
+	if (ZSTD_isError (isize)) {
+		grub_dprintf ("zstd", "first frame is invalid: %s\n",
+			      ZSTD_getErrorName (isize));
+		goto err;
+	}
+
+	/* Decompress and check for errors */
+	zstd_ret = ZSTD_decompressDCtx (dctx, otmpbuf, otmpsize, ibuf, isize);
+	if (ZSTD_isError (zstd_ret)) {
+		grub_dprintf ("zstd", "zstd failed with message: %s\n",
+			      ZSTD_getErrorName (zstd_ret));
+		goto err;
+	}
+
+	/* Move the requested data into the obuf.
+	 * obuf may be equal to otmpbuf, which is why grub_memmove() is required.
+	 */
+	grub_memmove (obuf, otmpbuf + off, osize);
+	ret = osize;
+
+err:
+	grub_free (allocated);
+	ZSTD_freeDCtx (dctx);
+	return ret;
+}
+
 static grub_ssize_t
 grub_btrfs_lzo_decompress(char *ibuf, grub_size_t isize, grub_off_t off,
 			  char *obuf, grub_size_t osize)
@@ -1087,7 +1176,8 @@  grub_btrfs_extent_read (struct grub_btrfs_data *data,

       if (data->extent->compression != GRUB_BTRFS_COMPRESSION_NONE
 	  && data->extent->compression != GRUB_BTRFS_COMPRESSION_ZLIB
-	  && data->extent->compression != GRUB_BTRFS_COMPRESSION_LZO)
+	  && data->extent->compression != GRUB_BTRFS_COMPRESSION_LZO
+	  && data->extent->compression != GRUB_BTRFS_COMPRESSION_ZSTD)
 	{
 	  grub_error (GRUB_ERR_NOT_IMPLEMENTED_YET,
 		      "compression type 0x%x not supported",
@@ -1127,6 +1217,15 @@  grub_btrfs_extent_read (struct grub_btrfs_data *data,
 		  != (grub_ssize_t) csize)
 		return -1;
 	    }
+	  else if (data->extent->compression == GRUB_BTRFS_COMPRESSION_ZSTD)
+	    {
+	      if (grub_btrfs_zstd_decompress(data->extent->inl, data->extsize -
+					     ((grub_uint8_t *) data->extent->inl
+					      - (grub_uint8_t *) data->extent),
+					     extoff, buf, csize)
+		  != (grub_ssize_t) csize)
+		return -1;
+	    }
 	  else
 	    grub_memcpy (buf, data->extent->inl + extoff, csize);
 	  break;
@@ -1164,6 +1263,10 @@  grub_btrfs_extent_read (struct grub_btrfs_data *data,
 		ret = grub_btrfs_lzo_decompress (tmp, zsize, extoff
 				    + grub_le_to_cpu64 (data->extent->offset),
 				    buf, csize);
+	      else if (data->extent->compression == GRUB_BTRFS_COMPRESSION_ZSTD)
+		ret = grub_btrfs_zstd_decompress (tmp, zsize, extoff
+				    + grub_le_to_cpu64 (data->extent->offset),
+				    buf, csize);
 	      else
 		ret = -1;

diff --git a/grub-core/lib/zstd/up b/grub-core/lib/zstd/up
deleted file mode 120000
index 780297ae6..000000000
--- a/grub-core/lib/zstd/up
+++ /dev/null
@@ -1 +0,0 @@ 
-/Users/terrelln/Downloads/zstd-1.3.6/lib
\ No newline at end of file
diff --git a/tests/btrfs_test.in b/tests/btrfs_test.in
index 2b37ddd33..0c9bf3a68 100644
--- a/tests/btrfs_test.in
+++ b/tests/btrfs_test.in
@@ -18,6 +18,7 @@  fi
 "@builddir@/grub-fs-tester" btrfs
 "@builddir@/grub-fs-tester" btrfs_zlib
 "@builddir@/grub-fs-tester" btrfs_lzo
+"@builddir@/grub-fs-tester" btrfs_zstd
 "@builddir@/grub-fs-tester" btrfs_raid0
 "@builddir@/grub-fs-tester" btrfs_raid1
 "@builddir@/grub-fs-tester" btrfs_single
diff --git a/tests/util/grub-fs-tester.in b/tests/util/grub-fs-tester.in
index 15969d796..3470c89e3 100644
--- a/tests/util/grub-fs-tester.in
+++ b/tests/util/grub-fs-tester.in
@@ -626,7 +626,7 @@  for LOGSECSIZE in $(range "$MINLOGSECSIZE" "$MAXLOGSECSIZE" 1); do
 		    ;;
 		x"btrfs")
 		    "mkfs.btrfs" -s $SECSIZE -L "$FSLABEL" "${MOUNTDEVICE}" ;;
-		x"btrfs_zlib" | x"btrfs_lzo")
+		x"btrfs_zlib" | x"btrfs_lzo" | x"btrfs_zstd")
 		    "mkfs.btrfs" -s $SECSIZE -L "$FSLABEL" "${MOUNTDEVICE}"
 		    MOUNTOPTS="compress=${fs/btrfs_/},"
 		    MOUNTFS="btrfs"