| Message ID | 20181008230621.2185884-3-terrelln@fb.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | btrfs: Add zstd support to grub btrfs | expand |
On Mon, Oct 08, 2018 at 04:06:21PM -0700, Nick Terrell wrote: > Adds zstd support to the btrfs module. I'm not sure that my changes to the > Makefiles are correct, please let me know if I need to do something > differently. > > Tested on Ubuntu-18.04 with a btrfs /boot partition with and without zstd > compression. A test case was also added to the test suite that fails before > the patch, and passes after. > > Signed-off-by: Nick Terrell <terrelln@fb.com> > --- > v1 -> v2: > - Fix comments from Daniel Kiper > > Makefile.util.def | 10 +++- > grub-core/Makefile.core.def | 10 +++- > grub-core/fs/btrfs.c | 105 ++++++++++++++++++++++++++++++++++- > grub-core/lib/zstd/up | 1 - Please do not take this file into patch #1. > tests/btrfs_test.in | 1 + > tests/util/grub-fs-tester.in | 2 +- > 6 files changed, 124 insertions(+), 5 deletions(-) > delete mode 120000 grub-core/lib/zstd/up > > diff --git a/Makefile.util.def b/Makefile.util.def > index f9caccb97..81898c509 100644 > --- a/Makefile.util.def > +++ b/Makefile.util.def > @@ -54,7 +54,7 @@ library = { > library = { > name = libgrubmods.a; > cflags = '-fno-builtin -Wno-undef'; > - cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -DMINILZO_HAVE_CONFIG_H'; > + cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -I$(top_srcdir)/grub-core/lib/zstd -DMINILZO_HAVE_CONFIG_H'; s/top_srcdir/srcdir/? > common_nodist = grub_script.tab.c; > common_nodist = grub_script.yy.c; > @@ -164,6 +164,14 @@ library = { > common = grub-core/lib/xzembed/xz_dec_bcj.c; > common = grub-core/lib/xzembed/xz_dec_lzma2.c; > common = grub-core/lib/xzembed/xz_dec_stream.c; > + common = grub-core/lib/zstd/debug.c; > + common = grub-core/lib/zstd/entropy_common.c; > + common = grub-core/lib/zstd/error_private.c; > + common = grub-core/lib/zstd/fse_decompress.c; > + common = grub-core/lib/zstd/xxhash.c; > + common = grub-core/lib/zstd/zstd_common.c; > + common = grub-core/lib/zstd/huf_decompress.c; > + common = grub-core/lib/zstd/zstd_decompress.c; > }; > > program = { > diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def > index 2c1d62cee..2a97bb807 100644 > --- a/grub-core/Makefile.core.def > +++ b/grub-core/Makefile.core.def > @@ -1265,8 +1265,16 @@ module = { > name = btrfs; > common = fs/btrfs.c; > common = lib/crc.c; > + common = lib/zstd/debug.c; > + common = lib/zstd/entropy_common.c; > + common = lib/zstd/error_private.c; > + common = lib/zstd/fse_decompress.c; > + common = lib/zstd/xxhash.c; > + common = lib/zstd/zstd_common.c; > + common = lib/zstd/huf_decompress.c; > + common = lib/zstd/zstd_decompress.c; Could you sort the names of files here? > cflags = '$(CFLAGS_POSIX) -Wno-undef'; > - cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -DMINILZO_HAVE_CONFIG_H'; > + cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -I$(srcdir)/lib/zstd -DMINILZO_HAVE_CONFIG_H'; > }; > > module = { > diff --git a/grub-core/fs/btrfs.c b/grub-core/fs/btrfs.c > index 4849c1ceb..cfeb7995e 100644 > --- a/grub-core/fs/btrfs.c > +++ b/grub-core/fs/btrfs.c > @@ -17,6 +17,8 @@ > * along with GRUB. If not, see <http://www.gnu.org/licenses/>. > */ > > +#define ZSTD_STATIC_LINKING_ONLY > + > #include <grub/err.h> > #include <grub/file.h> > #include <grub/mm.h> > @@ -27,6 +29,7 @@ > #include <grub/lib/crc.h> > #include <grub/deflate.h> > #include <minilzo.h> > +#include <zstd.h> > #include <grub/i18n.h> > #include <grub/btrfs.h> > > @@ -45,6 +48,9 @@ GRUB_MOD_LICENSE ("GPLv3+"); > #define GRUB_BTRFS_LZO_BLOCK_MAX_CSIZE (GRUB_BTRFS_LZO_BLOCK_SIZE + \ > (GRUB_BTRFS_LZO_BLOCK_SIZE / 16) + 64 + 3) > > +#define ZSTD_BTRFS_MAX_WINDOWLOG 17 > +#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) > + > typedef grub_uint8_t grub_btrfs_checksum_t[0x20]; > typedef grub_uint16_t grub_btrfs_uuid_t[8]; > > @@ -212,6 +218,7 @@ struct grub_btrfs_extent_data > #define GRUB_BTRFS_COMPRESSION_NONE 0 > #define GRUB_BTRFS_COMPRESSION_ZLIB 1 > #define GRUB_BTRFS_COMPRESSION_LZO 2 > +#define GRUB_BTRFS_COMPRESSION_ZSTD 3 > > #define GRUB_BTRFS_OBJECT_ID_CHUNK 0x100 > > @@ -912,6 +919,88 @@ grub_btrfs_read_inode (struct grub_btrfs_data *data, > return grub_btrfs_read_logical (data, elemaddr, inode, sizeof (*inode), 0); > } > > +static void* grub_zstd_malloc (void* state, size_t size) > +{ > + (void)state; Do we need that? > + return grub_malloc (size); > +} > + > +static void grub_zstd_free (void* state, void* address) > +{ > + (void)state; Ditto? > + return grub_free (address); > +} > + > +static ZSTD_customMem grub_zstd_allocator (void) > +{ > + ZSTD_customMem allocator; Please add empty line here. > + allocator.customAlloc = &grub_zstd_malloc; > + allocator.customFree = &grub_zstd_free; > + allocator.opaque = NULL; And here. > + return allocator; > +} > + > +static grub_ssize_t > +grub_btrfs_zstd_decompress (char *ibuf, grub_size_t isize, grub_off_t off, > + char *obuf, grub_size_t osize) > +{ > + void *allocated = NULL; > + char *otmpbuf = obuf; > + grub_size_t otmpsize = osize; > + ZSTD_DCtx *dctx = NULL; > + grub_size_t zstd_ret; > + grub_ssize_t ret = -1; > + > + /* Zstd will fail if it can't fit the entire output in the destination > + * buffer, so if osize isn't large enough, allocate a temporary buffer. > + */ /* * Zstd will fail if it can't fit the entire output in the destination * buffer, so if osize isn't large enough, allocate a temporary buffer. */ > + if (otmpsize < ZSTD_BTRFS_MAX_INPUT) { > + allocated = grub_malloc (ZSTD_BTRFS_MAX_INPUT); > + if (!allocated) { > + grub_dprintf ("zstd", "outtmpbuf allocation failed\n"); > + goto err; > + } > + otmpbuf = (char*)allocated; > + otmpsize = ZSTD_BTRFS_MAX_INPUT; > + } > + > + /* Create the ZSTD_DCtx. */ > + dctx = ZSTD_createDCtx_advanced (grub_zstd_allocator ()); > + if (!dctx) { > + grub_dprintf ("zstd", "failed to allocate the ZSTD_DCtx\n"); Should not you set grub_errno here? > + goto err; > + } > + > + /* Get the real input size, there may be junk at the > + * end of the frame. > + */ Please reformat the comment like above. > + isize = ZSTD_findFrameCompressedSize (ibuf, isize); > + if (ZSTD_isError (isize)) { > + grub_dprintf ("zstd", "first frame is invalid: %s\n", > + ZSTD_getErrorName (isize)); grub_errno? > + goto err; > + } > + > + /* Decompress and check for errors */ This can stay as is. You can add a full stop at the end. > + zstd_ret = ZSTD_decompressDCtx (dctx, otmpbuf, otmpsize, ibuf, isize); > + if (ZSTD_isError (zstd_ret)) { > + grub_dprintf ("zstd", "zstd failed with message: %s\n", > + ZSTD_getErrorName (zstd_ret)); grub_errno? > + goto err; > + } > + > + /* Move the requested data into the obuf. > + * obuf may be equal to otmpbuf, which is why grub_memmove() is required. > + */ Begs for reformat. Daniel
> On Oct 9, 2018, at 12:07 PM, Daniel Kiper <dkiper@net-space.pl> wrote: > On Mon, Oct 08, 2018 at 04:06:21PM -0700, Nick Terrell wrote: >> Adds zstd support to the btrfs module. I'm not sure that my changes to the >> Makefiles are correct, please let me know if I need to do something >> differently. >> >> Tested on Ubuntu-18.04 with a btrfs /boot partition with and without zstd >> compression. A test case was also added to the test suite that fails before >> the patch, and passes after. >> >> Signed-off-by: Nick Terrell <terrelln@fb.com> >> --- >> v1 -> v2: >> - Fix comments from Daniel Kiper >> >> Makefile.util.def | 10 +++- >> grub-core/Makefile.core.def | 10 +++- >> grub-core/fs/btrfs.c | 105 ++++++++++++++++++++++++++++++++++- >> grub-core/lib/zstd/up | 1 - > > Please do not take this file into patch #1. Oops, sorry about that. >> >> >> +static void* grub_zstd_malloc (void* state, size_t size) >> +{ >> + (void)state; > > Do we need that? Yeah, otherwise my compiler (gcc-7.3) gives an unused parameter error. Thanks for the fast review! I'll have an updated version up later today. Nick
diff --git a/Makefile.util.def b/Makefile.util.def index f9caccb97..81898c509 100644 --- a/Makefile.util.def +++ b/Makefile.util.def @@ -54,7 +54,7 @@ library = { library = { name = libgrubmods.a; cflags = '-fno-builtin -Wno-undef'; - cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -DMINILZO_HAVE_CONFIG_H'; + cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo -I$(srcdir)/grub-core/lib/xzembed -I$(top_srcdir)/grub-core/lib/zstd -DMINILZO_HAVE_CONFIG_H'; common_nodist = grub_script.tab.c; common_nodist = grub_script.yy.c; @@ -164,6 +164,14 @@ library = { common = grub-core/lib/xzembed/xz_dec_bcj.c; common = grub-core/lib/xzembed/xz_dec_lzma2.c; common = grub-core/lib/xzembed/xz_dec_stream.c; + common = grub-core/lib/zstd/debug.c; + common = grub-core/lib/zstd/entropy_common.c; + common = grub-core/lib/zstd/error_private.c; + common = grub-core/lib/zstd/fse_decompress.c; + common = grub-core/lib/zstd/xxhash.c; + common = grub-core/lib/zstd/zstd_common.c; + common = grub-core/lib/zstd/huf_decompress.c; + common = grub-core/lib/zstd/zstd_decompress.c; }; program = { diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def index 2c1d62cee..2a97bb807 100644 --- a/grub-core/Makefile.core.def +++ b/grub-core/Makefile.core.def @@ -1265,8 +1265,16 @@ module = { name = btrfs; common = fs/btrfs.c; common = lib/crc.c; + common = lib/zstd/debug.c; + common = lib/zstd/entropy_common.c; + common = lib/zstd/error_private.c; + common = lib/zstd/fse_decompress.c; + common = lib/zstd/xxhash.c; + common = lib/zstd/zstd_common.c; + common = lib/zstd/huf_decompress.c; + common = lib/zstd/zstd_decompress.c; cflags = '$(CFLAGS_POSIX) -Wno-undef'; - cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -DMINILZO_HAVE_CONFIG_H'; + cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo -I$(srcdir)/lib/zstd -DMINILZO_HAVE_CONFIG_H'; }; module = { diff --git a/grub-core/fs/btrfs.c b/grub-core/fs/btrfs.c index 4849c1ceb..cfeb7995e 100644 --- a/grub-core/fs/btrfs.c +++ b/grub-core/fs/btrfs.c @@ -17,6 +17,8 @@ * along with GRUB. If not, see <http://www.gnu.org/licenses/>. */ +#define ZSTD_STATIC_LINKING_ONLY + #include <grub/err.h> #include <grub/file.h> #include <grub/mm.h> @@ -27,6 +29,7 @@ #include <grub/lib/crc.h> #include <grub/deflate.h> #include <minilzo.h> +#include <zstd.h> #include <grub/i18n.h> #include <grub/btrfs.h> @@ -45,6 +48,9 @@ GRUB_MOD_LICENSE ("GPLv3+"); #define GRUB_BTRFS_LZO_BLOCK_MAX_CSIZE (GRUB_BTRFS_LZO_BLOCK_SIZE + \ (GRUB_BTRFS_LZO_BLOCK_SIZE / 16) + 64 + 3) +#define ZSTD_BTRFS_MAX_WINDOWLOG 17 +#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) + typedef grub_uint8_t grub_btrfs_checksum_t[0x20]; typedef grub_uint16_t grub_btrfs_uuid_t[8]; @@ -212,6 +218,7 @@ struct grub_btrfs_extent_data #define GRUB_BTRFS_COMPRESSION_NONE 0 #define GRUB_BTRFS_COMPRESSION_ZLIB 1 #define GRUB_BTRFS_COMPRESSION_LZO 2 +#define GRUB_BTRFS_COMPRESSION_ZSTD 3 #define GRUB_BTRFS_OBJECT_ID_CHUNK 0x100 @@ -912,6 +919,88 @@ grub_btrfs_read_inode (struct grub_btrfs_data *data, return grub_btrfs_read_logical (data, elemaddr, inode, sizeof (*inode), 0); } +static void* grub_zstd_malloc (void* state, size_t size) +{ + (void)state; + return grub_malloc (size); +} + +static void grub_zstd_free (void* state, void* address) +{ + (void)state; + return grub_free (address); +} + +static ZSTD_customMem grub_zstd_allocator (void) +{ + ZSTD_customMem allocator; + allocator.customAlloc = &grub_zstd_malloc; + allocator.customFree = &grub_zstd_free; + allocator.opaque = NULL; + return allocator; +} + +static grub_ssize_t +grub_btrfs_zstd_decompress (char *ibuf, grub_size_t isize, grub_off_t off, + char *obuf, grub_size_t osize) +{ + void *allocated = NULL; + char *otmpbuf = obuf; + grub_size_t otmpsize = osize; + ZSTD_DCtx *dctx = NULL; + grub_size_t zstd_ret; + grub_ssize_t ret = -1; + + /* Zstd will fail if it can't fit the entire output in the destination + * buffer, so if osize isn't large enough, allocate a temporary buffer. + */ + if (otmpsize < ZSTD_BTRFS_MAX_INPUT) { + allocated = grub_malloc (ZSTD_BTRFS_MAX_INPUT); + if (!allocated) { + grub_dprintf ("zstd", "outtmpbuf allocation failed\n"); + goto err; + } + otmpbuf = (char*)allocated; + otmpsize = ZSTD_BTRFS_MAX_INPUT; + } + + /* Create the ZSTD_DCtx. */ + dctx = ZSTD_createDCtx_advanced (grub_zstd_allocator ()); + if (!dctx) { + grub_dprintf ("zstd", "failed to allocate the ZSTD_DCtx\n"); + goto err; + } + + /* Get the real input size, there may be junk at the + * end of the frame. + */ + isize = ZSTD_findFrameCompressedSize (ibuf, isize); + if (ZSTD_isError (isize)) { + grub_dprintf ("zstd", "first frame is invalid: %s\n", + ZSTD_getErrorName (isize)); + goto err; + } + + /* Decompress and check for errors */ + zstd_ret = ZSTD_decompressDCtx (dctx, otmpbuf, otmpsize, ibuf, isize); + if (ZSTD_isError (zstd_ret)) { + grub_dprintf ("zstd", "zstd failed with message: %s\n", + ZSTD_getErrorName (zstd_ret)); + goto err; + } + + /* Move the requested data into the obuf. + * obuf may be equal to otmpbuf, which is why grub_memmove() is required. + */ + grub_memmove (obuf, otmpbuf + off, osize); + ret = osize; + +err: + grub_free (allocated); + ZSTD_freeDCtx (dctx); + return ret; +} + static grub_ssize_t grub_btrfs_lzo_decompress(char *ibuf, grub_size_t isize, grub_off_t off, char *obuf, grub_size_t osize) @@ -1087,7 +1176,8 @@ grub_btrfs_extent_read (struct grub_btrfs_data *data, if (data->extent->compression != GRUB_BTRFS_COMPRESSION_NONE && data->extent->compression != GRUB_BTRFS_COMPRESSION_ZLIB - && data->extent->compression != GRUB_BTRFS_COMPRESSION_LZO) + && data->extent->compression != GRUB_BTRFS_COMPRESSION_LZO + && data->extent->compression != GRUB_BTRFS_COMPRESSION_ZSTD) { grub_error (GRUB_ERR_NOT_IMPLEMENTED_YET, "compression type 0x%x not supported", @@ -1127,6 +1217,15 @@ grub_btrfs_extent_read (struct grub_btrfs_data *data, != (grub_ssize_t) csize) return -1; } + else if (data->extent->compression == GRUB_BTRFS_COMPRESSION_ZSTD) + { + if (grub_btrfs_zstd_decompress(data->extent->inl, data->extsize - + ((grub_uint8_t *) data->extent->inl + - (grub_uint8_t *) data->extent), + extoff, buf, csize) + != (grub_ssize_t) csize) + return -1; + } else grub_memcpy (buf, data->extent->inl + extoff, csize); break; @@ -1164,6 +1263,10 @@ grub_btrfs_extent_read (struct grub_btrfs_data *data, ret = grub_btrfs_lzo_decompress (tmp, zsize, extoff + grub_le_to_cpu64 (data->extent->offset), buf, csize); + else if (data->extent->compression == GRUB_BTRFS_COMPRESSION_ZSTD) + ret = grub_btrfs_zstd_decompress (tmp, zsize, extoff + + grub_le_to_cpu64 (data->extent->offset), + buf, csize); else ret = -1; diff --git a/grub-core/lib/zstd/up b/grub-core/lib/zstd/up deleted file mode 120000 index 780297ae6..000000000 --- a/grub-core/lib/zstd/up +++ /dev/null @@ -1 +0,0 @@ -/Users/terrelln/Downloads/zstd-1.3.6/lib \ No newline at end of file diff --git a/tests/btrfs_test.in b/tests/btrfs_test.in index 2b37ddd33..0c9bf3a68 100644 --- a/tests/btrfs_test.in +++ b/tests/btrfs_test.in @@ -18,6 +18,7 @@ fi "@builddir@/grub-fs-tester" btrfs "@builddir@/grub-fs-tester" btrfs_zlib "@builddir@/grub-fs-tester" btrfs_lzo +"@builddir@/grub-fs-tester" btrfs_zstd "@builddir@/grub-fs-tester" btrfs_raid0 "@builddir@/grub-fs-tester" btrfs_raid1 "@builddir@/grub-fs-tester" btrfs_single diff --git a/tests/util/grub-fs-tester.in b/tests/util/grub-fs-tester.in index 15969d796..3470c89e3 100644 --- a/tests/util/grub-fs-tester.in +++ b/tests/util/grub-fs-tester.in @@ -626,7 +626,7 @@ for LOGSECSIZE in $(range "$MINLOGSECSIZE" "$MAXLOGSECSIZE" 1); do ;; x"btrfs") "mkfs.btrfs" -s $SECSIZE -L "$FSLABEL" "${MOUNTDEVICE}" ;; - x"btrfs_zlib" | x"btrfs_lzo") + x"btrfs_zlib" | x"btrfs_lzo" | x"btrfs_zstd") "mkfs.btrfs" -s $SECSIZE -L "$FSLABEL" "${MOUNTDEVICE}" MOUNTOPTS="compress=${fs/btrfs_/}," MOUNTFS="btrfs"
Adds zstd support to the btrfs module. I'm not sure that my changes to the Makefiles are correct, please let me know if I need to do something differently. Tested on Ubuntu-18.04 with a btrfs /boot partition with and without zstd compression. A test case was also added to the test suite that fails before the patch, and passes after. Signed-off-by: Nick Terrell <terrelln@fb.com> --- v1 -> v2: - Fix comments from Daniel Kiper Makefile.util.def | 10 +++- grub-core/Makefile.core.def | 10 +++- grub-core/fs/btrfs.c | 105 ++++++++++++++++++++++++++++++++++- grub-core/lib/zstd/up | 1 - tests/btrfs_test.in | 1 + tests/util/grub-fs-tester.in | 2 +- 6 files changed, 124 insertions(+), 5 deletions(-) delete mode 120000 grub-core/lib/zstd/up -- 2.17.1