[GRASS-SVN] r71890 - in grass/trunk: include/defs lib/gis
svn_grass at osgeo.org
svn_grass at osgeo.org
Mon Dec 4 10:17:17 PST 2017
Author: mmetz
Date: 2017-12-04 10:17:17 -0800 (Mon, 04 Dec 2017)
New Revision: 71890
Added:
grass/trunk/lib/gis/cmprzstd.c
Modified:
grass/trunk/include/defs/gis.h
grass/trunk/lib/gis/cmprbzip.c
grass/trunk/lib/gis/cmprlz4.c
grass/trunk/lib/gis/cmprrle.c
grass/trunk/lib/gis/cmprzlib.c
grass/trunk/lib/gis/compress.c
grass/trunk/lib/gis/compress.h
Log:
libgis: add ZSTD (Zstandard) compression
Modified: grass/trunk/include/defs/gis.h
===================================================================
--- grass/trunk/include/defs/gis.h 2017-12-04 18:15:57 UTC (rev 71889)
+++ grass/trunk/include/defs/gis.h 2017-12-04 18:17:17 UTC (rev 71890)
@@ -191,10 +191,12 @@
/* compress.c */
int G_compressor_number(char *);
char *G_compressor_name(int);
+int G_default_compressor(void);
int G_check_compressor(int);
int G_write_compressed(int, unsigned char *, int, int);
int G_write_unompressed(int, unsigned char *, int);
int G_read_compressed(int, int, unsigned char *, int, int);
+int G_compress_bound(int, int);
int G_compress(unsigned char *, int, unsigned char *, int, int);
int G_expand(unsigned char *, int, unsigned char *, int, int);
@@ -238,6 +240,14 @@
G_bz2_expand(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz);
+/* cmprzstd.c : ZSTD, compression similar to ZLIB's DEFLATE but faster */
+int
+G_zstd_compress(unsigned char *src, int src_sz, unsigned char *dst,
+ int dst_sz);
+int
+G_zstd_expand(unsigned char *src, int src_sz, unsigned char *dst,
+ int dst_sz);
+
/* add more compression methods here */
/* copy_dir.c */
Modified: grass/trunk/lib/gis/cmprbzip.c
===================================================================
--- grass/trunk/lib/gis/cmprbzip.c 2017-12-04 18:15:57 UTC (rev 71889)
+++ grass/trunk/lib/gis/cmprbzip.c 2017-12-04 18:17:17 UTC (rev 71890)
@@ -71,11 +71,26 @@
int
+G_bz2_compress_bound(int src_sz)
+{
+ /* from the documentation:
+ * To guarantee that the compressed data will fit in its buffer,
+ * allocate an output buffer of size 1% larger than the uncompressed data,
+ * plus six hundred extra bytes.
+ * bzip2 does not provide a compressbound fn
+ * and apparently does not have a fast version if destLen is
+ * large enough to hold a worst case result
+ */
+ return src_sz;
+}
+
+int
G_bz2_compress(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz)
{
int err;
- unsigned int i, nbytes, buf_sz;
+ int i, buf_sz;
+ unsigned int nbytes;
unsigned char *buf;
#ifndef HAVE_BZLIB_H
@@ -92,12 +107,17 @@
return 0;
/* Output buffer has to be 1% + 600 bytes bigger for single pass compression */
- buf_sz = (unsigned int)((double)dst_sz * 1.01 + (double)600);
+ buf = dst;
+ buf_sz = G_bz2_compress_bound(src_sz);
+ if (buf_sz > dst_sz) {
+ G_warning("G_bz2_compress(): programmer error, destination is too small");
+ if (NULL == (buf = (unsigned char *)
+ G_calloc(buf_sz, sizeof(unsigned char))))
+ return -1;
+ }
+ else
+ buf_sz = dst_sz;
- if (NULL == (buf = (unsigned char *)
- G_calloc(buf_sz, sizeof(unsigned char))))
- return -1;
-
/* Do single pass compression */
nbytes = buf_sz;
err = BZ2_bzBuffToBuffCompress((char *)buf, &nbytes, /* destination */
@@ -105,28 +125,29 @@
9, /* blockSize100k */
0, /* verbosity */
100); /* workFactor */
+
if (err != BZ_OK) {
- G_free(buf);
+ if (buf != dst)
+ G_free(buf);
return -1;
}
/* updated buf_sz is bytes of compressed data */
if (nbytes >= (unsigned int)src_sz) {
/* compression not possible */
- G_free(buf);
+ if (buf != dst)
+ G_free(buf);
return -2;
}
- /* dst too small */
- if ((unsigned int)dst_sz < nbytes)
- return -2;
+ if (buf != dst) {
+ /* Copy the data from buf to dst */
+ for (i = 0; i < nbytes; i++)
+ dst[i] = buf[i];
- /* Copy the data from buf to dst */
- for (i = 0; i < nbytes; i++)
- dst[i] = buf[i];
+ G_free(buf);
+ }
- G_free(buf);
-
return nbytes;
#endif
} /* G_bz2_compress() */
@@ -159,11 +180,15 @@
0, /* small */
0); /* verbosity */
+ if (err != BZ_OK) {
+ return -1;
+ }
+
/* Number of bytes inflated to output stream is
* updated buffer size
*/
- if (!(err == BZ_OK)) {
+ if (nbytes != dst_sz) {
return -1;
}
Modified: grass/trunk/lib/gis/cmprlz4.c
===================================================================
--- grass/trunk/lib/gis/cmprlz4.c 2017-12-04 18:15:57 UTC (rev 71889)
+++ grass/trunk/lib/gis/cmprlz4.c 2017-12-04 18:17:17 UTC (rev 71890)
@@ -26,7 +26,7 @@
* int src_sz, dst_sz; *
* unsigned char *src, *dst; *
* ---------------------------------------------------------------- *
- * This function is a wrapper around the LZ4 cimpression function. *
+ * This function is a wrapper around the LZ4 compression function. *
* It uses an all or nothing call. *
* If you need a continuous compression scheme, you'll have to code *
* your own. *
@@ -67,6 +67,14 @@
#include "lz4.h"
+int
+G_lz4_compress_bound(int src_sz)
+{
+ /* LZ4 has a fast version if destLen is large enough
+ * to hold a worst case result
+ */
+ return LZ4_compressBound(src_sz);
+}
int
G_lz4_compress(unsigned char *src, int src_sz, unsigned char *dst,
@@ -83,32 +91,43 @@
if (src_sz <= 0 || dst_sz <= 0)
return 0;
- /* Output buffer has to be larger for single pass compression */
- buf_sz = LZ4_compressBound(src_sz);
- if (NULL == (buf = (unsigned char *)
- G_calloc(buf_sz, sizeof(unsigned char))))
- return -1;
+ /* Output buffer should be large enough for single pass compression */
+ buf = dst;
+ buf_sz = G_lz4_compress_bound(src_sz);
+ if (buf_sz > dst_sz) {
+ G_warning("G_lz4_compress(): programmer error, destination is too small");
+ if (NULL == (buf = (unsigned char *)
+ G_calloc(buf_sz, sizeof(unsigned char))))
+ return -1;
+ }
+ else
+ buf_sz = dst_sz;
/* Do single pass compression */
err = LZ4_compress_default((char *)src, (char *)buf, src_sz, buf_sz);
+
if (err <= 0) {
- G_free(buf);
+ if (buf != dst)
+ G_free(buf);
return -1;
}
if (err >= src_sz) {
/* compression not possible */
- G_free(buf);
+ if (buf != dst)
+ G_free(buf);
return -2;
}
/* bytes of compressed data is return value */
nbytes = err;
- /* Copy the data from buf to dst */
- for (err = 0; err < nbytes; err++)
- dst[err] = buf[err];
+ if (buf != dst) {
+ /* Copy the data from buf to dst */
+ for (err = 0; err < nbytes; err++)
+ dst[err] = buf[err];
- G_free(buf);
+ G_free(buf);
+ }
return nbytes;
}
Modified: grass/trunk/lib/gis/cmprrle.c
===================================================================
--- grass/trunk/lib/gis/cmprrle.c 2017-12-04 18:15:57 UTC (rev 71889)
+++ grass/trunk/lib/gis/cmprrle.c 2017-12-04 18:17:17 UTC (rev 71890)
@@ -63,6 +63,12 @@
#include <grass/gis.h>
#include <grass/glocale.h>
+/* no fast mode if destination is large enough to hold
+ * worst case compression */
+int G_rle_compress_bound(int src_sz)
+{
+ return ((src_sz >> 1) * 3 + (src_sz & 1));
+}
int
G_rle_compress(unsigned char *src, int src_sz, unsigned char *dst,
@@ -188,5 +194,4 @@
return nbytes;
}
-
/* vim: set softtabstop=4 shiftwidth=4 expandtab: */
Modified: grass/trunk/lib/gis/cmprzlib.c
===================================================================
--- grass/trunk/lib/gis/cmprzlib.c 2017-12-04 18:15:57 UTC (rev 71889)
+++ grass/trunk/lib/gis/cmprzlib.c 2017-12-04 18:17:17 UTC (rev 71890)
@@ -74,24 +74,25 @@
#include "G.h"
-static void _init_zstruct(z_stream * z)
+
+int
+G_zlib_compress_bound(int src_sz)
{
- /* The types are defined in zlib.h, we set to NULL so zlib uses
- * its default functions.
+ /* from zlib.h:
+ * "when using compress or compress2,
+ * destLen must be at least the value returned by
+ * compressBound(sourceLen)"
+ * no explanation for the "must be"
*/
- z->zalloc = (alloc_func) 0;
- z->zfree = (free_func) 0;
- z->opaque = (voidpf) 0;
+ return compressBound(src_sz);
}
-
int
G_zlib_compress(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz)
{
- int err, nbytes, buf_sz;
+ uLong err, nbytes, buf_sz;
unsigned char *buf;
- z_stream c_stream;
/* Catch errors early */
if (src == NULL || dst == NULL)
@@ -103,66 +104,51 @@
/* Output buffer has to be 1% + 12 bytes bigger for single pass deflate */
/* buf_sz = (int)((double)dst_sz * 1.01 + (double)12); */
- buf_sz = compressBound(src_sz);
- if (NULL == (buf = (unsigned char *)
- G_calloc(buf_sz, sizeof(unsigned char))))
- return -1;
- /* Set-up for default zlib memory handling */
- _init_zstruct(&c_stream);
+ /* Output buffer should be large enough for single pass compression */
+ buf = dst;
+ buf_sz = G_zlib_compress_bound(src_sz);
+ if (buf_sz > dst_sz) {
+ G_warning("G_zlib_compress(): programmer error, destination is too small");
+ if (NULL == (buf = (unsigned char *)
+ G_calloc(buf_sz, sizeof(unsigned char))))
+ return -1;
+ }
+ else
+ buf_sz = dst_sz;
- /* Set-up the stream */
- c_stream.avail_in = src_sz;
- c_stream.next_in = (unsigned char *) src;
- c_stream.avail_out = buf_sz;
- c_stream.next_out = buf;
-
- /* Initialize */
/* Valid zlib compression levels -1 - 9 */
/* zlib default: Z_DEFAULT_COMPRESSION = -1, equivalent to 6
* as used here, 1 gives the best compromise between speed and compression */
- err = deflateInit(&c_stream, G__.compression_level);
- /* If there was an error initializing, return -1 */
+ /* Do single pass compression */
+ nbytes = buf_sz;
+ err = compress2((Bytef *)buf, &nbytes, /* destination */
+ (const Bytef *)src, src_sz, /* source */
+ G__.compression_level); /* level */
+
if (err != Z_OK) {
- G_free(buf);
+ if (buf != dst)
+ G_free(buf);
return -1;
}
- /* Do single pass compression */
- err = deflate(&c_stream, Z_FINISH);
- if (err != Z_STREAM_END) {
- switch (err) {
- case Z_OK: /* Destination too small */
- G_free(buf);
- deflateEnd(&c_stream);
- return -2;
- break;
- default: /* Give other error */
- G_free(buf);
- deflateEnd(&c_stream);
- return -1;
- break;
- }
- }
-
- /* avail_out is updated to bytes remaining in buf, so bytes of compressed
- * data is the original size minus that
- */
- nbytes = buf_sz - c_stream.avail_out;
+ /* updated buf_sz is bytes of compressed data */
if (nbytes >= src_sz) {
/* compression not possible */
- G_free(buf);
- deflateEnd(&c_stream);
+ if (buf != dst)
+ G_free(buf);
return -2;
}
- /* Copy the data from buf to dst */
- for (err = 0; err < nbytes; err++)
- dst[err] = buf[err];
- G_free(buf);
- deflateEnd(&c_stream);
+ if (buf != dst) {
+ /* Copy the data from buf to dst */
+ for (err = 0; err < nbytes; err++)
+ dst[err] = buf[err];
+ G_free(buf);
+ }
+
return nbytes;
} /* G_zlib_compress() */
@@ -171,8 +157,8 @@
G_zlib_expand(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz)
{
- int err, nbytes;
- z_stream c_stream;
+ int err;
+ uLong ss, nbytes;
/* Catch error condition */
if (src == NULL || dst == NULL)
@@ -182,45 +168,25 @@
if (src_sz <= 0 || dst_sz <= 0)
return 0;
- /* Set-up default zlib memory handling */
- _init_zstruct(&c_stream);
+ ss = src_sz;
- /* Set-up I/O streams */
- c_stream.avail_in = src_sz;
- c_stream.next_in = (unsigned char *)src;
- c_stream.avail_out = dst_sz;
- c_stream.next_out = dst;
+ /* Do single pass decompression */
+ nbytes = dst_sz;
+ err = uncompress((Bytef *)dst, &nbytes, /* destination */
+ (const Bytef *)src, ss); /* source */
- /* Call zlib initialization function */
- err = inflateInit(&c_stream);
-
/* If not Z_OK return error -1 */
if (err != Z_OK)
return -1;
- /* Do single pass inflate */
- err = inflate(&c_stream, Z_FINISH);
-
/* Number of bytes inflated to output stream is
- * original bytes available minus what avail_out now says
+ * updated buffer size
*/
- nbytes = dst_sz - c_stream.avail_out;
- /* Z_STREAM_END means all input was consumed,
- * Z_OK means only some was processed (not enough room in dst)
- */
- if (!(err == Z_STREAM_END || err == Z_OK)) {
- if (!(err == Z_BUF_ERROR && nbytes == dst_sz)) {
- inflateEnd(&c_stream);
- return -1;
- }
- /* Else, there was extra input, but requested output size was
- * decompressed successfully.
- */
+ if (nbytes != dst_sz) {
+ return -1;
}
- inflateEnd(&c_stream);
-
return nbytes;
} /* G_zlib_expand() */
Copied: grass/trunk/lib/gis/cmprzstd.c (from rev 71755, grass/trunk/lib/gis/cmprbzip.c)
===================================================================
--- grass/trunk/lib/gis/cmprzstd.c (rev 0)
+++ grass/trunk/lib/gis/cmprzstd.c 2017-12-04 18:17:17 UTC (rev 71890)
@@ -0,0 +1,184 @@
+/*
+ ****************************************************************************
+ * -- GRASS Development Team --
+ *
+ * MODULE: GRASS gis library
+ * FILENAME: cmprlz4.c
+ * AUTHOR(S): Eric G. Miller <egm2 at jps.net>
+ * Markus Metz
+ * PURPOSE: To provide an interface to lz4 for compressing and
+ * decompressing data using LZ$. It's primary use is in
+ * the storage and reading of GRASS floating point rasters.
+ *
+ * ALGORITHM: https://code.google.com/p/lz4/
+ * DATE CREATED: Dec 18 2015
+ * COPYRIGHT: (C) 2015 by the GRASS Development Team
+ *
+ * This program is free software under the GNU General Public
+ * License (version 2 or greater). Read the file COPYING that
+ * comes with GRASS for details.
+ *
+ *****************************************************************************/
+
+/********************************************************************
+ * int *
+ * G_zstd_compress (src, srz_sz, dst, dst_sz) *
+ * int src_sz, dst_sz; *
+ * unsigned char *src, *dst; *
+ * ---------------------------------------------------------------- *
+ * This function is a wrapper around the Zstd compression function. *
+ * It uses an all or nothing call. *
+ * If you need a continuous compression scheme, you'll have to code *
+ * your own. *
+ * In order to do a single pass compression, the input src must be *
+ * copied to a buffer larger than the data. This may cause *
+ * performance degradation. *
+ * *
+ * The function either returns the number of bytes of compressed *
+ * data in dst, or an error code. *
+ * *
+ * Errors include: *
+ * -1 -- Compression failed. *
+ * -2 -- dst is too small. *
+ * *
+ * ================================================================ *
+ * int *
+ * G_zstd_expand (src, src_sz, dst, dst_sz) *
+ * int src_sz, dst_sz; *
+ * unsigned char *src, *dst; *
+ * ---------------------------------------------------------------- *
+ * This function is a wrapper around the zstd decompression *
+ * function. It uses a single pass call. If you need a continuous *
+ * expansion scheme, you'll have to code your own. *
+ * *
+ * The function returns the number of bytes expanded into 'dst' or *
+ * and error code. *
+ * *
+ * Errors include: *
+ * -1 -- Expansion failed. *
+ * *
+ ********************************************************************
+ */
+
+#include <grass/config.h>
+
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
+#include <grass/gis.h>
+#include <grass/glocale.h>
+
+
+int
+G_zstd_compress_bound(int src_sz)
+{
+ /* ZSTD has a fast version if destLen is large enough
+ * to hold a worst case result
+ */
+#ifndef HAVE_ZSTD_H
+ G_fatal_error(_("GRASS needs to be compiled with ZSTD for ZSTD compression"));
+ return -1;
+#else
+ return ZSTD_compressBound(src_sz);
+#endif
+}
+
+int
+G_zstd_compress(unsigned char *src, int src_sz, unsigned char *dst,
+ int dst_sz)
+{
+ int err, nbytes, buf_sz;
+ unsigned char *buf;
+
+#ifndef HAVE_ZSTD_H
+ G_fatal_error(_("GRASS needs to be compiled with ZSTD for ZSTD compression"));
+ return -1;
+#else
+
+ /* Catch errors early */
+ if (src == NULL || dst == NULL)
+ return -1;
+
+ /* Don't do anything if either of these are true */
+ if (src_sz <= 0 || dst_sz <= 0)
+ return 0;
+
+ /* Output buffer has to be larger for single pass compression */
+ buf = dst;
+ buf_sz = G_zstd_compress_bound(src_sz);
+ if (buf_sz > dst_sz) {
+ G_warning("G_zstd_compress(): programmer error, destination is too small");
+ if (NULL == (buf = (unsigned char *)
+ G_calloc(buf_sz, sizeof(unsigned char))))
+ return -1;
+ }
+ else
+ buf_sz = dst_sz;
+
+ /* Do single pass compression */
+ err = ZSTD_compress((char *)buf, buf_sz, (char *)src, src_sz, 3);
+
+ if (err <= 0 || ZSTD_isError(err)) {
+ if (buf != dst)
+ G_free(buf);
+ return -1;
+ }
+ if (err >= src_sz) {
+ /* compression not possible */
+ if (buf != dst)
+ G_free(buf);
+ return -2;
+ }
+
+ /* bytes of compressed data is return value */
+ nbytes = err;
+
+ if (buf != dst) {
+ /* Copy the data from buf to dst */
+ for (err = 0; err < nbytes; err++)
+ dst[err] = buf[err];
+
+ G_free(buf);
+ }
+
+ return nbytes;
+#endif
+}
+
+int
+G_zstd_expand(unsigned char *src, int src_sz, unsigned char *dst,
+ int dst_sz)
+{
+ int err, nbytes;
+
+#ifndef HAVE_ZSTD_H
+ G_fatal_error(_("GRASS needs to be compiled with ZSTD for ZSTD compression"));
+ return -1;
+#else
+
+ /* Catch error condition */
+ if (src == NULL || dst == NULL)
+ return -2;
+
+ /* Don't do anything if either of these are true */
+ if (src_sz <= 0 || dst_sz <= 0)
+ return 0;
+
+ /* Do single pass decompress */
+ err = ZSTD_decompress((char *)dst, dst_sz, (char *)src, src_sz);
+ /* err = LZ4_decompress_fast(src, dst, src_sz); */
+
+ /* Number of bytes inflated to output stream is return value */
+ nbytes = err;
+
+ if (nbytes != dst_sz) {
+ return -1;
+ }
+
+ return nbytes;
+#endif
+}
+
+
+/* vim: set softtabstop=4 shiftwidth=4 expandtab: */
Modified: grass/trunk/lib/gis/compress.c
===================================================================
--- grass/trunk/lib/gis/compress.c 2017-12-04 18:15:57 UTC (rev 71889)
+++ grass/trunk/lib/gis/compress.c 2017-12-04 18:17:17 UTC (rev 71890)
@@ -24,6 +24,7 @@
* 2 : ZLIB's DEFLATE (good speed and compression) *
* 3 : LZ4 (fastest, low compression) *
* 4 : BZIP2 (slowest, high compression) *
+ * 5 : ZSTD (faster than ZLIB, higher compression than ZLIB) *
* *
* int *
* G_read_compressed (fd, rbytes, dst, nbytes, compression_type) *
@@ -121,6 +122,16 @@
return compressor[number].name;
}
+int G_default_compressor(void)
+{
+#ifdef HAVE_ZSTD_H
+ /* ZSTD */
+ return 5;
+#endif
+ /* ZLIB */
+ return 2;
+}
+
/* check compressor number
* return -1 on error
* return 0 known but not available
@@ -135,6 +146,11 @@
return compressor[number].available;
}
+int G_no_compress_bound(int src_sz)
+{
+ return src_sz;
+}
+
int
G_no_compress(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz)
@@ -179,6 +195,22 @@
return src_sz;
}
+/* G_*_compress_bound() returns an upper bound on the compressed size
+ * which can be larger than the input size
+ * some compressors are a bit faster if the size of the destination
+ * is at least the upper bound (no need to test for buffer overlflow)
+ * read comments on the specific compressor interfaces
+ */
+int G_compress_bound(int src_sz, int number)
+{
+ if (number < 0 || number >= n_compressors) {
+ G_fatal_error(_("Request for unsupported compressor"));
+ return -1;
+ }
+
+ return compressor[number].bound(src_sz);
+}
+
/* G_*_compress() returns
* > 0: number of bytes in dst
* 0: nothing done
@@ -214,7 +246,7 @@
}
int G_read_compressed(int fd, int rbytes, unsigned char *dst, int nbytes,
- int compressor)
+ int number)
{
int bsize, nread, err;
unsigned char *b;
@@ -262,7 +294,7 @@
/* Just call G_expand() with the buffer we read,
* Account for first byte being a flag
*/
- err = G_expand(b + 1, bsize - 1, dst, nbytes, compressor);
+ err = G_expand(b + 1, bsize - 1, dst, nbytes, number);
/* We're done with b */
G_free(b);
@@ -273,7 +305,7 @@
} /* G_read_compressed() */
int G_write_compressed(int fd, unsigned char *src, int nbytes,
- int compressor)
+ int number)
{
int dst_sz, nwritten, err;
unsigned char *dst, compressed;
@@ -282,19 +314,20 @@
if (src == NULL || nbytes < 0)
return -1;
- dst_sz = nbytes;
+ /* get upper bound of compressed size */
+ dst_sz = G_compress_bound(nbytes, number);
if (NULL == (dst = (unsigned char *)
G_calloc(dst_sz, sizeof(unsigned char))))
return -1;
/* Now just call G_compress() */
- err = G_compress(src, nbytes, dst, dst_sz, compressor);
+ err = G_compress(src, nbytes, dst, dst_sz, number);
/* If compression succeeded write compressed row,
* otherwise write uncompressed row. Compression will fail
* if dst is too small (i.e. compressed data is larger)
*/
- if (err > 0 && err <= dst_sz) {
+ if (err > 0 && err < nbytes) {
dst_sz = err;
/* Write the compression flag */
compressed = G_COMPRESSED_YES;
Modified: grass/trunk/lib/gis/compress.h
===================================================================
--- grass/trunk/lib/gis/compress.h 2017-12-04 18:15:57 UTC (rev 71889)
+++ grass/trunk/lib/gis/compress.h 2017-12-04 18:17:17 UTC (rev 71890)
@@ -7,25 +7,36 @@
* 2: ZLIB's DEFLATE (default)
* 3: LZ4, fastest but lowest compression ratio
* 4: BZIP2: slowest but highest compression ratio
+ * 5: ZSTD: faster than ZLIB, higher compression than ZLIB
*/
/* adding a new compressor:
* add the corresponding functions G_*compress() and G_*_expand()
* if needed, add checks to configure.in and include/config.in
- * modify compress.h
- * modify G_compress(), G_expand()
+ * modify compress.h (this file)
+ * nothing to change in compress.c
*/
+/* upper bounds of the size of the compressed buffer */
+int G_no_compress_bound(int);
+int G_rle_compress_bound(int);
+int G_zlib_compress_bound(int);
+int G_lz4_compress_bound(int);
+int G_bz2_compress_bound(int);
+int G_zstd_compress_bound(int);
+
typedef int compress_fn(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz);
typedef int expand_fn(unsigned char *src, int src_sz, unsigned char *dst,
int dst_sz);
+typedef int bound_fn(int src_sz);
struct compressor_list
{
int available;
compress_fn *compress;
expand_fn *expand;
+ bound_fn *bound;
char *name;
};
@@ -34,20 +45,27 @@
* 1: RLE
* 2: ZLIB
* 3: LZ4
- * 4: BZIP2 */
+ * 4: BZIP2
+ * 5: ZSTD
+ */
-static int n_compressors = 5;
+static int n_compressors = 6;
struct compressor_list compressor[] = {
- {1, G_no_compress, G_no_expand, "NONE"},
- {1, G_rle_compress, G_rle_expand, "RLE"},
- {1, G_zlib_compress, G_zlib_expand, "ZLIB"},
- {1, G_lz4_compress, G_lz4_expand, "LZ4"},
+ {1, G_no_compress, G_no_expand, G_no_compress_bound, "NONE"},
+ {1, G_rle_compress, G_rle_expand, G_rle_compress_bound, "RLE"},
+ {1, G_zlib_compress, G_zlib_expand, G_zlib_compress_bound, "ZLIB"},
+ {1, G_lz4_compress, G_lz4_expand, G_lz4_compress_bound, "LZ4"},
#ifdef HAVE_BZLIB_H
- {1, G_bz2_compress, G_bz2_expand, "BZIP2"},
+ {1, G_bz2_compress, G_bz2_expand, G_bz2_compress_bound, "BZIP2"},
#else
- {0, G_bz2_compress, G_bz2_expand, "BZIP2"},
+ {0, G_bz2_compress, G_bz2_expand, G_bz2_compress_bound, "BZIP2"},
#endif
- {0, NULL, NULL, NULL}
+#ifdef HAVE_ZSTD_H
+ {1, G_zstd_compress, G_zstd_expand, G_zstd_compress_bound, "ZSTD"},
+#else
+ {0, G_zstd_compress, G_zstd_expand, G_zstd_compress_bound, "ZSTD"},
+#endif
+ {0, NULL, NULL, NULL, NULL}
};
More information about the grass-commit
mailing list