[GRASS-SVN] r65272 - in grass/trunk: include/defs lib/raster
svn_grass at osgeo.org
svn_grass at osgeo.org
Mon May 18 12:41:19 PDT 2015
Author: glynn
Date: 2015-05-18 12:41:18 -0700 (Mon, 18 May 2015)
New Revision: 65272
Modified:
grass/trunk/include/defs/raster.h
grass/trunk/lib/raster/R.h
grass/trunk/lib/raster/close.c
grass/trunk/lib/raster/format.c
grass/trunk/lib/raster/get_row.c
grass/trunk/lib/raster/init.c
grass/trunk/lib/raster/open.c
grass/trunk/lib/raster/put_row.c
Log:
Support compressed null bitmap (issue #2349)
Modified: grass/trunk/include/defs/raster.h
===================================================================
--- grass/trunk/include/defs/raster.h 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/include/defs/raster.h 2015-05-18 19:41:18 UTC (rev 65272)
@@ -266,7 +266,9 @@
/* format.c */
int Rast__check_format(int);
int Rast__read_row_ptrs(int);
+int Rast__read_null_row_ptrs(int, int);
int Rast__write_row_ptrs(int);
+int Rast__write_null_row_ptrs(int, int);
/* fpreclass.c */
void Rast_fpreclass_clear(struct FPReclass *);
Modified: grass/trunk/lib/raster/R.h
===================================================================
--- grass/trunk/lib/raster/R.h 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/lib/raster/R.h 2015-05-18 19:41:18 UTC (rev 65272)
@@ -67,6 +67,7 @@
struct Quant quant;
struct GDAL_link *gdal;
int data_fd; /* Raster data fd */
+ off_t *null_row_ptr; /* Null file row addresses */
};
struct R__ /* Structure of library globals */
@@ -77,6 +78,7 @@
int want_histogram;
int nbytes;
int compression_type;
+ int compress_nulls;
int window_set; /* Flag: window set? */
int split_window; /* Separate windows for input and output */
struct Cell_head rd_window; /* Window used for input */
Modified: grass/trunk/lib/raster/close.c
===================================================================
--- grass/trunk/lib/raster/close.c 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/lib/raster/close.c 2015-05-18 19:41:18 UTC (rev 65272)
@@ -33,6 +33,7 @@
#define FORMAT_FILE "f_format"
#define QUANT_FILE "f_quant"
#define NULL_FILE "null"
+#define NULL2_FILE "null2"
static int close_old(int);
static int close_new(int, int);
@@ -278,6 +279,9 @@
G_file_name_misc(path, "cell_misc", NULL_FILE, fcb->name,
G_mapset());
remove(path);
+ G_file_name_misc(path, "cell_misc", NULL2_FILE, fcb->name,
+ G_mapset());
+ remove(path);
/* write 0-length cell file */
G_make_mapset_element("cell");
@@ -364,10 +368,15 @@
/* create path : full null file name */
G__make_mapset_element_misc("cell_misc", fcb->name);
- G_file_name_misc(path, "cell_misc", NULL_FILE, fcb->name,
- G_mapset());
+ G_file_name_misc(path, "cell_misc", NULL_FILE, fcb->name, G_mapset());
remove(path);
+ G_file_name_misc(path, "cell_misc", NULL2_FILE, fcb->name, G_mapset());
+ remove(path);
+ G_file_name_misc(path, "cell_misc",
+ fcb->null_row_ptr ? NULL2_FILE : NULL_FILE,
+ fcb->name, G_mapset());
+
if (fcb->null_cur_row > 0) {
/* if temporary NULL file exists, write it into cell_misc/name/null */
if (rename(fcb->null_temp_name, path)) {
@@ -390,6 +399,11 @@
Rast__write_row_ptrs(fd);
}
+ if (fcb->null_row_ptr) { /* compressed nulls */
+ fcb->null_row_ptr[fcb->cellhd.rows] = lseek(fcb->null_fd, 0L, SEEK_CUR);
+ Rast__write_null_row_ptrs(fd, fcb->null_fd);
+ }
+
if (fcb->map_type != CELL_TYPE) { /* floating point map */
int cell_fd;
Modified: grass/trunk/lib/raster/format.c
===================================================================
--- grass/trunk/lib/raster/format.c 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/lib/raster/format.c 2015-05-18 19:41:18 UTC (rev 65272)
@@ -91,10 +91,8 @@
return Rast__read_row_ptrs(fd);
}
-int Rast__read_row_ptrs(int fd)
+static int read_row_ptrs(int nrows, int old, off_t *row_ptr, int fd)
{
- struct fileinfo *fcb = &R__.fileinfo[fd];
- int nrows = fcb->cellhd.rows;
unsigned char nbytes;
unsigned char *buf, *b;
int n;
@@ -105,9 +103,9 @@
* (this makes them machine dependent)
*/
- if (fcb->cellhd.compressed < 0) {
+ if (old) {
n = (nrows + 1) * sizeof(off_t);
- if (read(fcb->data_fd, fcb->row_ptr, n) != n)
+ if (read(fd, row_ptr, n) != n)
goto badread;
return 1;
}
@@ -119,14 +117,14 @@
* actual values do not exceed the capability of the off_t)
*/
- if (read(fcb->data_fd, &nbytes, 1) != 1)
+ if (read(fd, &nbytes, 1) != 1)
goto badread;
if (nbytes == 0)
goto badread;
n = (nrows + 1) * nbytes;
buf = G_malloc(n);
- if (read(fcb->data_fd, buf, n) != n)
+ if (read(fd, buf, n) != n)
goto badread;
for (row = 0, b = buf; row <= nrows; row++) {
@@ -143,7 +141,7 @@
v += c;
}
- fcb->row_ptr[row] = v;
+ row_ptr[row] = v;
}
G_free(buf);
@@ -151,27 +149,52 @@
return 1;
badread:
- G_warning(_("Fail of initial read of compressed file [%s in %s]"),
- fcb->name, fcb->mapset);
return -1;
}
-int Rast__write_row_ptrs(int fd)
+int Rast__read_row_ptrs(int fd)
{
struct fileinfo *fcb = &R__.fileinfo[fd];
int nrows = fcb->cellhd.rows;
+ int old = fcb->cellhd.compressed < 0;
+
+ if (read_row_ptrs(nrows, old, fcb->row_ptr, fcb->data_fd) < 0) {
+ G_warning(_("Fail of initial read of compressed file [%s in %s]"),
+ fcb->name, fcb->mapset);
+ return -1;
+ }
+
+ return 1;
+}
+
+int Rast__read_null_row_ptrs(int fd, int null_fd)
+{
+ struct fileinfo *fcb = &R__.fileinfo[fd];
+ int nrows = fcb->cellhd.rows;
+
+ if (read_row_ptrs(nrows, 0, fcb->null_row_ptr, null_fd) < 0) {
+ G_warning(_("Fail of initial read of compressed null file [%s in %s]"),
+ fcb->name, fcb->mapset);
+ return -1;
+ }
+
+ return 1;
+}
+
+static int write_row_ptrs(int nrows, off_t *row_ptr, int fd)
+{
int nbytes = sizeof(off_t);
unsigned char *buf, *b;
int len, row, result;
- lseek(fcb->data_fd, 0L, SEEK_SET);
+ lseek(fd, 0L, SEEK_SET);
len = (nrows + 1) * nbytes + 1;
b = buf = G_malloc(len);
*b++ = nbytes;
for (row = 0; row <= nrows; row++) {
- off_t v = fcb->row_ptr[row];
+ off_t v = row_ptr[row];
int i;
for (i = nbytes - 1; i >= 0; i--) {
@@ -182,8 +205,24 @@
b += nbytes;
}
- result = (write(fcb->data_fd, buf, len) == len);
+ result = (write(fd, buf, len) == len);
G_free(buf);
return result;
}
+
+int Rast__write_row_ptrs(int fd)
+{
+ struct fileinfo *fcb = &R__.fileinfo[fd];
+ int nrows = fcb->cellhd.rows;
+
+ return write_row_ptrs(nrows, fcb->row_ptr, fcb->data_fd);
+}
+
+int Rast__write_null_row_ptrs(int fd, int null_fd)
+{
+ struct fileinfo *fcb = &R__.fileinfo[fd];
+ int nrows = fcb->cellhd.rows;
+
+ return write_row_ptrs(nrows, fcb->null_row_ptr, null_fd);
+}
Modified: grass/trunk/lib/raster/get_row.c
===================================================================
--- grass/trunk/lib/raster/get_row.c 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/lib/raster/get_row.c 2015-05-18 19:41:18 UTC (rev 65272)
@@ -804,6 +804,40 @@
Rast_get_row(fd, buf, row, DCELL_TYPE);
}
+static int read_null_bits_compressed(int null_fd, unsigned char *flags,
+ int row, size_t size, int fd)
+{
+ struct fileinfo *fcb = &R__.fileinfo[fd];
+ off_t t1 = fcb->null_row_ptr[row];
+ off_t t2 = fcb->null_row_ptr[row + 1];
+ size_t readamount = t2 - t1;
+ unsigned char *compressed_buf;
+
+ if (lseek(null_fd, t1, SEEK_SET) < 0)
+ G_fatal_error(_("Error reading null data for row %d of <%s>"),
+ row, fcb->name);
+
+ if (readamount == size) {
+ if (read(null_fd, flags, size) != size)
+ G_fatal_error(_("Error reading null data for row %d of <%s>"),
+ row, fcb->name);
+ }
+
+ compressed_buf = G_alloca(readamount);
+
+ if (read(null_fd, compressed_buf, readamount) != readamount) {
+ G_freea(compressed_buf);
+ G_fatal_error(_("Error reading null data for row %d of <%s>"),
+ row, fcb->name);
+ }
+
+ G_zlib_expand(compressed_buf, readamount, flags, size);
+
+ G_freea(compressed_buf);
+
+ return 1;
+}
+
static int read_null_bits(int fd, int row)
{
struct fileinfo *fcb = &R__.fileinfo[fd];
@@ -823,6 +857,10 @@
return 0;
size = Rast__null_bitstream_size(cols);
+
+ if (fcb->null_row_ptr)
+ return read_null_bits_compressed(null_fd, flags, R, size, fd);
+
offset = (off_t) size * R;
if (lseek(null_fd, offset, SEEK_SET) < 0)
Modified: grass/trunk/lib/raster/init.c
===================================================================
--- grass/trunk/lib/raster/init.c 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/lib/raster/init.c 2015-05-18 19:41:18 UTC (rev 65272)
@@ -77,7 +77,7 @@
static int init(void)
{
- char *zlib;
+ char *zlib, *nulls;
Rast__init_window();
@@ -96,6 +96,9 @@
zlib = getenv("GRASS_INT_ZLIB");
R__.compression_type = (!zlib || atoi(zlib)) ? 2 : 1;
+ nulls = getenv("GRASS_COMPRESS_NULLS");
+ R__.compress_nulls = (nulls && atoi(nulls)) ? 1 : 0;
+
G_add_error_handler(Rast__error_handler, NULL);
initialized = 1;
Modified: grass/trunk/lib/raster/open.c
===================================================================
--- grass/trunk/lib/raster/open.c 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/lib/raster/open.c 2015-05-18 19:41:18 UTC (rev 65272)
@@ -28,6 +28,7 @@
#include "R.h"
#define FORMAT_FILE "f_format"
#define NULL_FILE "null"
+#define NULL2_FILE "null2"
static int new_fileinfo(void)
{
@@ -328,6 +329,7 @@
fcb->io_error = 0;
fcb->map_type = MAP_TYPE;
fcb->nbytes = MAP_NBYTES;
+ fcb->null_row_ptr = NULL;
if (!gdal) {
if (!G_find_file2_misc("cell_misc", NULL_FILE, r_name, r_mapset)) {
@@ -335,7 +337,19 @@
fcb->null_file_exists = 0;
}
else {
- fcb->null_fd = G_open_old_misc("cell_misc", NULL_FILE, r_name, r_mapset);
+ /* First, check for compressed null file */
+ fcb->null_fd = G_open_old_misc("cell_misc", NULL2_FILE, r_name, r_mapset);
+ if (fcb->null_fd >= 0) {
+ fcb->null_row_ptr = G_calloc(fcb->cellhd.rows + 1, sizeof(off_t));
+ if (Rast__read_null_row_ptrs(fd, fcb->null_fd) < 0) {
+ close(fcb->null_fd);
+ fcb->null_fd = -1;
+ G_free(fcb->null_row_ptr);
+ fcb->null_row_ptr = NULL;
+ }
+ }
+ if (fcb->null_fd < 0)
+ fcb->null_fd = G_open_old_misc("cell_misc", NULL_FILE, r_name, r_mapset);
fcb->null_file_exists = fcb->null_fd >= 0;
}
}
@@ -514,6 +528,7 @@
fcb->null_cur_row = 0;
fcb->null_bits = NULL;
fcb->null_fd = -1;
+ fcb->null_row_ptr = NULL;
if (fcb->map_type != CELL_TYPE)
Rast_quant_init(&(fcb->quant));
@@ -669,6 +684,12 @@
fcb->null_temp_name = tempname;
+ if (R__.compress_nulls) {
+ fcb->null_row_ptr = G_calloc(fcb->cellhd.rows + 1, sizeof(off_t));
+ G_zero(fcb->row_ptr, (fcb->cellhd.rows + 1) * sizeof(off_t));
+ Rast__write_null_row_ptrs(fd, fcb->null_fd);
+ }
+
/* next row to be written (in order) is zero */
fcb->null_cur_row = 0;
Modified: grass/trunk/lib/raster/put_row.c
===================================================================
--- grass/trunk/lib/raster/put_row.c 2015-05-18 19:36:53 UTC (rev 65271)
+++ grass/trunk/lib/raster/put_row.c 2015-05-18 19:41:18 UTC (rev 65272)
@@ -531,6 +531,33 @@
return null_fd;
}
+static void write_null_bits_compressed(int null_fd, const unsigned char *flags,
+ int row, size_t size, int fd)
+{
+ struct fileinfo *fcb = &R__.fileinfo[fd];
+ unsigned char *compressed_buf;
+ ssize_t nwrite;
+
+ fcb->null_row_ptr[row] = lseek(null_fd, 0L, SEEK_CUR);
+
+ compressed_buf = G_alloca(size + 1);
+
+ nwrite = G_zlib_compress(flags, size, compressed_buf, size);
+
+ if (nwrite < size) {
+ if (write(null_fd, compressed_buf, nwrite) != nwrite)
+ G_fatal_error(_("Error writing compressed null data for row %d of <%s>"),
+ row, fcb->name);
+ }
+ else {
+ if (write(null_fd, flags, size) != size)
+ G_fatal_error(_("Error writing compressed null data for row %d of <%s>"),
+ row, fcb->name);
+ }
+
+ G_freea(compressed_buf);
+}
+
/*!
\brief Write null data
@@ -550,6 +577,12 @@
size_t size;
size = Rast__null_bitstream_size(cols);
+
+ if (fcb->null_row_ptr) {
+ write_null_bits_compressed(null_fd, flags, row, size, fd);
+ return;
+ }
+
offset = (off_t) size *row;
if (lseek(null_fd, offset, SEEK_SET) < 0)
More information about the grass-commit
mailing list