[GRASS-SVN] r51482 - grass/trunk/lib/segment
svn_grass at osgeo.org
svn_grass at osgeo.org
Sun Apr 22 13:11:23 EDT 2012
Author: mmetz
Date: 2012-04-22 10:11:23 -0700 (Sun, 22 Apr 2012)
New Revision: 51482
Modified:
grass/trunk/lib/segment/address.c
grass/trunk/lib/segment/format.c
grass/trunk/lib/segment/get.c
grass/trunk/lib/segment/get_row.c
grass/trunk/lib/segment/init.c
grass/trunk/lib/segment/put.c
grass/trunk/lib/segment/put_row.c
grass/trunk/lib/segment/seek.c
grass/trunk/lib/segment/setup.c
Log:
segment lib: support for more than 2 billion cells
Modified: grass/trunk/lib/segment/address.c
===================================================================
--- grass/trunk/lib/segment/address.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/address.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -14,20 +14,47 @@
#include <grass/segment.h>
-int segment_address_fast(const SEGMENT * SEG, int row, int col, int *n,
+#define SEG_N_ROW_NONZERO(SEG, row, col) \
+ (((row) >> (SEG)->srowbits) * (SEG)->spr + ((col) >> (SEG)->scolbits))
+
+#define SEG_INDEX_ROW_NONZERO(SEG, row, col) \
+ ((((row) & ((SEG)->srows - 1)) << (SEG)->scolbits) + ((col) & ((SEG)->scols - 1)))
+
+#define SEG_N_ROW_ZERO(SEG, col) ((col) >> (SEG)->scolbits)
+
+#define SEG_INDEX_ROW_ZERO(SEG, col) ((col) & ((SEG)->scols - 1))
+
+#define INDEX_ADJ(SEG, i) \
+ ((SEG)->fast_seek ? ((i) << (SEG)->lenbits) : ((i) * (SEG)->len))
+
+int segment_address_fast(const SEGMENT * SEG, off_t row, off_t col, int *n,
int *index)
{
+
+#if 1
if (row) {
- int seg_r = row >> SEG->srowbits;
- int seg_c = col >> SEG->scolbits;
+ *n = SEG_N_ROW_NONZERO(SEG, row, col);
+ *index = INDEX_ADJ(SEG, SEG_INDEX_ROW_NONZERO(SEG, row, col));
+ }
+ /* for simple arrays */
+ else {
+ *n = SEG_N_ROW_ZERO(SEG, col);
+ *index = INDEX_ADJ(SEG, SEG_INDEX_ROW_ZERO(SEG, col));
+ }
+#else
+ if (row) {
+ *n = (row >> SEG->srowbits) * SEG->spr + (col >> SEG->scolbits);
+ *index = ((row & (SEG->srows - 1)) << SEG->scolbits) + (col & (SEG->scols - 1));
+ /* slower version for testing */
+ /*
+ off_t seg_r = row >> SEG->srowbits;
+ off_t seg_c = col >> SEG->scolbits;
+
*n = seg_r * SEG->spr + seg_c;
+
*index = ((row - (seg_r << SEG->srowbits)) << SEG->scolbits) +
col - (seg_c << SEG->scolbits);
-
- /*
- *n = (row >> SEG->srowbits) * SEG->spr + (col >> SEG->scolbits);
- *index = ((row & (SEG->srows - 1)) << SEG->scolbits) + (col & (SEG->scols - 1));
*/
}
/* for simple arrays */
@@ -35,20 +62,20 @@
*n = col >> SEG->scolbits;
*index = col - ((*n) << SEG->scolbits);
}
- if (!SEG->slow_seek)
- *index = *index << SEG->lenbits;
- else
- *index *= SEG->len;
+ *index = SEG->fast_seek ? (*index << SEG->lenbits) : (*index * SEG->len);
+
+#endif
+
return 0;
}
-int segment_address_slow(const SEGMENT * SEG, int row, int col, int *n,
+int segment_address_slow(const SEGMENT * SEG, off_t row, off_t col, int *n,
int *index)
{
if (row) {
- int seg_r = row / SEG->srows;
- int seg_c = col / SEG->scols;
+ off_t seg_r = row / SEG->srows;
+ off_t seg_c = col / SEG->scols;
*n = seg_r * SEG->spr + seg_c;
*index = (row - seg_r * SEG->srows) * SEG->scols + col -
@@ -77,7 +104,7 @@
* \return always returns 0
*/
-int segment_address(const SEGMENT * SEG, int row, int col, int *n, int *index)
+int segment_address(const SEGMENT * SEG, off_t row, off_t col, int *n, int *index)
{
/* old code
*n = row / SEG->srows * SEG->spr + col / SEG->scols;
Modified: grass/trunk/lib/segment/format.c
===================================================================
--- grass/trunk/lib/segment/format.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/format.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -22,8 +22,9 @@
#include <grass/segment.h>
-static int _segment_format(int, int, int, int, int, int, int);
+static int _segment_format(int, off_t, off_t, int, int, int, int);
static int write_int(int, int);
+static int write_off_t(int, off_t);
static int zero_fill(int, off_t);
/* fd must be open for write */
@@ -60,7 +61,7 @@
* \return -3 if illegal parameters are passed
*/
-int segment_format(int fd, int nrows, int ncols, int srows, int scols,
+int segment_format(int fd, off_t nrows, off_t ncols, int srows, int scols,
int len)
{
return _segment_format(fd, nrows, ncols, srows, scols, len, 1);
@@ -100,7 +101,7 @@
* \return -3 if illegal parameters are passed
*/
-int segment_format_nofill(int fd, int nrows, int ncols, int srows, int scols,
+int segment_format_nofill(int fd, off_t nrows, off_t ncols, int srows, int scols,
int len)
{
return _segment_format(fd, nrows, ncols, srows, scols, len, 0);
@@ -108,33 +109,35 @@
static int _segment_format(int fd,
- int nrows, int ncols,
+ off_t nrows, off_t ncols,
int srows, int scols, int len, int fill)
{
off_t nbytes;
int spr, size;
if (nrows <= 0 || ncols <= 0 || len <= 0 || srows <= 0 || scols <= 0) {
- G_warning("segment_format(fd,%d,%d,%d,%d,%d): illegal value(s)",
+ G_warning("segment_format(fd,%lld,%lld,%d,%d,%d): illegal value(s)",
nrows, ncols, srows, scols, len);
return -3;
}
+ spr = ncols / scols;
+ if (ncols % scols)
+ spr++;
+
+ size = srows * scols * len;
+
if (sizeof(off_t) == 4 && sizeof(double) >= 8) {
double d_size;
off_t o_size;
- spr = ncols / scols;
- if (ncols % scols)
- spr++;
-
/* calculate total number of segments */
d_size = (double) spr * ((nrows + srows - 1) / srows);
/* multiply with segment size */
- d_size *= srows * scols * len;
+ d_size *= size;
/* add header */
- d_size += 5 * sizeof(int);
+ d_size += 2 * sizeof(off_t) + 3 * sizeof(int);
o_size = (off_t) d_size;
@@ -152,7 +155,7 @@
return -1;
}
- if (!write_int(fd, nrows) || !write_int(fd, ncols)
+ if (!write_off_t(fd, nrows) || !write_off_t(fd, ncols)
|| !write_int(fd, srows) || !write_int(fd, scols)
|| !write_int(fd, len))
return -1;
@@ -160,12 +163,6 @@
if (!fill)
return 1;
- spr = ncols / scols;
- if (ncols % scols)
- spr++;
-
- size = srows * scols * len;
-
/* calculate total number of segments */
nbytes = spr * ((nrows + srows - 1) / srows);
nbytes *= size;
@@ -193,7 +190,16 @@
return 1;
}
+static int write_off_t(int fd, off_t n)
+{
+ if (write(fd, &n, sizeof(off_t)) != sizeof(off_t)) {
+ G_warning("segment_format(): Unable to write (%s)", strerror(errno));
+ return 0;
+ }
+ return 1;
+}
+
static int zero_fill(int fd, off_t nbytes)
{
#ifndef USE_LSEEK
Modified: grass/trunk/lib/segment/get.c
===================================================================
--- grass/trunk/lib/segment/get.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/get.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -38,7 +38,7 @@
* \return -1 if unable to seek or read segment file
*/
-int segment_get(SEGMENT * SEG, void *buf, int row, int col)
+int segment_get(SEGMENT * SEG, void *buf, off_t row, off_t col)
{
int index, n, i;
Modified: grass/trunk/lib/segment/get_row.c
===================================================================
--- grass/trunk/lib/segment/get_row.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/get_row.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -40,12 +40,12 @@
* \return -1 if unable to seek or read segment file
*/
-int segment_get_row(const SEGMENT * SEG, void *buf, int row)
+int segment_get_row(const SEGMENT * SEG, void *buf, off_t row)
{
int size;
- int ncols;
+ off_t ncols, col;
int scols;
- int n, index, col;
+ int n, index;
ncols = SEG->ncols - SEG->spill;
scols = SEG->scols;
@@ -53,8 +53,7 @@
for (col = 0; col < ncols; col += scols) {
SEG->segment_address(SEG, row, col, &n, &index);
- if (SEG->segment_seek(SEG, n, index) < 0)
- return -1;
+ SEG->segment_seek(SEG, n, index);
if (read(SEG->fd, buf, size) != size) {
G_warning("segment_get_row: %s", strerror(errno));
@@ -70,8 +69,7 @@
}
if ((size = SEG->spill * SEG->len)) {
SEG->segment_address(SEG, row, col, &n, &index);
- if (SEG->segment_seek(SEG, n, index) < 0)
- return -1;
+ SEG->segment_seek(SEG, n, index);
if (read(SEG->fd, buf, size) != size) {
G_warning("segment_get_row: %s", strerror(errno));
Modified: grass/trunk/lib/segment/init.c
===================================================================
--- grass/trunk/lib/segment/init.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/init.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -26,6 +26,7 @@
static int read_int(int, int *);
+static int read_off_t(int, off_t *);
/* fd must be open for read and write */
@@ -67,8 +68,8 @@
}
/* read the header */
- if (!read_int(fd, &SEG->nrows)
- || !read_int(fd, &SEG->ncols)
+ if (!read_off_t(fd, &SEG->nrows)
+ || !read_off_t(fd, &SEG->ncols)
|| !read_int(fd, &SEG->srows)
|| !read_int(fd, &SEG->scols)
|| !read_int(fd, &SEG->len))
@@ -89,3 +90,15 @@
return bytes_read;
}
+
+static int read_off_t(int fd, off_t *n)
+{
+ int bytes_read;
+
+ if ((bytes_read = read(fd, n, sizeof(off_t))) == -1)
+ G_warning("read_off_t: %s", strerror(errno));
+
+ bytes_read = (bytes_read == sizeof(off_t));
+
+ return bytes_read;
+}
Modified: grass/trunk/lib/segment/put.c
===================================================================
--- grass/trunk/lib/segment/put.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/put.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -42,7 +42,7 @@
* \return -1 if unable to seek or write segment file
*/
-int segment_put(SEGMENT * SEG, const void *buf, int row, int col)
+int segment_put(SEGMENT * SEG, const void *buf, off_t row, off_t col)
{
int index, n, i;
Modified: grass/trunk/lib/segment/put_row.c
===================================================================
--- grass/trunk/lib/segment/put_row.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/put_row.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -43,13 +43,14 @@
* \return -1 if unable to seek or write segment file
*/
-int segment_put_row(const SEGMENT * SEG, const void *buf, int row)
+int segment_put_row(const SEGMENT * SEG, const void *buf, off_t row)
{
int size;
- int ncols;
+ off_t ncols;
int scols;
- int n, index, col;
+ int n, index;
int result;
+ off_t col;
ncols = SEG->ncols - SEG->spill;
scols = SEG->scols;
@@ -58,12 +59,7 @@
for (col = 0; col < ncols; col += scols) {
SEG->segment_address(SEG, row, col, &n, &index);
- if (SEG->segment_seek(SEG, n, index) < 0) {
- G_warning
- ("Failed seek in segment file for index = %d n = %d at col:row %d:%d",
- index, n, col, row);
- return -1;
- }
+ SEG->segment_seek(SEG, n, index);
if ((result = write(SEG->fd, buf, size)) != size) {
G_warning("segment_put_row write error %s", strerror(errno));
@@ -81,12 +77,8 @@
if ((size = SEG->spill * SEG->len)) {
SEG->segment_address(SEG, row, col, &n, &index);
- if (SEG->segment_seek(SEG, n, index) < 0) {
- G_warning
- ("Failed seek in segment file for index = %d n = %d at col:row %d:%d",
- index, n, col, row);
- return -1;
- }
+ SEG->segment_seek(SEG, n, index);
+
if (write(SEG->fd, buf, size) != size) {
G_warning("segment_put_row final write error: %s",
strerror(errno));
Modified: grass/trunk/lib/segment/seek.c
===================================================================
--- grass/trunk/lib/segment/seek.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/seek.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -33,13 +33,17 @@
* \return -1 if unable to seek
*/
+#define SEG_SEEK_FAST(SEG, n, index) \
+ ((((off_t) (n)) << (SEG)->sizebits) + (index) + (SEG)->offset)
+
+#define SEG_SEEK_SLOW(SEG, n, index) \
+ ((off_t) (n) * (SEG)->size + (index) + (SEG)->offset)
+
int segment_seek_fast(const SEGMENT * SEG, int n, int index)
{
- off_t offset = (((off_t) n) << SEG->sizebits) + index + SEG->offset;
-
- if (lseek(SEG->fd, offset, SEEK_SET) == (off_t) - 1) {
- G_warning("segment_seek: %s", strerror(errno));
- return -1;
+ if (lseek((SEG)->fd, SEG_SEEK_FAST(SEG, n, index),
+ SEEK_SET) == (off_t) -1) {
+ G_fatal_error("segment_seek: %s", strerror(errno));
}
return 0;
@@ -47,11 +51,9 @@
int segment_seek_slow(const SEGMENT * SEG, int n, int index)
{
- off_t offset = (off_t) n * SEG->size + index + SEG->offset;
-
- if (lseek(SEG->fd, offset, SEEK_SET) == (off_t) - 1) {
- G_warning("segment_seek: %s", strerror(errno));
- return -1;
+ if (lseek((SEG)->fd, SEG_SEEK_SLOW(SEG, n, index),
+ SEEK_SET) == (off_t) -1) {
+ G_fatal_error("segment_seek: %s", strerror(errno));
}
return 0;
Modified: grass/trunk/lib/segment/setup.c
===================================================================
--- grass/trunk/lib/segment/setup.c 2012-04-22 17:11:04 UTC (rev 51481)
+++ grass/trunk/lib/segment/setup.c 2012-04-22 17:11:23 UTC (rev 51482)
@@ -56,7 +56,7 @@
SEG->spr++;
/* fast address */
- SEG->slow_adrs = 1;
+ SEG->fast_adrs = 0;
seg_exp = 0;
while (SEG->scols - (1 << seg_exp) > 0)
@@ -70,32 +70,32 @@
if (SEG->srows - (1 << seg_exp) == 0) {
SEG->srowbits = seg_exp;
SEG->segbits = SEG->srowbits + SEG->scolbits;
- SEG->slow_adrs = 0;
+ SEG->fast_adrs = 1;
G_debug(1, "segment_setup: fast address activated");
}
}
- if (SEG->slow_adrs)
+ if (SEG->fast_adrs)
+ SEG->segment_address = segment_address_fast;
+ else
SEG->segment_address = segment_address_slow;
- else
- SEG->segment_address = segment_address_fast;
/* fast seek */
- SEG->slow_seek = 1;
- if (SEG->slow_adrs == 0) {
+ SEG->fast_seek = 0;
+ if (SEG->fast_adrs == 1) {
seg_exp = 0;
while (SEG->len - (1 << seg_exp) > 0)
seg_exp++;
if (SEG->len - (1 << seg_exp) == 0) {
SEG->lenbits = seg_exp;
SEG->sizebits = SEG->segbits + SEG->lenbits;
- SEG->slow_seek = 0;
+ SEG->fast_seek = 1;
G_debug(1, "segment_setup: fast seek activated");
}
}
- if (SEG->slow_seek)
+ if (SEG->fast_seek)
+ SEG->segment_seek = segment_seek_fast;
+ else
SEG->segment_seek = segment_seek_slow;
- else
- SEG->segment_seek = segment_seek_fast;
/* adjust number of open segments if larger than number of total segments */
n_total_segs = SEG->spr * ((SEG->nrows + SEG->srows - 1) / SEG->srows);
More information about the grass-commit
mailing list