[GRASS-SVN] r44623 - grass-addons/grass7/raster/r.stream.extract
svn_grass at osgeo.org
svn_grass at osgeo.org
Thu Dec 16 13:23:25 EST 2010
Author: mmetz
Date: 2010-12-16 10:23:25 -0800 (Thu, 16 Dec 2010)
New Revision: 44623
Modified:
grass-addons/grass7/raster/r.stream.extract/local_proto.h
grass-addons/grass7/raster/r.stream.extract/main.c
grass-addons/grass7/raster/r.stream.extract/streams.c
Log:
updated memory distribution
Modified: grass-addons/grass7/raster/r.stream.extract/local_proto.h
===================================================================
--- grass-addons/grass7/raster/r.stream.extract/local_proto.h 2010-12-16 17:29:14 UTC (rev 44622)
+++ grass-addons/grass7/raster/r.stream.extract/local_proto.h 2010-12-16 18:23:25 UTC (rev 44623)
@@ -16,7 +16,7 @@
#define HEAP_PNT struct heap_point
HEAP_PNT {
- int added;
+ unsigned int added;
CELL ele;
POINT pnt;
};
Modified: grass-addons/grass7/raster/r.stream.extract/main.c
===================================================================
--- grass-addons/grass7/raster/r.stream.extract/main.c 2010-12-16 17:29:14 UTC (rev 44622)
+++ grass-addons/grass7/raster/r.stream.extract/main.c 2010-12-16 18:23:25 UTC (rev 44623)
@@ -65,6 +65,7 @@
int min_stream_length = 0, memory;
int seg_cols, seg_rows;
int num_open_segs, num_open_array_segs, num_seg_total;
+ double memory_divisor, heap_mem;
const char *mapset;
G_gisinit(argv[0]);
@@ -230,7 +231,6 @@
}
else
memory = 300;
- memory = 4000;
/* Check for some output map */
if ((output.stream_rast->answer == NULL)
@@ -282,26 +282,58 @@
* stream: 4 byte -> 16 KB / segment
* flag: 1 byte -> 4 KB / segment
*
- * Total MB / segment so far: <0.082
+ * Total MB / segment so far: 0.07
*
- * astar_points: 9 byte -> XX KB / segment
- * heap_points: 17 byte -> XX KB / segment
+ * astar_points: 8 byte -> 32 KB / segment
+ * heap_points: 16 byte -> 64 KB / segment
+ *
+ * Total MB / segment: 0.16
*/
-
- num_open_segs = memory / 0.082;
+
+ /* balance segment files */
+ /* elevation + accumulation: * 2 */
+ memory_divisor = 48 * 2;
+ /* aspect: as is */
+ memory_divisor += 4;
+ /* stream ids: / 2 */
+ memory_divisor += 16 / 2;
+ /* flags: * 4 */
+ memory_divisor += 4 * 4;
+ /* astar_points: / 16 */
+ /* ideally only a few but large segments */
+ memory_divisor += 32 / 16;
+ /* heap points: / 5 */
+ memory_divisor += 64. / 5.;
+
+ /* KB -> MB */
+ memory_divisor /= 1024.;
+
+ num_open_segs = memory / memory_divisor;
+ heap_mem = num_open_segs * 64. / (5. * 1024.);
num_seg_total = (ncols / seg_cols + 1) * (nrows / seg_rows + 1);
- if (num_open_segs > num_seg_total)
+ if (num_open_segs > num_seg_total) {
+ heap_mem += (num_open_segs - num_seg_total) * memory_divisor;
+ heap_mem -= (num_open_segs - num_seg_total) * 64. / (5. * 1024.);
num_open_segs = num_seg_total;
+ }
+ if (num_open_segs < 16) {
+ num_open_segs = 16;
+ heap_mem = num_open_segs * 64. / 5.;
+ }
G_verbose_message(_("%d of %d segments are kept in memory"),
num_open_segs, num_seg_total);
/* open segment files */
G_verbose_message(_("Create temporary files..."));
- seg_open(&watalt, nrows, ncols, seg_rows, seg_cols, num_open_segs,
+ seg_open(&watalt, nrows, ncols, seg_rows, seg_cols, num_open_segs * 2,
sizeof(WAT_ALT), 1);
- cseg_open(&stream, seg_rows, seg_cols, num_open_segs);
+ if (num_open_segs * 2 > num_seg_total)
+ heap_mem += (num_open_segs * 2 - num_seg_total) * 48. * 2. / 1024.;
+ cseg_open(&stream, seg_rows, seg_cols, num_open_segs / 2.);
bseg_open(&asp, seg_rows, seg_cols, num_open_segs);
- bseg_open(&bitflags, seg_rows, seg_cols, num_open_segs);
+ bseg_open(&bitflags, seg_rows, seg_cols, num_open_segs * 4);
+ if (num_open_segs * 4 > num_seg_total)
+ heap_mem += (num_open_segs * 4 - num_seg_total) * 4. * 4. / 1024.;
/* load maps */
if (load_maps(ele_fd, acc_fd) < 0)
@@ -310,52 +342,43 @@
G_fatal_error(_("No non-NULL cells in input map(s)"));
G_debug(1, "open segments for A* points");
- /* rounded down power of 2 */
- seg_cols = (int) (pow(2, (int)(log(num_open_segs / 8.0) / log(2) + 0.5)) + 0.5);
- if (seg_cols < 2)
- seg_cols = 2;
- num_open_array_segs = num_open_segs / seg_cols;
- if (num_open_array_segs == 0)
- num_open_array_segs = 1;
- /* n cols in segment */
- seg_cols *= seg_rows * seg_rows;
- /* n segments in row */
+ /* columns per segment */
+ seg_cols = seg_rows * seg_rows;
num_seg_total = n_points / seg_cols;
if (n_points % seg_cols > 0)
num_seg_total++;
/* no need to have more segments open than exist */
+ num_open_array_segs = num_open_segs / 16.;
if (num_open_array_segs > num_seg_total)
num_open_array_segs = num_seg_total;
-
- if (num_open_array_segs > 4)
- num_open_array_segs = 4;
+ if (num_open_array_segs < 1)
+ num_open_array_segs = 1;
- G_debug(0, "segment size for A* points: %d", seg_cols);
+ G_debug(1, "segment size for A* points: %d", seg_cols);
seg_open(&astar_pts, 1, n_points, 1, seg_cols, num_open_array_segs,
sizeof(POINT), 1);
/* one-based d-ary search_heap with astar_pts */
G_debug(1, "open segments for A* search heap");
- /* rounded down power of 2 */
- seg_cols = (int) (pow(2, (int)(log(num_open_segs / 8.0) / log(2) + 0.5)) + 0.5);
- if (seg_cols < 2)
- seg_cols = 2;
- num_open_array_segs = num_open_segs / seg_cols;
- if (num_open_array_segs == 0)
- num_open_array_segs = 1;
- /* n cols in segment */
- seg_cols *= seg_rows * seg_rows;
- /* n segments in row */
- num_seg_total = (n_points + 1) / seg_cols;
- if ((n_points + 1) % seg_cols > 0)
+
+ /* allowed memory for search heap in MB */
+ G_debug(1, "heap memory %.2f MB", heap_mem);
+ /* columns per segment */
+ /* larger is faster */
+ seg_cols = seg_rows * seg_rows * seg_rows;
+ num_seg_total = n_points / seg_cols;
+ if (n_points % seg_cols > 0)
num_seg_total++;
/* no need to have more segments open than exist */
+ num_open_array_segs = 1024. * 1024. * heap_mem / (seg_cols * 16.);
if (num_open_array_segs > num_seg_total)
num_open_array_segs = num_seg_total;
+ if (num_open_array_segs < 2)
+ num_open_array_segs = 2;
- G_debug(0, "A* search heap open segments %d, target 8, total %d",
+ G_debug(1, "A* search heap open segments %d, total %d",
num_open_array_segs, num_seg_total);
- G_debug(0, "segment size for heap points: %d", seg_cols);
+ G_debug(1, "segment size for heap points: %d", seg_cols);
/* the search heap will not hold more than 5% of all points at any given time ? */
/* chances are good that the heap will fit into one large segment */
seg_open(&search_heap, 1, n_points + 1, 1, seg_cols,
Modified: grass-addons/grass7/raster/r.stream.extract/streams.c
===================================================================
--- grass-addons/grass7/raster/r.stream.extract/streams.c 2010-12-16 17:29:14 UTC (rev 44622)
+++ grass-addons/grass7/raster/r.stream.extract/streams.c 2010-12-16 18:23:25 UTC (rev 44623)
@@ -483,7 +483,7 @@
/* can only happen with real depressions */
if (!have_depressions)
G_fatal_error(_("Bug in stream extraction"));
- G_debug(1, "bottom of real depression");
+ G_debug(2, "bottom of real depression");
}
continue;
}
@@ -698,7 +698,7 @@
if (r_max == r && c_max == c) {
/* can't continue stream, add outlet point
* r_max == r && c_max == c should not happen */
- G_debug(0, "can't continue stream at r %d c %d", r, c);
+ G_debug(1, "can't continue stream at r %d c %d", r, c);
if (n_outlets >= n_alloc_outlets) {
n_alloc_outlets += stream_node_step;
@@ -726,9 +726,9 @@
G_free(ele_nbr);
G_free(flag_nbr);
- G_debug(0, "%d outlets", n_outlets);
- G_debug(0, "%d nodes", n_stream_nodes);
- G_debug(0, "%d streams", stream_no);
+ G_debug(1, "%d outlets", n_outlets);
+ G_debug(1, "%d nodes", n_stream_nodes);
+ G_debug(1, "%d streams", stream_no);
return 1;
}
More information about the grass-commit
mailing list