[postgis-tickets] r16784 - Avoid array overflow in ANALYZE (References #2985)

[Paul Ramsey]

Author: pramsey
Date: 2018-09-13 12:25:10 -0700 (Thu, 13 Sep 2018)
New Revision: 16784

Modified:
   branches/2.5/postgis/gserialized_estimate.c
Log:
Avoid array overflow in ANALYZE (References #2985)


Modified: branches/2.5/postgis/gserialized_estimate.c
===================================================================
--- branches/2.5/postgis/gserialized_estimate.c	2018-09-13 19:25:01 UTC (rev 16783)
+++ branches/2.5/postgis/gserialized_estimate.c	2018-09-13 19:25:10 UTC (rev 16784)
@@ -776,7 +776,7 @@
 	for ( d = 0; d < ndims; d++ )
 	{
 		/* Initialize counts for this dimension */
-		memset(counts, 0, sizeof(int)*NUM_BINS);
+		memset(counts, 0, sizeof(counts));
 
 		smin = extent->min[d];
 		smax = extent->max[d];
@@ -802,7 +802,7 @@
 			minoffset = ndb->min[d] - smin;
 			maxoffset = ndb->max[d] - smin;
 
-			/* Skip boxes that our outside our working range */
+			/* Skip boxes that are outside our working range */
 			if ( minoffset < 0 || minoffset > swidth ||
 			     maxoffset < 0 || maxoffset > swidth )
 			{
@@ -810,9 +810,12 @@
 			}
 
 			/* What bins does this range correspond to? */
-			bmin = NUM_BINS * (minoffset) / swidth;
-			bmax = NUM_BINS * (maxoffset) / swidth;
+			bmin = floor(NUM_BINS * minoffset / swidth);
+			bmax = floor(NUM_BINS * maxoffset / swidth);
 
+			/* Should only happen when maxoffset==swidth */
+			bmax = bmax >= NUM_BINS ? NUM_BINS-1 : bmax;
+
 			POSTGIS_DEBUGF(4, " dimension %d, feature %d: bin %d to bin %d", d, i, bmin, bmax);
 
 			/* Increment the counts in all the bins this feature overlaps */