[postgis-tickets] r16785 - Avoid array overflow in ANALYZE (Closes #2985)

[Paul Ramsey]

Author: pramsey
Date: 2018-09-13 12:25:24 -0700 (Thu, 13 Sep 2018)
New Revision: 16785

Modified:
   trunk/postgis/gserialized_estimate.c
Log:
Avoid array overflow in ANALYZE (Closes #2985)


Modified: trunk/postgis/gserialized_estimate.c
===================================================================
--- trunk/postgis/gserialized_estimate.c	2018-09-13 19:25:10 UTC (rev 16784)
+++ trunk/postgis/gserialized_estimate.c	2018-09-13 19:25:24 UTC (rev 16785)
@@ -775,7 +775,7 @@
 	for ( d = 0; d < ndims; d++ )
 	{
 		/* Initialize counts for this dimension */
-		memset(counts, 0, sizeof(int)*NUM_BINS);
+		memset(counts, 0, sizeof(counts));
 
 		smin = extent->min[d];
 		smax = extent->max[d];
@@ -809,9 +809,13 @@
 			}
 
 			/* What bins does this range correspond to? */
-			bmin = NUM_BINS * (minoffset) / swidth;
-			bmax = NUM_BINS * (maxoffset) / swidth;
+			bmin = floor(NUM_BINS * minoffset / swidth);
+			bmax = floor(NUM_BINS * maxoffset / swidth);
 
+			/* Should only happen when maxoffset==swidth */
+			if (bmax >= NUM_BINS)
+				bmax = NUM_BINS-1;
+
 			POSTGIS_DEBUGF(4, " dimension %d, feature %d: bin %d to bin %d", d, i, bmin, bmax);
 
 			/* Increment the counts in all the bins this feature overlaps */