[postgis-tickets] r17020 - Avoid undefined behaviour in gserialized_estimate

[Raul]

Author: algunenano
Date: 2018-11-15 08:07:47 -0800 (Thu, 15 Nov 2018)
New Revision: 17020

Modified:
   branches/2.4/NEWS
   branches/2.4/postgis/gserialized_estimate.c
Log:
Avoid undefined behaviour in gserialized_estimate

References #4190



Modified: branches/2.4/NEWS
===================================================================
--- branches/2.4/NEWS	2018-11-15 09:08:17 UTC (rev 17019)
+++ branches/2.4/NEWS	2018-11-15 16:07:47 UTC (rev 17020)
@@ -17,6 +17,7 @@
   - #4136, Proper repeated point removal on small polygons (Paul Ramsey)
   - #4223, ST_DistanceTree error for near parallel boxes (Paul Ramsey)
   - #4326, Allocate enough memory in gidx_to_string (Raúl Marín)
+  - #4190, Avoid undefined behaviour in gserialized_estimate
 
 
 PostGIS 2.4.5

Modified: branches/2.4/postgis/gserialized_estimate.c
===================================================================
--- branches/2.4/postgis/gserialized_estimate.c	2018-11-15 09:08:17 UTC (rev 17019)
+++ branches/2.4/postgis/gserialized_estimate.c	2018-11-15 16:07:47 UTC (rev 17020)
@@ -610,7 +610,9 @@
 	for ( d = 0; d < ND_DIMS; d++ )
 	{
 		size = nd_box->max[d] - nd_box->min[d];
-		if ( size <= 0 ) continue;
+		/* Avoid expanding boxes that are either too wide or too narrow*/
+		if (size < MIN_DIMENSION_WIDTH || size > MAX_DIMENSION_WIDTH)
+			continue;
 		nd_box->min[d] -= size * expansion_factor / 2;
 		nd_box->max[d] += size * expansion_factor / 2;
 	}
@@ -637,18 +639,26 @@
 		double smin = nd_stats->extent.min[d];
 		double smax = nd_stats->extent.max[d];
 		double width = smax - smin;
-		int size = roundf(nd_stats->size[d]);
 
-		/* ... find cells the box overlaps with in this dimension */
-		nd_ibox->min[d] = floor(size * (nd_box->min[d] - smin) / width);
-		nd_ibox->max[d] = floor(size * (nd_box->max[d] - smin) / width);
+		if (width < MIN_DIMENSION_WIDTH)
+		{
+			nd_ibox->min[d] = nd_ibox->max[d] = nd_stats->extent.min[d];
+		}
+		else
+		{
+			int size = (int)roundf(nd_stats->size[d]);
 
-		POSTGIS_DEBUGF(5, " stats: dim %d: min %g: max %g: width %g", d, smin, smax, width);
-		POSTGIS_DEBUGF(5, " overlap: dim %d: (%d, %d)", d, nd_ibox->min[d], nd_ibox->max[d]);
+			/* ... find cells the box overlaps with in this dimension */
+			nd_ibox->min[d] = floor(size * (nd_box->min[d] - smin) / width);
+			nd_ibox->max[d] = floor(size * (nd_box->max[d] - smin) / width);
 
-		/* Push any out-of range values into range */
-		nd_ibox->min[d] = Max(nd_ibox->min[d], 0);
-		nd_ibox->max[d] = Min(nd_ibox->max[d], size-1);
+			POSTGIS_DEBUGF(5, " stats: dim %d: min %g: max %g: width %g", d, smin, smax, width);
+			POSTGIS_DEBUGF(5, " overlap: dim %d: (%d, %d)", d, nd_ibox->min[d], nd_ibox->max[d]);
+
+			/* Push any out-of range values into range */
+			nd_ibox->min[d] = Max(nd_ibox->min[d], 0);
+			nd_ibox->max[d] = Min(nd_ibox->max[d], size - 1);
+		}
 	}
 	return TRUE;
 }
@@ -1855,7 +1865,9 @@
 
 	POSTGIS_DEBUGF(3, " attribute stat target: %d", attr->attstattarget);
 
-	/* Setup the minimum rows and the algorithm function */
+	/* Setup the minimum rows and the algorithm function.
+	 * 300 matches the default value set in
+	 * postgresql/src/backend/commands/analyze.c */
 	stats->minrows = 300 * stats->attr->attstattarget;
 	stats->compute_stats = compute_gserialized_stats;