[postgis-tickets] r15447 - Fix handling of EMPTY geometries in btree operator

[Sandro Santilli]

Author: strk
Date: 2017-06-23 07:47:13 -0700 (Fri, 23 Jun 2017)
New Revision: 15447

Modified:
   trunk/postgis/lwgeom_btree.c
   trunk/regress/operators.sql
   trunk/regress/operators_expected
Log:
Fix handling of EMPTY geometries in btree operator

Includes regress test

See #3777

Modified: trunk/postgis/lwgeom_btree.c
===================================================================
--- trunk/postgis/lwgeom_btree.c	2017-06-23 11:40:21 UTC (rev 15446)
+++ trunk/postgis/lwgeom_btree.c	2017-06-23 14:47:13 UTC (rev 15447)
@@ -53,6 +53,7 @@
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_lt called");
 
@@ -60,14 +61,19 @@
 
 	POSTGIS_DEBUG(3, "lwgeom_lt passed getSRID test");
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
 	POSTGIS_DEBUG(3, "lwgeom_lt getbox2d_p passed");
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin < box2.xmin)
@@ -102,17 +108,23 @@
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_le called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin < box2.xmin)
@@ -168,9 +180,10 @@
 
 	gbox_init(&box1);
 	gbox_init(&box2);
-	
+
 	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
 	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
+
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
@@ -198,17 +211,23 @@
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_ge called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin > box2.xmin)
@@ -255,17 +274,23 @@
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_gt called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin > box2.xmin)
@@ -308,17 +333,23 @@
 	GSERIALIZED *geom2 = PG_GETARG_GSERIALIZED_P(1);
 	GBOX box1;
 	GBOX box2;
+	bool empty1, empty2;
 
 	POSTGIS_DEBUG(2, "lwgeom_cmp called");
 
 	error_if_srid_mismatch(gserialized_get_srid(geom1), gserialized_get_srid(geom2));
 
-	gserialized_get_gbox_p(geom1, &box1);
-	gserialized_get_gbox_p(geom2, &box2);
+	empty1 = ( gserialized_get_gbox_p(geom1, &box1) == LW_FAILURE );
+	empty2 = ( gserialized_get_gbox_p(geom2, &box2) == LW_FAILURE );
 
 	PG_FREE_IF_COPY(geom1, 0);
 	PG_FREE_IF_COPY(geom2, 1);
 
+	if  ( empty1 != empty2 )
+	{
+		PG_RETURN_BOOL(FALSE);
+	}
+
 	if  ( ! FPeq(box1.xmin , box2.xmin) )
 	{
 		if  (box1.xmin < box2.xmin)

Modified: trunk/regress/operators.sql
===================================================================
--- trunk/regress/operators.sql	2017-06-23 11:40:21 UTC (rev 15446)
+++ trunk/regress/operators.sql	2017-06-23 14:47:13 UTC (rev 15447)
@@ -140,4 +140,15 @@
 SELECT 'ndovm2', array_agg(i) FROM v WHERE g &&& 'POINTZ(0 0 1)'::geometry
 ORDER BY 1;
 
+-- GROUP BY on empty
+SELECT '#3777', ST_AsText(geom), count(*) FROM (
+SELECT 'POINT(0 0)'::geometry geom UNION ALL
+SELECT 'POINT(0 0)'::geometry UNION ALL
+SELECT 'POINT(0 0)'::geometry UNION ALL
+SELECT 'POINT(0 1)'::geometry UNION ALL
+SELECT 'LINESTRING(0 0,0 1)'::geometry UNION ALL
+SELECT 'GEOMETRYCOLLECTION EMPTY'::geometry UNION ALL
+SELECT 'POINT EMPTY'::geometry
+) foo
+GROUP BY geom ORDER BY 2;
 

Modified: trunk/regress/operators_expected
===================================================================
--- trunk/regress/operators_expected	2017-06-23 11:40:21 UTC (rev 15446)
+++ trunk/regress/operators_expected	2017-06-23 14:47:13 UTC (rev 15447)
@@ -56,3 +56,7 @@
 ndov7|t
 ndovm1|{1,2,3,4,5,8}
 ndovm2|{1,2,4,6,7}
+#3777|GEOMETRYCOLLECTION EMPTY|2
+#3777|LINESTRING(0 0,0 1)|1
+#3777|POINT(0 0)|3
+#3777|POINT(0 1)|1