[postgis-tickets] r17320 - Force early flushing of postgis_restore.pl output

[Sandro Santilli]

Author: strk
Date: 2019-03-10 11:51:21 -0700 (Sun, 10 Mar 2019)
New Revision: 17320

Modified:
   trunk/NEWS
   trunk/utils/postgis_restore.pl.in
Log:
Force early flushing of postgis_restore.pl output

Reduces memory usage when piping output to further processing filter.

Patch by Hugh Ranalli

Closes #4330

Modified: trunk/NEWS
===================================================================
--- trunk/NEWS	2019-03-10 18:51:13 UTC (rev 17319)
+++ trunk/NEWS	2019-03-10 18:51:21 UTC (rev 17320)
@@ -35,6 +35,8 @@
   - #4342, Move deprecated functions into legacy.sql file
   - #4341, Using "support function" API in PgSQL 12+ to replace SQL inlining
            as the mechanism for providing index support under ST_Intersects, et al
+  - #4330, postgis_restore OOM when output piped to an intermediate process
+           (Hugh Ranalli)
   - #4322, Support for Proj 6+ API, bringing more accurate datum transforms
            and support for WKT projections
   - #4153, ST_Segmentize now splits segments proportionally (Darafei

Modified: trunk/utils/postgis_restore.pl.in
===================================================================
--- trunk/utils/postgis_restore.pl.in	2019-03-10 18:51:13 UTC (rev 17319)
+++ trunk/utils/postgis_restore.pl.in	2019-03-10 18:51:21 UTC (rev 17320)
@@ -94,6 +94,8 @@
 
 print STDERR "Converting $dumpfile to ASCII on stdout...\n";
 
+STDOUT->autoflush(1);
+
 ######################################################################
 # Load the signatures of things to skip.
 #
@@ -265,7 +267,7 @@
   #
   elsif ( $l =~ /CREATE TABLE *([^ ,]*)/)
   {
-    my @sublines = ($l);
+    print STDOUT $l;
     while( my $subline = <INPUT>)
     {
       if ( $subline =~ /CONSTRAINT "?enforce_dims_/i ) {
@@ -281,10 +283,9 @@
           print STDERR "WARNING: could not find SRID value in: $subline";
         }
       }
-      push(@sublines, $subline);
+      print STDOUT $subline;
       last if $subline =~ /;[\t ]*$/;
     }
-    print STDOUT @sublines;
     next;
   }
 
@@ -292,15 +293,13 @@
   # See http://trac.osgeo.org/postgis/ticket/2759
   elsif ( $l =~ /^COMMENT ON .* IS '(.*)/)
   {
-    my @sublines = ($l);
-    my $comment = $1;
+    print STDOUT $l;
+    while( my $subline = <INPUT>)
     # A comment ends with an odd number of single quotes and a semicolon
-    while( $comment !~ /('*)[\t ]*;[\t ]*$/ || length($1) % 2 == 0)
     {
-      $comment = <INPUT>;
-      push(@sublines, $comment);
+      print STDOUT $subline;
+      last if ( $subline !~ /('*)[\t ]*;[\t ]*$/ || length($1) % 2 == 0)
     }
-    print STDOUT @sublines;
     next;
   }