[GRASS-dev] G7: v.in.ascii problem with broken lines: Busy SQLITE db

[Markus Neteler]

Hi

when importing a (big) file into GRASS GIS 7 with some broken lines in
it, the import gets stuck at some point:

Importing points...
DBMI-SQLite driver error:
Error in sqlite3_prepare():
table occurrence_plantae has 6 columns but 5 values were supplied

DBMI-SQLite driver error:
Error in sqlite3_prepare():
table occurrence_plantae has 6 columns but 5 values were supplied

ERROR: Unable to insert new record: insert into occurrence_plantae values (
       29222, 4.75738, 51.79876, 'Plantae', 'Bryophyta')
WARNING: Busy SQLITE db, already waiting for 10 seconds...
WARNING: Busy SQLITE db, already waiting for 20 seconds...
WARNING: Busy SQLITE db, already waiting for 30 seconds...
WARNING: Busy SQLITE db, already waiting for 40 seconds...
WARNING: Busy SQLITE db, already waiting for 50 seconds...
WARNING: Busy SQLITE db, already waiting for 60 seconds...
WARNING: Busy SQLITE db, already waiting for 70 seconds...
WARNING: Busy SQLITE db, already waiting for 80 seconds...
WARNING: Busy SQLITE db, already waiting for 90 seconds...
WARNING: Busy SQLITE db, already waiting for 100 seconds...
...
CTRL-C

The reason is (I think) that it just calls G_fatal_error() in the
points.c file rather than shutting down the DBMI stuff properly,
closing files etc. before.

While at it:
I have started to implement a -i "ignore" flag to simply skip broken
lines. This doesn't fully work yet: "category out of range" but I am
not vector C expert enough to complete that.
We have 180 million lines to import and some are broken, so a -i flag
would be quite handy rather than manually identifying those lines.

Anyone able to help to solve both issues?

thanks,
Markus
-------------- next part --------------
Index: vector/v.in.ascii/local_proto.h
===================================================================
--- vector/v.in.ascii/local_proto.h	(revision 62616)
+++ vector/v.in.ascii/local_proto.h	(working copy)
@@ -9,6 +9,6 @@
 		   int **, int, int, int, int);
 
 int points_to_bin(FILE *, int, struct Map_info *, dbDriver *,
-		  char *, char *, int, int, int *, int, int, int, int, int);
+		  char *, char *, int, int, int *, int, int, int, int, int, int);
 
 #endif /* __LOCAL_PROTO_H__ */
Index: vector/v.in.ascii/main.c
===================================================================
--- vector/v.in.ascii/main.c	(revision 62616)
+++ vector/v.in.ascii/main.c	(working copy)
@@ -35,7 +35,7 @@
 	*ycol_opt, *zcol_opt, *catcol_opt, *format_opt, *skip_opt;
     int xcol, ycol, zcol, catcol, format, skip_lines;
     struct Flag *zcoorf, *t_flag, *e_flag, *noheader_flag, *notopol_flag,
-	*region_flag;
+	*region_flag, *ignore_flag;
     char *table;
     char *fs;
     char *desc;
@@ -171,6 +171,11 @@
 	_("Only import points falling within current region (points mode)");
     region_flag->guisection = _("Points");
 
+    ignore_flag = G_define_flag();
+    ignore_flag->key = 'i';
+    ignore_flag->description = _("Ignore broken line(s) in points mode");
+    ignore_flag->guisection = _("Points");
+
     if (G_parser(argc, argv))
 	exit(EXIT_FAILURE);
 
@@ -497,7 +502,7 @@
 	}
 
 	points_to_bin(tmpascii, rowlen, &Map, driver, table, fs, nrows, ncols,
-		      coltype2, xcol, ycol, zcol, catcol, skip_lines);
+		      coltype2, xcol, ycol, zcol, catcol, skip_lines, ignore_flag->answer);
 
 	if (driver) {
 	    G_message(_("Populating table..."));
Index: vector/v.in.ascii/points.c
===================================================================
--- vector/v.in.ascii/points.c	(revision 62616)
+++ vector/v.in.ascii/points.c	(working copy)
@@ -282,7 +282,7 @@
 int points_to_bin(FILE * ascii, int rowlen, struct Map_info *Map,
 		  dbDriver * driver, char *table, char *fs, int nrows,
 		  int ncols, int *coltype, int xcol, int ycol, int zcol,
-		  int catcol, int skip_lines)
+		  int catcol, int skip_lines, int ignore_flag)
 {
     char *buf, buf2[4000];
     int cat = 0;
@@ -406,8 +406,16 @@
 	    G_debug(3, db_get_string(&sql));
 
 	    if (db_execute_immediate(driver, &sql) != DB_OK) {
-		G_fatal_error(_("Unable to insert new record: %s"),
+		if (ignore_flag) {
+		   /* who cares, go on */
+		   G_warning(_("Ignoring unreadable/missing/extra entry in line %d of input file: <%s>"), i, buf);
+		   Vect_line_delete_point(Points, cat);
+		   Vect_append_point(Points, x, y, z);
+		}
+		else {
+		   G_fatal_error(_("Unable to insert new record: %s"),
 			      db_get_string(&sql));
+		}
 	    }
 	}