[GRASS-SVN] r51245 - in grass/trunk: lib/python/temporal
temporal/t.register
svn_grass at osgeo.org
svn_grass at osgeo.org
Tue Apr 3 14:08:20 EDT 2012
Author: huhabla
Date: 2012-04-03 11:08:20 -0700 (Tue, 03 Apr 2012)
New Revision: 51245
Modified:
grass/trunk/lib/python/temporal/abstract_dataset.py
grass/trunk/lib/python/temporal/abstract_map_dataset.py
grass/trunk/lib/python/temporal/abstract_space_time_dataset.py
grass/trunk/lib/python/temporal/base.py
grass/trunk/lib/python/temporal/space_time_datasets_tools.py
grass/trunk/temporal/t.register/test.t.register.vector.file.layer.sh
grass/trunk/temporal/t.register/test.t.register.vector.sh
Log:
Reorganization of the SQL statement handling. Methods
to gather SQL statements have been implemented to speed up
the registration and unregistration of massive amounts of maps (>10000)
Modified: grass/trunk/lib/python/temporal/abstract_dataset.py
===================================================================
--- grass/trunk/lib/python/temporal/abstract_dataset.py 2012-04-02 14:03:09 UTC (rev 51244)
+++ grass/trunk/lib/python/temporal/abstract_dataset.py 2012-04-03 18:08:20 UTC (rev 51245)
@@ -181,19 +181,12 @@
bottom = self.spatial_extent.get_bottom()
return (north, south, east, west, top, bottom)
-
+
def select(self, dbif=None):
"""!Select temporal dataset entry from database and fill up the internal structure"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
- dbif.cursor.execute("BEGIN TRANSACTION")
-
self.base.select(dbif)
if self.is_time_absolute():
self.absolute_time.select(dbif)
@@ -202,102 +195,88 @@
self.spatial_extent.select(dbif)
self.metadata.select(dbif)
- dbif.cursor.execute("COMMIT TRANSACTION")
-
if connect:
dbif.close()
-
+
def is_in_db(self, dbif=None):
- """!Check if the temporal dataset entry is in the database"""
- return self.base.is_in_db(dbif)
+ """!Check if the temporal dataset entry is in the database"""
+ return self.base.is_in_db(dbif)
def delete(self):
"""!Delete temporal dataset entry from database if it exists"""
raise IOError("This method must be implemented in the subclasses")
- def insert(self, dbif=None):
- """!Insert temporal dataset entry into database from the internal structure"""
+ def insert(self, dbif=None, execute=True):
+ """!Insert temporal dataset entry into database from the internal structure
- connect = False
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
+ @param dbif: The database interface to be used
+ @param execute: If True the SQL statements will be executed.
+ If False the prepared SQL statements are returned and must be executed by the caller.
+ """
- dbif.cursor.execute("BEGIN TRANSACTION")
-
-
- self.base.insert(dbif)
- if self.is_time_absolute():
- self.absolute_time.insert(dbif)
+ # Build the INSERT SQL statement
+ statement = self.base.get_insert_statement_mogrified(dbif)
+ if self.is_time_absolute():
+ statement += self.absolute_time.get_insert_statement_mogrified(dbif)
if self.is_time_relative():
- self.relative_time.insert(dbif)
- self.spatial_extent.insert(dbif)
- self.metadata.insert(dbif)
+ statement += self.relative_time.get_insert_statement_mogrified(dbif)
+ statement += self.spatial_extent.get_insert_statement_mogrified(dbif)
+ statement += self.metadata.get_insert_statement_mogrified(dbif)
- dbif.cursor.execute("COMMIT TRANSACTION")
+ if execute == True:
+ execute_transaction(statement, dbif)
+ return ""
- if connect:
- dbif.close()
-
- def update(self, dbif=None):
+ return statement
+
+ def update(self, dbif=None, execute=True):
"""!Update temporal dataset entry of database from the internal structure
excluding None variables
+
+ @param dbif: The database interface to be used
+ @param execute: If True the SQL statements will be executed.
+ If False the prepared SQL statements are returned and must be executed by the caller.
"""
- connect = False
-
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
- dbif.cursor.execute("BEGIN TRANSACTION")
-
-
- self.base.update(dbif)
+ # Build the UPDATE SQL statement
+ statement = self.base.get_update_statement_mogrified(dbif)
if self.is_time_absolute():
- self.absolute_time.update(dbif)
+ statement += self.absolute_time.get_update_statement_mogrified(dbif)
if self.is_time_relative():
- self.relative_time.update(dbif)
- self.spatial_extent.update(dbif)
- self.metadata.update(dbif)
+ statement += self.relative_time.get_update_statement_mogrified(dbif)
+ statement += self.spatial_extent.get_update_statement_mogrified(dbif)
+ statement += self.metadata.get_update_statement_mogrified(dbif)
- dbif.cursor.execute("COMMIT TRANSACTION")
-
- if connect:
- dbif.close()
+ if execute == True:
+ execute_transaction(statement, dbif)
+ return ""
- def update_all(self, dbif=None):
+ return statement
+
+ def update_all(self, dbif=None, execute=True):
"""!Update temporal dataset entry of database from the internal structure
and include None varuables.
@param dbif: The database interface to be used
+ @param execute: If True the SQL statements will be executed.
+ If False the prepared SQL statements are returned and must be executed by the caller.
"""
- connect = False
-
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
- dbif.cursor.execute("BEGIN TRANSACTION")
-
-
- self.base.update_all(dbif)
+ # Build the UPDATE SQL statement
+ statement = self.base.get_update_all_statement_mogrified(dbif)
if self.is_time_absolute():
- self.absolute_time.update_all(dbif)
+ statement += self.absolute_time.get_update_all_statement_mogrified(dbif)
if self.is_time_relative():
- self.relative_time.update_all(dbif)
- self.spatial_extent.update_all(dbif)
- self.metadata.update_all(dbif)
+ statement += self.relative_time.get_update_all_statement_mogrified(dbif)
+ statement += self.spatial_extent.get_update_all_statement_mogrified(dbif)
+ statement += self.metadata.get_update_all_statement_mogrified(dbif)
- dbif.cursor.execute("COMMIT TRANSACTION")
+ if execute == True:
+ execute_transaction(statement, dbif)
+ return ""
- if connect:
- dbif.close()
+ return statement
def set_time_to_absolute(self):
self.base.set_ttype("absolute")
Modified: grass/trunk/lib/python/temporal/abstract_map_dataset.py
===================================================================
--- grass/trunk/lib/python/temporal/abstract_map_dataset.py 2012-04-02 14:03:09 UTC (rev 51244)
+++ grass/trunk/lib/python/temporal/abstract_map_dataset.py 2012-04-03 18:08:20 UTC (rev 51245)
@@ -211,13 +211,8 @@
@param end_time: a datetime object specifying the end time of the map
@param timezone: Thee timezone of the map
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
self.set_absolute_time(start_time, end_time, timezone)
self.absolute_time.update_all(dbif)
self.base.update(dbif)
@@ -290,13 +285,8 @@
@param end_time: A double value
@param dbif: The database interface to be used
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
if self.set_relative_time(start_time, end_time, unit):
self.relative_time.update_all(dbif)
self.base.update(dbif)
@@ -366,14 +356,9 @@
@return The SQL statements if execute == False, else an empty string, None in case of a failure
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
statement = ""
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
if self.is_in_db(dbif):
# SELECT all needed information from the database
@@ -389,27 +374,11 @@
core.verbose(_("Delete %s dataset <%s> from temporal database") % (self.get_type(), self.get_id()))
# Delete yourself from the database, trigger functions will take care of dependencies
- statement += self.base.get_delete_statement() + ";\n"
+ statement += self.base.get_delete_statement()
if execute == True:
- sql_script = ""
- sql_script += "BEGIN TRANSACTION;\n"
- sql_script += statement
- sql_script += "END TRANSACTION;"
- # print sql_script
- try:
- if dbmi.__name__ == "sqlite3":
- dbif.cursor.executescript(statement)
- else:
- dbif.cursor.execute(statement)
- except:
- if connect == True:
- dbif.close()
- core.error(_("Unable to correctly delete %s map <%s>") % (self.get_type(), self.get_id()))
- raise
+ execute_transaction(statement, dbif)
- dbif.connection.commit()
-
# Remove the timestamp from the file system
if self.get_type() == "vect":
if self.get_layer():
@@ -428,7 +397,6 @@
return ""
return statement
-
def unregister(self, dbif=None, update=True, execute=True):
"""! Remove the map entry in each space time dataset in which this map is registered
@@ -449,13 +417,8 @@
core.verbose(_("Unregister %s map <%s> from space time datasets") % (self.get_type(), self.get_map_id()))
statement = ""
- connect = False
-
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
+ dbif, connect = init_dbif(dbif)
+
# Get all datasets in which this map is registered
rows = self.get_registered_datasets(dbif)
@@ -477,25 +440,10 @@
stds.update_from_registered_maps(dbif)
core.percent(1, 1, 1)
+
if execute == True:
- sql_script = ""
- sql_script += "BEGIN TRANSACTION;\n"
- sql_script += statement
- sql_script += "END TRANSACTION;"
- # print sql_script
- try:
- if dbmi.__name__ == "sqlite3":
- dbif.cursor.executescript(statement)
- else:
- dbif.cursor.execute(statement)
- except:
- if connect == True:
- dbif.close()
- core.error(_("Unable to correctly unregister %s <%s>") % (self.get_type(), self.get_id()))
- raise
+ execute_transaction(statement, dbif)
- dbif.connection.commit()
-
if connect == True:
dbif.close()
@@ -511,13 +459,8 @@
@param dbif: The database interface to be used
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
rows = None
try:
Modified: grass/trunk/lib/python/temporal/abstract_space_time_dataset.py
===================================================================
--- grass/trunk/lib/python/temporal/abstract_space_time_dataset.py 2012-04-02 14:03:09 UTC (rev 51244)
+++ grass/trunk/lib/python/temporal/abstract_space_time_dataset.py 2012-04-03 18:08:20 UTC (rev 51245)
@@ -480,13 +480,8 @@
use_contain = False
use_equal = False
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
obj_list = []
sample_maps = stds.get_registered_maps_as_objects_with_gaps(where=None, dbif=dbif)
@@ -559,13 +554,8 @@
In case nothing found None is returned
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
obj_list = []
if gran == None:
@@ -623,13 +613,8 @@
In case nothing found None is returned
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
obj_list = []
maps = self.get_registered_maps_as_objects(where, "start_time", dbif)
@@ -675,13 +660,8 @@
In case nothing found None is returned
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
obj_list = []
rows = self.get_registered_maps("id,start_time,end_time", where, order, dbif)
@@ -718,13 +698,8 @@
In case nothing found None is returned
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
rows = None
if self.get_map_register():
@@ -775,19 +750,13 @@
core.verbose(_("Delete space time %s dataset <%s> from temporal database") % (self.get_new_map_instance(ident=None).get_type(), self.get_id()))
statement = ""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
# SELECT all needed information from the database
self.metadata.select(dbif)
- core.verbose(_("Drop map register table: %s") % (self.get_map_register()))
-
if self.get_map_register():
+ core.verbose(_("Drop map register table: %s") % (self.get_map_register()))
rows = self.get_registered_maps("id", None, None, dbif)
# Unregister each registered map in the table
if rows:
@@ -800,39 +769,26 @@
statement += self.unregister_map(map=map, dbif=dbif, execute=False)
count += 1
core.percent(1, 1, 1)
- # Safe the DROP table statement
- statement += "DROP TABLE " + self.get_map_register() + ";\n"
+ # Safe the DROP table statement
+ statement += "DROP TABLE " + self.get_map_register() + ";\n"
+
# Remove the primary key, the foreign keys will be removed by trigger
- statement += self.base.get_delete_statement() + ";\n"
-
+ statement += self.base.get_delete_statement()
+
if execute == True:
- sql_script = ""
- sql_script += "BEGIN TRANSACTION;\n"
- sql_script += statement
- sql_script += "END TRANSACTION;"
- print sql_script
- try:
- if dbmi.__name__ == "sqlite3":
- dbif.cursor.executescript(statement)
- else:
- dbif.cursor.execute(statement)
- except:
- if connect == True:
- dbif.close()
- core.error(_("Unable to correctly delete %s <%s>") % (self.get_type(), self.get_id()))
- raise
+ execute_transaction(statement, dbif)
self.reset(None)
if connect == True:
dbif.close()
-
+
if execute:
return ""
return statement
-
+
def register_map(self, map, dbif=None):
"""!Register a map in the space time dataset.
@@ -844,13 +800,8 @@
@param dbif: The database interface to be used
"""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
if map.is_in_db(dbif) == False:
dbif.close()
core.fatal(_("Only maps with absolute or relative valid time can be registered"))
@@ -882,6 +833,9 @@
stds_mapset = self.base.get_mapset()
stds_register_table = self.get_map_register()
stds_ttype = self.get_temporal_type()
+
+ # The gathered SQL statemets are stroed here
+ statement = ""
# Check temporal types
if stds_ttype != map_ttype:
@@ -895,7 +849,7 @@
self.map_counter == 0 and self.is_time_relative():
self.set_relative_time_unit(map_rel_time_unit)
- self.relative_time.update()
+ statement += self.relative_time.get_update_all_statement_mogrified(dbif)
core.verbose(_("Set temporal unit for space time %s dataset <%s> to %s") % (map.get_type(), self.get_id(), map_rel_time_unit))
stds_rel_time_unit = self.get_relative_time_unit()
@@ -919,18 +873,23 @@
sql = "SELECT id FROM " + stds_register_table + " WHERE id = (?)"
else:
sql = "SELECT id FROM " + stds_register_table + " WHERE id = (%s)"
- dbif.cursor.execute(sql, (map_id,))
- row = dbif.cursor.fetchone()
- # In case of no entry make a new one
+ try:
+ dbif.cursor.execute(sql, (map_id,))
+ row = dbif.cursor.fetchone()
+ except:
+ row = None
+ core.warning(_("Error in strds_register_table request"))
+ raise
+
if row and row[0] == map_id:
if connect == True:
dbif.close()
-
+
if map.get_layer():
core.warning(_("Map <%s> with layer %s is already registered.") % (map.get_map_id(), map.get_layer()))
else:
core.warning(_("Map <%s> is already registered.") % (map.get_map_id()))
- return False
+ return ""
# Create tables
sql_path = get_sql_template_path()
@@ -950,25 +909,12 @@
sql = sql.replace("TABLE_NAME", uuid_rand )
sql = sql.replace("MAP_ID", map_id)
sql = sql.replace("STDS", self.get_type())
- try:
- if dbmi.__name__ == "sqlite3":
- dbif.cursor.executescript(sql)
- else:
- dbif.cursor.execute(sql)
- except:
- if connect == True:
- dbif.close()
- if map.get_layer():
- core.error(_("Unable to create the space time %s dataset register table for map <%s> with layer %s") % \
- (map.get_type(), map.get_map_id(), map.get_layer()))
- else:
- core.error(_("Unable to create the space time %s dataset register table for <%s>") % \
- (map.get_type(), map.get_map_id()))
- raise
+
+ statement += sql
# Set the stds register table name and put it into the DB
map.set_stds_register(map_register_table)
- map.metadata.update(dbif)
+ statement += map.metadata.get_update_statement_mogrified(dbif)
if map.get_layer():
core.verbose(_("Created register table <%s> for %s map <%s> with layer %s") % \
@@ -988,70 +934,59 @@
sql = sql.replace("SPACETIME_NAME", stds_name + "_" + stds_mapset )
sql = sql.replace("SPACETIME_ID", self.base.get_id())
sql = sql.replace("STDS", self.get_type())
+ statement += sql
- sql_script = ""
- sql_script += "BEGIN TRANSACTION;\n"
- sql_script += sql
- sql_script += "\n"
- sql_script += "END TRANSACTION;"
- try:
- if dbmi.__name__ == "sqlite3":
- dbif.cursor.executescript(sql_script)
- else:
- dbif.cursor.execute(sql_script)
- dbif.connection.commit()
- except:
- if connect == True:
- dbif.close()
- if map.get_layer():
- core.error(_("Unable to create the space time %s dataset register table for map <%s> with layer %s") % \
- (map.get_type(), map.get_map_id(), map.get_layer()))
- else:
- core.error(_("Unable to create the space time %s dataset register table for <%s>") % \
- (map.get_type(), map.get_map_id()))
- raise
-
# Set the map register table name and put it into the DB
self.set_map_register(stds_register_table)
- self.metadata.update(dbif)
+ statement += self.metadata.get_update_statement_mogrified(dbif)
core.verbose(_("Created register table <%s> for space time %s dataset <%s>") % \
(stds_register_table, map.get_type(), self.get_id()))
+ # We need to execute the statement at this time
+ if statement != "":
+ execute_transaction(statement, dbif)
+
+ statement = ""
+
# Register the stds in the map stds register table
# Check if the entry is already there
if dbmi.paramstyle == "qmark":
sql = "SELECT id FROM " + map_register_table + " WHERE id = ?"
else:
sql = "SELECT id FROM " + map_register_table + " WHERE id = %s"
- dbif.cursor.execute(sql, (self.base.get_id(),))
- row = dbif.cursor.fetchone()
+ try:
+ dbif.cursor.execute(sql, (self.base.get_id(),))
+ row = dbif.cursor.fetchone()
+ except:
+ row = None
# In case of no entry make a new one
if row == None:
if dbmi.paramstyle == "qmark":
- sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (?)"
+ sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (?);\n"
else:
- sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (%s)"
- #print sql
- dbif.cursor.execute(sql, (self.base.get_id(),))
+ sql = "INSERT INTO " + map_register_table + " (id) " + "VALUES (%s);\n"
+ statement += dbif._mogrify_sql_statement((sql, (self.base.get_id(),)), dbif)
+
# Now put the raster name in the stds map register table
if dbmi.paramstyle == "qmark":
- sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (?)"
+ sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (?);\n"
else:
- sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (%s)"
- #print sql
- dbif.cursor.execute(sql, (map_id,))
+ sql = "INSERT INTO " + stds_register_table + " (id) " + "VALUES (%s);\n"
+ statement += dbif._mogrify_sql_statement((sql, (map_id,)), dbif)
+
+ # Now execute the insert transaction
+ execute_transaction(statement, dbif)
+
if connect == True:
dbif.close()
-
+
# increase the counter
self.map_counter += 1
- return True
-
def unregister_map(self, map, dbif = None, execute=True):
"""!Unregister a map from the space time dataset.
@@ -1068,13 +1003,8 @@
statement = ""
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
# First select needed data from the database
map.metadata.select(dbif)
@@ -1092,8 +1022,11 @@
sql = "SELECT id FROM " + map_register_table + " WHERE id = ?"
else:
sql = "SELECT id FROM " + map_register_table + " WHERE id = %s"
- dbif.cursor.execute(sql, (self.base.get_id(),))
- row = dbif.cursor.fetchone()
+ try:
+ dbif.cursor.execute(sql, (self.base.get_id(),))
+ row = dbif.cursor.fetchone()
+ except:
+ row = None
# Break if the map is not registered
if row == None:
@@ -1103,34 +1036,29 @@
core.warning(_("Map <%s> is not registered in space time dataset <%s>") %(map.get_map_id(), self.base.get_id()))
if connect == True:
dbif.close()
- return None
+ return ""
# Remove the space time raster dataset from the raster dataset register
if map_register_table != None:
+ if dbmi.paramstyle == "qmark":
+ sql = "DELETE FROM " + map_register_table + " WHERE id = ?;\n"
+ else:
+ sql = "DELETE FROM " + map_register_table + " WHERE id = %s;\n"
- statement += "DELETE FROM " + map_register_table + " WHERE id = \'%s\';\n"%(self.base.get_id())
+ statement += dbif._mogrify_sql_statement((sql, (self.base.get_id(),)), dbif)
- if execute == True:
- if dbmi.paramstyle == "qmark":
- sql = "DELETE FROM " + map_register_table + " WHERE id = ?"
- else:
- sql = "DELETE FROM " + map_register_table + " WHERE id = %s"
-
- dbif.cursor.execute(sql, (self.base.get_id(),))
-
# Remove the raster map from the space time raster dataset register
if stds_register_table != None:
+ if dbmi.paramstyle == "qmark":
+ sql = "DELETE FROM " + stds_register_table + " WHERE id = ?;\n"
+ else:
+ sql = "DELETE FROM " + stds_register_table + " WHERE id = %s;\n"
- statement += "DELETE FROM " + stds_register_table + " WHERE id = \'%s\';\n"%(map_id)
+ statement += dbif._mogrify_sql_statement((sql, (map_id,)), dbif)
- if execute == True:
- if dbmi.paramstyle == "qmark":
- sql = "DELETE FROM " + stds_register_table + " WHERE id = ?"
- else:
- sql = "DELETE FROM " + stds_register_table + " WHERE id = %s"
-
- dbif.cursor.execute(sql, (map_id,))
-
+ if execute == True:
+ execute_transaction(statement, dbif)
+
if connect == True:
dbif.close()
@@ -1164,13 +1092,8 @@
if not self.get_map_register():
return
- connect = False
+ dbif, connect = init_dbif(dbif)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
map_time = None
use_start_time = False
@@ -1182,7 +1105,6 @@
#We create a transaction
sql_script = ""
- sql_script += "BEGIN TRANSACTION;\n"
# Update the spatial and temporal extent from registered maps
# Read the SQL template
@@ -1205,12 +1127,7 @@
sql_script += sql
sql_script += "\n"
- sql_script += "END TRANSACTION;"
-
- if dbmi.__name__ == "sqlite3":
- dbif.cursor.executescript(sql_script)
- else:
- dbif.cursor.execute(sql_script)
+ execute_transaction(sql_script, dbif)
# Read and validate the selected end time
self.select()
@@ -1280,10 +1197,7 @@
sql = sql.replace("SPACETIME_ID", self.base.get_id())
sql = sql.replace("STDS", self.get_type())
- if dbmi.__name__ == "sqlite3":
- dbif.cursor.executescript(sql)
- else:
- dbif.cursor.execute(sql)
+ execute_transaction(sql, dbif)
# Count the temporal map types
maps = self.get_registered_maps_as_objects(dbif=dbif)
Modified: grass/trunk/lib/python/temporal/base.py
===================================================================
--- grass/trunk/lib/python/temporal/base.py 2012-04-02 14:03:09 UTC (rev 51244)
+++ grass/trunk/lib/python/temporal/base.py 2012-04-03 18:08:20 UTC (rev 51245)
@@ -59,6 +59,7 @@
sql += ' FROM ' + table + ' '
if where:
sql += where
+ sql += ";\n"
# Create insert statement
if type =="INSERT":
@@ -90,6 +91,7 @@
if where:
sql += where
+ sql += ";\n"
# Create update statement for existing entries
if type =="UPDATE":
@@ -114,6 +116,7 @@
args.append(self.D[key])
if where:
sql += where
+ sql += ";\n"
# Create update statement for all entries
if type =="UPDATE ALL":
@@ -136,6 +139,7 @@
args.append(self.D[key])
if where:
sql += where
+ sql += ";\n"
return sql, tuple(args)
@@ -219,7 +223,7 @@
self.cursor.execute("PRAGMA journal_mode = MEMORY")
elif dbmi.__name__ == "psycopg2":
self.connection = dbmi.connect(init)
- self.connection.set_isolation_level(dbmi.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+ #self.connection.set_isolation_level(dbmi.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
self.cursor = self.connection.cursor(cursor_factory=dbmi.extras.DictCursor)
def close(self):
@@ -228,8 +232,65 @@
self.connection.commit()
self.cursor.close()
+
+ def _mogrify_sql_statement(self, content, dbif=None):
+ """!Return the SQL statement and arguments as executable SQL string
+ """
+ sql = content[0]
+ args = content[1]
+
+ if dbmi.__name__ == "psycopg2":
+ if len(args) == 0:
+ return sql
+ else:
+ if dbif:
+ try:
+ return dbif.cursor.mogrify(sql, args)
+ except:
+ print sql, args
+ raise
+ else:
+ self.connect()
+ statement = self.cursor.mogrify(sql, args)
+ self.close()
+ return statement
+
+ elif dbmi.__name__ == "sqlite3":
+ if len(args) == 0:
+ return sql
+ else:
+ # Unfortunately as sqlite does not support
+ # the transformation of sql strings and qmarked or
+ # named arguments we must make our hands dirty
+ # and do it by ourself. :(
+ # Doors are open for SQL injection because of the
+ # limited python sqlite3 implementation!!!
+ pos = 0
+ count = 0
+ maxcount = 100
+ statement = sql
+
+ while count < maxcount:
+ pos = statement.find("?", pos + 1)
+ if pos == -1:
+ break
+
+ if args[count] == None:
+ statement = "%sNULL%s"%(statement[0:pos], statement[pos+1:])
+ elif isinstance(args[count], (int, long)):
+ statement = "%s%d%s"%(statement[0:pos], args[count],statement[pos+1:])
+ elif isinstance(args[count], float):
+ statement = "%s%f%s"%(statement[0:pos], args[count],statement[pos+1:])
+ else:
+ # Default is a string, this works for datetime objects too
+ statement = "%s\'%s\'%s"%(statement[0:pos], str(args[count]),statement[pos+1:])
+ count += 1
+
+ return statement
+
def get_delete_statement(self):
- return "DELETE FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\'"
+ """!Return the delete string"""
+ return "DELETE FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\';\n"
def delete(self, dbif=None):
"""!Delete the entry of this object from the temporal database"""
@@ -244,7 +305,8 @@
self.close()
def get_is_in_db_statement(self):
- return "SELECT id FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\'"
+ """Return the selection string"""
+ return "SELECT id FROM " + self.get_table_name() + " WHERE id = \'" + str(self.ident) + "\';\n"
def is_in_db(self, dbif=None):
"""!Check if this object is present in the temporal database
@@ -271,8 +333,13 @@
return True
def get_select_statement(self):
+ """!Return the sql statement and the argument list in database specific style"""
return self.serialize("SELECT", self.get_table_name(), "WHERE id = \'" + str(self.ident) + "\'")
-
+
+ def get_select_statement_mogrified(self, dbif=None):
+ """!Return the select statement as mogrified string"""
+ return self._mogrify_sql_statement(self.get_select_statement(), dbif)
+
def select(self, dbif=None):
"""!Select the content from the temporal database and store it
in the internal dictionary structure
@@ -310,7 +377,12 @@
return True
def get_insert_statement(self):
+ """!Return the sql statement and the argument list in database specific style"""
return self.serialize("INSERT", self.get_table_name())
+
+ def get_insert_statement_mogrified(self, dbif=None):
+ """!Return the insert statement as mogrified string"""
+ return self._mogrify_sql_statement(self.get_insert_statement(), dbif)
def insert(self, dbif=None):
"""!Serialize the content of this object and store it in the temporal
@@ -330,8 +402,13 @@
self.close()
def get_update_statement(self):
+ """!Return the sql statement and the argument list in database specific style"""
return self.serialize("UPDATE", self.get_table_name(), "WHERE id = \'" + str(self.ident) + "\'")
+ def get_update_statement_mogrified(self,dbif=None):
+ """!Return the update statement as mogrified string"""
+ return self._mogrify_sql_statement(self.get_update_statement(), dbif)
+
def update(self, dbif=None):
"""!Serialize the content of this object and update it in the temporal
database using the internal identifier
@@ -355,8 +432,13 @@
self.close()
def get_update_all_statement(self):
+ """!Return the sql statement and the argument list in database specific style"""
return self.serialize("UPDATE ALL", self.get_table_name(), "WHERE id = \'" + str(self.ident) + "\'")
+ def get_update_all_statement_mogrified(self, dbif=None):
+ """!Return the update all statement as mogrified string"""
+ return self._mogrify_sql_statement(self.get_update_all_statement(), dbif)
+
def update_all(self, dbif=None):
"""!Serialize the content of this object, including None objects, and update it in the temporal
database using the internal identifier
@@ -672,3 +754,52 @@
stds_base.__init__(self, "stvds_base", ident, name, mapset, semantic_type, creator, creation_time,\
modification_time, temporal_type, revision)
+###############################################################################
+
+def init_dbif(dbif):
+ """!This method checks if the database interface exists, if not a new one
+ will be created and True will be returned
+
+ Usage code sample:
+ dbif, connect = self._init_dbif(dbif)
+ if connect:
+ dbif.close()
+ """
+ if dbif == None:
+ dbif = sql_database_interface()
+ dbif.connect()
+ return dbif, True
+
+ return dbif, False
+
+###############################################################################
+
+def execute_transaction(statement, dbif=None):
+ """!Execute a transactional SQL statement
+
+ The BEGIN and END TRANSACTION statements will be added automatically
+ to the sql statement
+
+ @param statement The executable SQL statement or SQL script
+ @param dbif The database interface, if None a new db interface will be created temporary
+ """
+ dbif, connect = init_dbif(dbif)
+
+ sql_script = ""
+ sql_script += "BEGIN TRANSACTION;\n"
+ sql_script += statement
+ sql_script += "END TRANSACTION;"
+ try:
+ if dbmi.__name__ == "sqlite3":
+ dbif.cursor.executescript(statement)
+ else:
+ dbif.cursor.execute(statement)
+ dbif.connection.commit()
+ except:
+ if connect == True:
+ dbif.close()
+ core.error(_("Unable to execute transaction:\n %s") % (statement))
+ raise
+
+ if connect:
+ dbif.close()
Modified: grass/trunk/lib/python/temporal/space_time_datasets_tools.py
===================================================================
--- grass/trunk/lib/python/temporal/space_time_datasets_tools.py 2012-04-02 14:03:09 UTC (rev 51244)
+++ grass/trunk/lib/python/temporal/space_time_datasets_tools.py 2012-04-03 18:08:20 UTC (rev 51245)
@@ -86,13 +86,8 @@
core.fatal(_("Unkown map type: %s")%(type))
- connect = False
+ dbif, connect = init_dbif(None)
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
if name:
# Read content from temporal database
sp.select(dbif)
@@ -163,6 +158,11 @@
maplist.append(row)
num_maps = len(maplist)
+ map_object_list = []
+ statement = ""
+
+ core.message(_("Gathering map informations"))
+
for count in range(len(maplist)):
core.percent(count, num_maps, 1)
@@ -174,9 +174,12 @@
start = maplist[count]["start"]
if maplist[count].has_key("end"):
end = maplist[count]["end"]
+
+ is_in_db = False
# Put the map into the database
if map.is_in_db(dbif) == False:
+ is_in_db = False
# Break in case no valid time is provided
if start == "" or start == None:
dbif.close()
@@ -186,17 +189,14 @@
else:
core.fatal(_("Unable to register %s map <%s>. The map has no valid time and the start time is not set.") % \
(map.get_type(), map.get_map_id() ))
- # Load the data from the grass file database
- map.load()
if unit:
map.set_time_to_relative()
else:
map.set_time_to_absolute()
-
- # Put it into the temporal database
- map.insert(dbif)
+
else:
+ is_in_db = True
map.select(dbif)
if name and map.get_temporal_type() != sp.get_temporal_type():
dbif.close()
@@ -206,20 +206,55 @@
core.fatal(_("Unable to register %s map <%s>. The temporal types are different.") % \
(map.get_type(), map.get_map_id()))
- # In case the time is in the input file we ignore the increment counter
- if start_time_in_file:
- count = 1
+ # Load the data from the grass file database
+ map.load()
# Set the valid time
if start:
- assign_valid_time_to_map(ttype=map.get_temporal_type(), map=map, start=start, end=end, unit=unit, increment=increment, mult=count, dbif=dbif, interval=interval)
+ # In case the time is in the input file we ignore the increment counter
+ if start_time_in_file:
+ count = 1
+ assign_valid_time_to_map(ttype=map.get_temporal_type(), map=map, start=start, end=end, unit=unit, increment=increment, mult=count, interval=interval)
- # Finally Register map in the space time dataset
+ if is_in_db:
+ # Gather the SQL update statement
+ statement += map.update_all(dbif=dbif, execute=True)
+ else:
+ # Gather the SQL insert statement
+ statement += map.insert(dbif=dbif, execute=False)
+
+ # Sqlite3 performace better for huge datasets when committing in small chunks
+ if dbmi.__name__ == "sqlite3":
+ if count % 100 == 0:
+ if statement != None and statement != "":
+ core.message(_("Registering 100 maps in the temporal database"))
+ execute_transaction(statement, dbif)
+ statement = ""
+
+ # Store the maps in a list to register in a space time dataset
if name:
- sp.register_map(map, dbif)
+ map_object_list.append(map)
+ core.percent(num_maps, num_maps, 1)
+
+ if statement != None and statement != "":
+ core.message(_("Register maps in the temporal database"))
+ execute_transaction(statement, dbif)
+
+ # Finally Register the maps in the space time dataset
+ if name:
+ statement = ""
+ count = 0
+ num_maps = len(map_object_list)
+ core.message(_("Register maps in the space time raster dataset"))
+ for map in map_object_list:
+ core.percent(count, num_maps, 1)
+ sp.register_map(map=map, dbif=dbif)
+ count += 1
+
# Update the space time tables
if name:
+ core.message(_("Update space time raster dataset"))
sp.update_from_registered_maps(dbif)
if connect == True:
@@ -230,7 +265,7 @@
###############################################################################
-def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=1, dbif = None, interval=False):
+def assign_valid_time_to_map(ttype, map, start, end, unit, increment=None, mult=1, interval=False):
"""!Assign the valid time to a map dataset
@param ttype: The temporal type which should be assigned and which the time format is of
@@ -240,24 +275,16 @@
@param unit: The unit of the relative time: years, months, days, hours, minutes, seconds
@param increment: Time increment between maps for time stamp creation (format absolute: NNN seconds, minutes, hours, days, weeks, months, years; format relative is integer 1)
@param multi: A multiplier for the increment
- @param dbif: The database interface to use for sql queries
@param interval: If True, time intervals are created in case the start time and an increment is provided
"""
-
- connect = False
- if dbif == None:
- dbif = sql_database_interface()
- dbif.connect()
- connect = True
-
if ttype == "absolute":
start_time = string_to_datetime(start)
if start_time == None:
dbif.close()
core.fatal(_("Unable to convert string \"%s\"into a datetime object")%(start))
end_time = None
-
+
if end:
end_time = string_to_datetime(end)
if end_time == None:
@@ -274,7 +301,7 @@
else:
core.verbose(_("Set absolute valid time for map <%s> to %s - %s") % (map.get_map_id(), str(start_time), str(end_time)))
- map.update_absolute_time(start_time, end_time, None, dbif)
+ map.set_absolute_time(start_time, end_time, None)
else:
start_time = int(start)
end_time = None
@@ -292,11 +319,8 @@
else:
core.verbose(_("Set relative valid time for map <%s> to %i - %s with unit %s") % (map.get_map_id(), start_time, str(end_time), unit))
- map.update_relative_time(start_time, end_time, unit, dbif)
+ map.set_relative_time(start_time, end_time, unit)
- if connect == True:
- dbif.close()
-
###############################################################################
def dataset_factory(type, id):
Modified: grass/trunk/temporal/t.register/test.t.register.vector.file.layer.sh
===================================================================
--- grass/trunk/temporal/t.register/test.t.register.vector.file.layer.sh 2012-04-02 14:03:09 UTC (rev 51244)
+++ grass/trunk/temporal/t.register/test.t.register.vector.file.layer.sh 2012-04-03 18:08:20 UTC (rev 51245)
@@ -51,19 +51,19 @@
# Test with input files
# File 1
t.register --v type=vect input=lidar_abs_ds1 file="${n1}" start="2001-01-01" increment="1 months"
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
# File 1
t.register --v type=vect input=lidar_abs_ds1 file="${n1}" start="2001-01-01"
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
# File 2
t.register --v type=vect input=lidar_abs_ds1 file="${n2}"
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
# File 2
t.register --v type=vect input=lidar_abs_ds1 -i file="${n2}" start=file increment="1 months"
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
# File 3
t.register --v type=vect input=lidar_abs_ds1 file="${n3}" start=file
-t.list type=vect columns=id,name,start_time,end_time where='name = "lidar_abs_1"'
+t.list type=vect columns=id,name,start_time,end_time where="name='lidar_abs_1'"
t.unregister --v type=vect maps=lidar_abs_1:1,lidar_abs_1:2,lidar_abs_1:3,lidar_abs_1:4,lidar_abs_1:5,lidar_abs_1:6
t.remove type=stvds input=lidar_abs_ds1
Modified: grass/trunk/temporal/t.register/test.t.register.vector.sh
===================================================================
--- grass/trunk/temporal/t.register/test.t.register.vector.sh 2012-04-02 14:03:09 UTC (rev 51244)
+++ grass/trunk/temporal/t.register/test.t.register.vector.sh 2012-04-03 18:08:20 UTC (rev 51245)
@@ -62,3 +62,6 @@
t.unregister --v type=vect maps=lidar_abs_1,lidar_abs_2,lidar_abs_3
t.remove --v type=stvds input=lidar_abs_ds1,lidar_abs_ds2,lidar_abs_ds3,lidar_abs_ds4,lidar_abs_ds5,lidar_abs_ds6,lidar_abs_ds7
t.unregister --v type=vect maps=lidar_abs_4,lidar_abs_5,lidar_abs_6
+g.remove vect=lidar_abs_1,lidar_abs_2,lidar_abs_3,lidar_abs_4,lidar_abs_5,lidar_abs_6
+
+
More information about the grass-commit
mailing list