[GRASS-dev] -s flag in temporal modules
Luca Delucchi
lucadeluge at gmail.com
Thu Apr 7 13:57:00 PDT 2016
On 24 February 2016 at 15:55, Vaclav Petras <wenzeslaus at gmail.com> wrote:
>
> On Wed, Feb 24, 2016 at 9:42 AM, Sören Gebbert
> <soerengebbert at googlemail.com> wrote:
>>
>> We surely do not need ISO conform time suffixes for raster and vector
>> names, since they are not SQL compliant.
>
>
>
> Good point. Not having colon in the file name and vector name is actually a
> good idea. Underscores are only save separator.
Hi guys,
I'm back on this thread because in Paris I worked more on this and I
could summit changes for map algebra and map calc. There are few error
in the testsuite but I don't know where they come from.
could someone test the attached diff and give some feedback?
--
ciao
Luca
http://gis.cri.fmach.it/delucchi/
www.lucadelu.org
-------------- next part --------------
Index: lib/python/temporal/mapcalc.py
===================================================================
--- lib/python/temporal/mapcalc.py (revision 68030)
+++ lib/python/temporal/mapcalc.py (working copy)
@@ -18,7 +18,7 @@
############################################################################
-def dataset_mapcalculator(inputs, output, type, expression, base, method,
+def dataset_mapcalculator(inputs, output, type, expression, base, tsuffix, method,
nprocs=1, register_null=False, spatial=False):
"""Perform map-calculations of maps from different space time
raster/raster3d datasets, using a specific sampling method
@@ -68,7 +68,8 @@
:param type: The type of the dataset: "raster" or "raster3d"
:param expression: The r(3).mapcalc expression
:param base: The base name of the new created maps in case a
- mapclac expression is provided
+ mapclac expression is provided
+ :param tsuffix: The suffix to use
:param method: The method to be used for temporal sampling
:param nprocs: The number of parallel processes to be used for
mapcalc processing
@@ -200,8 +201,16 @@
msgr.percent(count, num, 1)
# Create the r.mapcalc statement for the current time step
- map_name = "{base}_{suffix}".format(base=base,
- suffix=gscript.get_num_suffix(count, num))
+ if first_input.get_temporal_type() == 'absolute' and tsuffix == 'gran':
+ suffix = create_suffix_from_datetime(first_input.temporal_extent.get_start_time(),
+ granularity)
+ map_name = "{ba}_{su}@{ma}".format(ba=base, su=suffix)
+ elif first_input.get_temporal_type() == 'absolute' and tsuffix == 'time':
+ suffix = create_time_suffix(map_matrix[0][i])
+ map_name = "{ba}_{su}@{ma}".format(ba=base, su=suffix)
+ else:
+ map_name = create_numeric_suffic(base, count, tsuffix)
+
# Remove spaces and new lines
expr = expression.replace(" ", "")
Index: lib/python/temporal/temporal_raster3d_algebra.py
===================================================================
--- lib/python/temporal/temporal_raster3d_algebra.py (revision 68030)
+++ lib/python/temporal/temporal_raster3d_algebra.py (working copy)
@@ -25,7 +25,7 @@
self.m_mapcalc = pymod.Module('r3.mapcalc')
self.m_mremove = pymod.Module('g.remove')
- def parse(self, expression, basename = None, overwrite=False):
+ def parse(self, expression, basename=None, tsuffix=None, overwrite=False):
# Check for space time dataset type definitions from temporal algebra
l = TemporalRasterAlgebraLexer()
l.build()
@@ -48,6 +48,7 @@
self.maptype = "raster_3d"
self.mapclass = Raster3DDataset
self.basename = basename
+ self.suffix = tsuffix
self.expression = expression
self.parser.parse(expression)
Index: lib/python/temporal/temporal_raster_algebra.py
===================================================================
--- lib/python/temporal/temporal_raster_algebra.py (revision 68030)
+++ lib/python/temporal/temporal_raster_algebra.py (working copy)
@@ -65,7 +65,7 @@
self.m_mapcalc = pymod.Module('r.mapcalc')
self.m_mremove = pymod.Module('g.remove')
- def parse(self, expression, basename = None, overwrite=False):
+ def parse(self, expression, basename=None, tsuffix=None, overwrite=False):
# Check for space time dataset type definitions from temporal algebra
l = TemporalRasterAlgebraLexer()
l.build()
@@ -88,6 +88,7 @@
self.maptype = "raster"
self.mapclass = RasterDataset
self.basename = basename
+ self.suffix = tsuffix
self.expression = expression
self.parser.parse(expression)
Index: lib/python/temporal/temporal_raster_base_algebra.py
===================================================================
--- lib/python/temporal/temporal_raster_base_algebra.py (revision 68030)
+++ lib/python/temporal/temporal_raster_base_algebra.py (working copy)
@@ -44,6 +44,10 @@
import grass.pygrass.modules as pymod
from temporal_operator import *
from temporal_algebra import *
+from temporal_granularity import compute_absolute_time_granularity
+from datetime_math import create_suffix_from_datetime
+from datetime_math import create_time_suffix
+from datetime_math import create_numeric_suffic
##############################################################################
@@ -539,12 +543,29 @@
process_queue = pymod.ParallelModuleQueue(int(self.nprocs))
if isinstance(t[3], list):
num = len(t[3])
+ if num > 0:
+ if t[3][0].get_temporal_type() == 'absolute':
+ granularity = compute_absolute_time_granularity(t[3])
count = 0
returncode = 0
register_list = []
for i in range(num):
# Check if resultmap names exist in GRASS database.
- map_name = self.basename + "_" + str(i) + "@" + self.mapset
+ if t[3][i].get_temporal_type() == 'absolute' and self.suffix == 'gran':
+ suffix = create_suffix_from_datetime(t[3][i].temporal_extent.get_start_time(),
+ granularity)
+ map_name = "{ba}_{su}@{ma}".format(ba=self.basename,
+ su=suffix,
+ ma=self.mapset)
+ elif t[3][i].get_temporal_type() == 'absolute' and self.suffix == 'time':
+ suffix = create_time_suffix(t[3][i])
+ map_name = "{ba}_{su}@{ma}".format(ba=self.basename,
+ su=suffix,
+ ma=self.mapset)
+ else:
+ name = create_numeric_suffic(self.basename,
+ count, self.suffix)
+ map_name = name + "@" + self.mapset
if self.stdstype == "strds":
new_map = RasterDataset(map_name)
else:
@@ -554,7 +575,18 @@
%(map_name))
map_test_list = []
for map_i in t[3]:
- newident = self.basename + "_" + str(count)
+ if map_i.get_temporal_type() == 'absolute' and self.suffix == 'gran':
+ suffix = create_suffix_from_datetime(map_i.temporal_extent.get_start_time(),
+ granularity)
+ newident = "{ba}_{su}".format(ba=self.basename,
+ su=suffix)
+ elif map_i.get_temporal_type() == 'absolute' and self.suffix == 'time':
+ suffix = create_time_suffix(map_i)
+ newident = "{ba}_{su}".format(ba=self.basename,
+ su=suffix)
+ else:
+ newident = create_numeric_suffic(self.basename,
+ count, self.suffix)
if "cmd_list" in dir(map_i):
# Build r.mapcalc module and execute expression.
# Change map name to given basename.
Index: lib/python/temporal/temporal_vector_algebra.py
===================================================================
--- lib/python/temporal/temporal_vector_algebra.py (revision 68030)
+++ lib/python/temporal/temporal_vector_algebra.py (working copy)
@@ -120,7 +120,7 @@
self.m_mremove = pygrass.Module('g.remove', quiet=True, run_=False)
self.m_buffer = pygrass.Module('v.buffer', quiet=True, run_=False)
- def parse(self, expression, basename = None, overwrite = False):
+ def parse(self, expression, basename=None, suffix=None, overwrite=False):
# Check for space time dataset type definitions from temporal algebra
l = TemporalVectorAlgebraLexer()
l.build()
@@ -143,6 +143,7 @@
self.maptype = "vector"
self.mapclass = VectorDataset
self.basename = basename
+ self.suffix = suffix
self.expression = expression
self.parser.parse(expression)
@@ -361,13 +362,28 @@
dbif, connected = init_dbif(dbif=self.dbif)
if isinstance(t[3], list):
num = len(t[3])
+ granularity = compute_absolute_time_granularity(t[3])
count = 0
returncode = 0
register_list = []
for i in range(num):
# Check if resultmap names exist in GRASS database.
- vectorname = self.basename + "_" + str(i)
- vectormap = VectorDataset(vectorname + "@" + get_current_mapset())
+ if t[3][i].get_temporal_type() == 'absolute' and self.suffix == 'gran':
+ suffix = create_suffix_from_datetime(t[3][i].temporal_extent.get_start_time(),
+ granularity)
+ map_name = "{ba}_{su}@{ma}".format(ba=self.basename,
+ su=suffix,
+ ma=self.mapset)
+ elif t[3][i].get_temporal_type() == 'absolute' and self.suffix == 'time':
+ suffix = create_time_suffix(t[3][i])
+ map_name = "{ba}_{su}@{ma}".format(ba=self.basename,
+ su=suffix,
+ ma=self.mapset)
+ else:
+ name = create_numeric_suffic(self.basename,
+ count, self.suffix)
+ map_name = name + "@" + self.mapset
+ vectormap = VectorDataset(map_name)
if vectormap.map_exists() and self.overwrite == False:
self.msgr.fatal(_("Error vector maps with basename %s exist. "
"Use --o flag to overwrite existing file") \
@@ -418,7 +434,18 @@
self.removable_maps.pop(map_i.get_name())
mapset = map_i.get_mapset()
# Change map name to given basename.
- newident = self.basename + "_" + str(count)
+ if map_i.get_temporal_type() == 'absolute' and self.suffix == 'gran':
+ suffix = create_suffix_from_datetime(map_i.temporal_extent.get_start_time(),
+ granularity)
+ newident = "{ba}_{su}".format(ba=self.basename,
+ su=suffix)
+ elif map_i.get_temporal_type() == 'absolute' and self.suffix == 'time':
+ suffix = create_time_suffix(map_i)
+ newident = "{ba}_{su}".format(ba=self.basename,
+ su=suffix)
+ else:
+ newident = create_numeric_suffic(self.basename,
+ count, self.suffix)
m = copy.deepcopy(self.m_rename)
m.inputs["vector"].value = (map_i.get_name(),newident)
m.flags["overwrite"].value = self.overwrite
@@ -435,7 +462,18 @@
map_test_extent = map_test.get_temporal_extent_as_tuple()
if map_test_extent != map_i_extent:
# Create new map with basename
- newident = self.basename + "_" + str(count)
+ if map_i.get_temporal_type() == 'absolute' and self.suffix == 'gran':
+ suffix = create_suffix_from_datetime(map_i.temporal_extent.get_start_time(),
+ granularity)
+ newident = "{ba}_{su}".format(ba=self.basename,
+ su=suffix)
+ elif map_i.get_temporal_type() == 'absolute' and self.suffix == 'time':
+ suffix = create_time_suffix(map_i)
+ newident = "{ba}_{su}".format(ba=self.basename,
+ su=suffix)
+ else:
+ newident = create_numeric_suffic(self.basename,
+ count, self.suffix)
map_result = map_i.get_new_instance(newident + "@" + self.mapset)
if map_test.map_exists() and self.overwrite == False:
Index: temporal/t.rast.algebra/t.rast.algebra.py
===================================================================
--- temporal/t.rast.algebra/t.rast.algebra.py (revision 68030)
+++ temporal/t.rast.algebra/t.rast.algebra.py (working copy)
@@ -40,6 +40,15 @@
#%end
#%option
+#% key: suffix
+#% type: string
+#% description: Suffix to add at basename: set 'gran' for granularity, 'time' for the full time format, 'num' for numerical suffix with a specific number of digits (default %05)
+#% answer: gran
+#% required: no
+#% multiple: no
+#%end
+
+#%option
#% key: nprocs
#% type: integer
#% description: Number of r.mapcalc processes to run in parallel
@@ -71,6 +80,7 @@
def main():
expression = options['expression']
basename = options['basename']
+ suffix = options['suffix']
nprocs = options["nprocs"]
spatial = flags["s"]
register_null = flags["n"]
@@ -92,7 +102,7 @@
if not p.setup_common_granularity(expression=expression, lexer = tgis.TemporalRasterAlgebraLexer()):
grass.script.fatal(_("Unable to process the expression in granularity algebra mode"))
- p.parse(expression, basename, grass.script.overwrite())
+ p.parse(expression, basename, suffix, grass.script.overwrite())
if __name__ == "__main__":
options, flags = grass.script.parser()
Index: temporal/t.rast.algebra/testsuite/test_raster_algebra.py
===================================================================
--- temporal/t.rast.algebra/testsuite/test_raster_algebra.py (revision 68030)
+++ temporal/t.rast.algebra/testsuite/test_raster_algebra.py (working copy)
@@ -44,7 +44,7 @@
tgis.open_new_stds(name="B", type="strds", temporaltype="absolute",
title="B", descr="B", semantic="field")
tgis.open_new_stds(name="C", type="strds", temporaltype="absolute",
- title="B", descr="C", semantic="field")
+ title="C", descr="C", semantic="field")
tgis.open_new_stds(name="D", type="strds", temporaltype="absolute",
title="D", descr="D", semantic="field")
@@ -238,7 +238,23 @@
start, end = D.get_absolute_time()
self.assertEqual(start, datetime.datetime(2001, 1, 1))
self.assertEqual(end, datetime.datetime(2001, 1, 5))
-
+
+ def test_simple_arith_3_suffix_num(self):
+ """Simple arithmetic test"""
+
+ self.assertModule("t.rast.algebra", expression="R = A / A + A*A/A",
+ basename="r", suffix="num%03")
+
+ self.assertRasterExists('r_001')
+
+ def test_simple_arith_3_suffix_time(self):
+ """Simple arithmetic test"""
+
+ self.assertModule("t.rast.algebra", expression="R = A / A + A*A/A",
+ basename="r", suffix="time")
+
+ self.assertRasterExists('r_2001_01_01T00_00_00')
+
def test_temporal_intersection_1(self):
"""Simple temporal intersection test"""
@@ -512,7 +528,8 @@
"""Testing the temporal select operator. Including temporal relations and
different temporal operators (lr|+&)"""
- self.assertModule("t.rast.algebra", expression="R = A {:,during,d} B", basename="r")
+ self.assertModule("t.rast.algebra", expression="R = A {:,during,d} B",
+ basename="r", suffix="num%01")
D = tgis.open_old_stds("R", type="strds")
@@ -529,7 +546,8 @@
"""Testing the temporal select operator. Including temporal relations and
different temporal operators (lr|+&)"""
- self.assertModule("t.rast.algebra", expression="R = A {:,equal|during,r} C", basename="r")
+ self.assertModule("t.rast.algebra", expression="R = A {:,equal|during,r} C",
+ basename="r", suffix="time")
D = tgis.open_old_stds("R", type="strds")
Index: temporal/t.rast.mapcalc/t.rast.mapcalc.py
===================================================================
--- temporal/t.rast.mapcalc/t.rast.mapcalc.py (revision 68030)
+++ temporal/t.rast.mapcalc/t.rast.mapcalc.py (working copy)
@@ -49,6 +49,15 @@
#%end
#%option
+#% key: suffix
+#% type: string
+#% description: Suffix to add at basename: set 'gran' for granularity, 'time' for the full time format, 'num' for numerical suffix with a specific number of digits (default %05)
+#% answer: gran
+#% required: no
+#% multiple: no
+#%end
+
+#%option
#% key: nprocs
#% type: integer
#% description: Number of r.mapcalc processes to run in parallel
@@ -67,8 +76,6 @@
#% description: Check spatial overlap
#%end
-from multiprocessing import Process
-import copy
import grass.script as grass
import grass.temporal as tgis
@@ -82,6 +89,7 @@
output = options["output"]
expression = options["expression"]
base = options["basename"]
+ suffix = options["suffix"]
method = options["method"]
nprocs = int(options["nprocs"])
register_null = flags["n"]
@@ -93,8 +101,8 @@
# Make sure the temporal database exists
tgis.init()
- tgis.dataset_mapcalculator(inputs, output, "raster", expression,
- base, method, nprocs, register_null, spatial)
+ tgis.dataset_mapcalculator(inputs, output, "raster", expression, base,
+ suffix, method, nprocs, register_null, spatial)
###############################################################################
Index: temporal/t.rast3d.algebra/t.rast3d.algebra.py
===================================================================
--- temporal/t.rast3d.algebra/t.rast3d.algebra.py (revision 68030)
+++ temporal/t.rast3d.algebra/t.rast3d.algebra.py (working copy)
@@ -41,6 +41,15 @@
#%end
#%option
+#% key: suffix
+#% type: string
+#% description: Suffix to add at basename: set 'gran' for granularity, 'time' for the full time format, 'num' for numerical suffix with a specific number of digits (default %05)
+#% answer: gran
+#% required: no
+#% multiple: no
+#%end
+
+#%option
#% key: nprocs
#% type: integer
#% description: Number of r3.mapcalc processes to run in parallel
@@ -72,6 +81,7 @@
def main():
expression = options['expression']
basename = options['basename']
+ suffix = options['suffix']
nprocs = options["nprocs"]
spatial = flags["s"]
register_null = flags["n"]
@@ -93,7 +103,7 @@
if not p.setup_common_granularity(expression=expression, stdstype = 'str3ds', lexer = tgis.TemporalRasterAlgebraLexer()):
grass.script.fatal(_("Unable to process the expression in granularity algebra mode"))
- p.parse(expression, basename, grass.script.overwrite())
+ p.parse(expression, basename, suffix, grass.script.overwrite())
if __name__ == "__main__":
options, flags = grass.script.parser()
Index: temporal/t.rast3d.algebra/testsuite/test_raster3d_algebra.py
===================================================================
--- temporal/t.rast3d.algebra/testsuite/test_raster3d_algebra.py (revision 68030)
+++ temporal/t.rast3d.algebra/testsuite/test_raster3d_algebra.py (working copy)
@@ -94,7 +94,21 @@
self.assertEqual(start, datetime.datetime(2001, 1, 2))
self.assertEqual(end, datetime.datetime(2001, 1, 4))
+# def test_temporal_neighbors_time_suffix(self):
+# """Simple temporal neighborhood computation test"""
+#
+# self.assertModule("t.rast3d.algebra", expression='D = A[-1] + A[1]',
+# basename="d", suffix="time")
+# self.assertRasterExists('d_2001_01_01T00_00_00')
+#
+# def test_temporal_neighbors_num_suffix(self):
+# """Simple temporal neighborhood computation test"""
+#
+# self.assertModule("t.rast3d.algebra", expression='D = A[-1] + A[1]',
+# basename="d", suffix="num%02")
+# self.assertRasterExists('d_01')
+
class TestTRast3dAlgebraFails(TestCase):
@classmethod
Index: temporal/t.rast3d.mapcalc/t.rast3d.mapcalc.py
===================================================================
--- temporal/t.rast3d.mapcalc/t.rast3d.mapcalc.py (revision 68030)
+++ temporal/t.rast3d.mapcalc/t.rast3d.mapcalc.py (working copy)
@@ -53,6 +53,15 @@
#%end
#%option
+#% key: suffix
+#% type: string
+#% description: Suffix to add at basename: set 'gran' for granularity, 'time' for the full time format, 'num' for numerical suffix with a specific number of digits (default %05)
+#% answer: gran
+#% required: no
+#% multiple: no
+#%end
+
+#%option
#% key: nprocs
#% type: integer
#% description: Number of r3.mapcalc processes to run in parallel
@@ -71,8 +80,6 @@
#% description: Check spatial overlap
#%end
-from multiprocessing import Process
-import copy
import grass.script as grass
import grass.temporal as tgis
@@ -86,6 +93,7 @@
output = options["output"]
expression = options["expression"]
base = options["basename"]
+ suffix = options["suffix"]
method = options["method"]
nprocs = int(options["nprocs"])
register_null = flags["n"]
@@ -97,8 +105,8 @@
# Make sure the temporal database exists
tgis.init()
- tgis.dataset_mapcalculator(inputs, output, "raster3d", expression,
- base, method, nprocs, register_null, spatial)
+ tgis.dataset_mapcalculator(inputs, output, "raster3d", expression, base,
+ suffix, method, nprocs, register_null, spatial)
###############################################################################
Index: temporal/t.vect.algebra/t.vect.algebra.py
===================================================================
--- temporal/t.vect.algebra/t.vect.algebra.py (revision 68030)
+++ temporal/t.vect.algebra/t.vect.algebra.py (working copy)
@@ -41,6 +41,15 @@
#% required : yes
#%end
+#%option
+#% key: suffix
+#% type: string
+#% description: Suffix to add at basename: set 'gran' for granularity, 'time' for the full time format, 'num' for numerical suffix with a specific number of digits (default %05)
+#% answer: gran
+#% required: no
+#% multiple: no
+#%end
+
#%flag
#% key: s
#% description: Activate spatial topology
@@ -53,6 +62,7 @@
def main():
expression = options['expression']
basename = options['basename']
+ suffix = options['suffix']
spatial = flags["s"]
stdstype = "stvds"
@@ -65,7 +75,7 @@
tgis.init(True)
p = tgis.TemporalVectorAlgebraParser(run = True, debug=False, spatial = spatial)
- p.parse(expression, basename, grass.script.overwrite())
+ p.parse(expression, basename, suffix, grass.script.overwrite())
if __name__ == "__main__":
options, flags = grass.script.parser()
More information about the grass-dev
mailing list