[GRASS-SVN] r62689 - grass-addons/grass7/vector/v.stats
svn_grass at osgeo.org
svn_grass at osgeo.org
Sun Nov 9 10:22:26 PST 2014
Author: zarch
Date: 2014-11-09 10:22:26 -0800 (Sun, 09 Nov 2014)
New Revision: 62689
Modified:
grass-addons/grass7/vector/v.stats/imp_csv.py
grass-addons/grass7/vector/v.stats/v.stats.py
grass-addons/grass7/vector/v.stats/vstats.py
Log:
v.stats: bug fix and improvement
Modified: grass-addons/grass7/vector/v.stats/imp_csv.py
===================================================================
--- grass-addons/grass7/vector/v.stats/imp_csv.py 2014-11-09 18:19:12 UTC (rev 62688)
+++ grass-addons/grass7/vector/v.stats/imp_csv.py 2014-11-09 18:22:26 UTC (rev 62689)
@@ -38,31 +38,23 @@
SKIPSHP = ['area_id', ]
-#SKIPSHP = ['areas', 'red__n', 'red__min', 'red__max', 'red__range', 'red__sum',
-# 'red__mean', 'red__stddev', 'red__variance', 'red__cf_var',
-# 'red__first_quartile', 'red__median', 'red__third_quartile',
-# 'red__percentile_90', 'green__n', 'green__min', 'green__max',
-# 'green__range', 'green__mean', 'green__stddev', 'green__variance',
-# 'green__cf_var', 'green__sum', 'green__first_quartile',
-# 'green__median', 'green__third_quartile', 'green__percentile_90',
-# 'blue__n', 'blue__min', 'blue__max', 'blue__range', 'blue__mean',
-# 'blue__stddev', 'blue__variance', 'blue__cf_var', 'blue__sum',
-# 'blue__first_quartile', 'blue__median', 'blue__third_quartile',
-# 'blue__percentile_90']
-
#----------------------------------------
-def getrow(shpdata, rstdata, dim):
- drow = np.zeros((dim, ))
+def getrow(shpdata, rstdata, cols):
+ drow = np.zeros((len(cols), ))
lenght = len(shpdata)
- for i, rows in enumerate(zip(shpdata, *[rst[:, 1:] for rst in rstdata])):
+ for i, rows in enumerate(zip(shpdata, *rstdata)):
glg.G_percent(i, lenght, 2)
start = 0
for row in rows:
end = start + row.shape[0]
+ #print('---')
+ #for c, r in zip(cols[start:end], row):
+ # print(c[0], r)
drow[start:end] = row
start = end
+ #import ipdb; ipdb.set_trace()
yield drow
@@ -91,7 +83,8 @@
skipshp=None, skiprst=None,
shpcat=0, rstcat=0,
allshpn=ALLSHPN, allrstn=ALLRSTN,
- allshpt=ALLSHPT, allrstt=ALLRSTT, overwrite=False):
+ allshpt=ALLSHPT, allrstt=ALLRSTT, overwrite=False,
+ separator=';'):
prefixes = prefixes if prefixes else [csv[:-4] for csv in rstcsv]
skipshp = skipshp if skipshp else []
skiprst = skiprst if skiprst else []
@@ -100,65 +93,61 @@
print("Start loading data from:")
print(" - %s." % shpcsv, end='')
- shpdata = np.genfromtxt(shpcsv, delimiter=';', names=True,
- usecols=useshpcols,
+ shpdata = np.genfromtxt(shpcsv, delimiter=separator, usecols=useshpcols,
dtype=getcols(allshpn, allshpt, skipshp))
shpdata.sort(order='cat')
+ # remove negative categories
+ shpdata = shpdata[shpdata['cat'] > 0]
print(' Done.')
rstdata = []
for rst in rstcsv:
print(" - %s." % rst, end='')
- rstdata.append(np.genfromtxt(rst, delimiter='|', names=True,
- usecols=userstcols,
- missing_values=('nan', '-nan'),
- dtype=getcols(allrstn, allrstt,
- userstcols)))
- rstdata[-1].sort(order='zone')
+ rstd = np.genfromtxt(rst, delimiter=separator, names=True,
+ usecols=userstcols,
+ missing_values=('nan', '-nan'),
+ dtype=getcols(allrstn, allrstt, skiprst))
+ rstd.sort(order='zone')
+ rstdata.append(rstd[rstd['zone'] > 0])
print(' Done.')
+ npz = 'csvfile.npz'
+ print("Save arrays to: %s" % npz)
+ kwargs = {shpcsv: shpdata}
+ for csv, rst in zip(rstcsv, rstdata):
+ kwargs[csv] = rst
+ np.savez_compressed(npz, **kwargs)
+
print("Cheking categories and zones correspondance:")
for i, rst in enumerate(rstdata):
print(" - <%s>." % rstcsv[i], end='')
# if rstcsv[i] == 'median5_contr.csv':
-# import ipdb; ipdb.set_trace()
+ #import ipdb; ipdb.set_trace()
if not (shpdata['cat'] == rst['zone']).all():
msg = "The categories and the zones are not equal, in <%s>"
raise ValueError(msg % rstcsv[i])
print(' Ok.')
print("Conversion from record array to array")
- shpdata = np.array(shpdata.tolist())
+ newdtype = np.dtype([(n, '<f8') for n in shpdata.dtype.names])
+ shpdata = shpdata.astype(newdtype).view('<f8').reshape((len(shpdata), -1))
print('.', end='')
for i, rst in enumerate(rstdata):
- rstdata[i] = np.array(rst.tolist())
+ # convert to <f8 and remove the first column with the zones
+ newdtype = np.dtype([(n, '<f8') for n in rst.dtype.names][1:])
+ rstdata[i] = rst.astype(newdtype).view('<f8').reshape((len(rst), -1))
print('.', end='')
print('Done.')
# create the new table
if tab.exist():
tab.drop(force=True)
cols = gettablecols(prefixes, allshpn, allshpt, skipshp,
- allrstn, allrstt, skiprst)
+ # remove zone from raster
+ allrstn[1:], allrstt[1:], skiprst)
tab.create(cols)
cur = tab.conn.cursor()
print("Merge shape table with raster csv.")
- tab.insert(getrow(shpdata, rstdata, len(cols)), cursor=cur, many=True)
+ tab.insert(getrow(shpdata, rstdata, cols), cursor=cur, many=True)
tab.conn.commit()
print("%d rows inserted." % tab.n_rows())
return tab
-
-
-#link = None
-#with VectorTopo(VSEG, mode='r') as vect:
-# #link = update_cols(vect.table, CSV, PREFIX, allcsvcols=ALL,
-# # skipcsv=SKIPCSV, skipshp=SKIPSHP)
-# link = update_cols(vect.table, ECSV, PREFIX, allcsvcols=ALL,
-# skipcsv=SKIPCSV, skipshp=ESKIPSHP)
-# link.layer = vect.layer + 1
-#
-##----------------------------
-#with Vector(VSEG, mode='rw') as vect:
-# link = Link(layer=NEW_LAYER, name=NEW_NAME_LAYER, table=NEW_TABLE_NAME)
-# vect.dblinks.add(link)
-
-
Modified: grass-addons/grass7/vector/v.stats/v.stats.py
===================================================================
--- grass-addons/grass7/vector/v.stats/v.stats.py 2014-11-09 18:19:12 UTC (rev 62688)
+++ grass-addons/grass7/vector/v.stats/v.stats.py 2014-11-09 18:22:26 UTC (rev 62689)
@@ -3,7 +3,7 @@
#
############################################################################
#
-# MODULE: i.segment.hierarchical
+# MODULE: v.stats
#
# AUTHOR(S): Pietro Zambelli (University of Trento)
#
@@ -67,7 +67,7 @@
#% multiple: yes
#% description: Skip shape columns
#% required: no
-#% answer: label,non_null_cells,null_cells,mean_of_abs,sum,sum_abs
+#% answer: label,all_cells,non_null_cells,null_cells,mean_of_abs,sum,sum_abs
#%end
#%option
#% key: shpcsv
@@ -113,19 +113,27 @@
#% multiple: no
#% required: no
#%end
+#%option
+#% key: separator
+#% type: string
+#% description: New vector layer that will be add to the vector map
+#% multiple: no
+#% required: no
+#% answer: ;
+#%end
+#%option
+#% key: nprocs
+#% type: integer
+#% description: Number of process that will be used
+#% multiple: no
+#% required: no
+#% answer: 1
+#%end
#%flag
#% key: r
#% description: Read from existing CSV files
#%end
-
#-----------------------------------------------------
-"""
-# convert segments to vector
-r.to.vect input=seg_0.05 at pietro output=seg005 type=area
-v.category input=seg005 layer=1,2,3,4,5 output=seg_005 type=area option=transfer
-v.to.rast input=seg_005 output=vseg_005
-
-"""
import sys
import os
@@ -161,6 +169,7 @@
rstcsv = (opt['rstcsv'].split(',') if opt['rstcsv']
else [split(rst)[0] + '.csv' for rst in rasters])
zones = opt['zones'] if opt['zones'] else vname + '_zones'
+ nprocs = int(opt.get('nprocs', 1))
if rasters:
if rprefix and len(rasters) != len(rprefix):
raise
@@ -176,25 +185,26 @@
else vname + '_stats')
newtabname = opt['newtabname'] if opt['newtabname'] else vname + '_stats'
rstpercentile = float(opt['rstpercentile'])
+ separator = opt.get('separator', ';')
#
# compute
#
if not os.path.exists(shpcsv):
- get_shp_csv(opt['vector'], shpcsv, overwrite)
+ get_shp_csv(opt['vector'], shpcsv, overwrite, separator)
if not get_mapset_raster(zones):
get_zones(opt['vector'], zones, layer)
if not rstcsv or not os.path.exists(rstcsv[0]):
- get_rst_csv(rasters, zones, rstcsv, rstpercentile, overwrite)
+ get_rst_csv(rasters, zones, rstcsv, rstpercentile, overwrite,
+ nprocs, separator)
newlink = Link(newlayer, newlayername, newtabname)
newtab = newlink.table()
-
with Vector(vname, vmset, mode='r', layer=layer) as vct:
mode = 'r' if newlink in vct.dblinks else 'rw'
with VectorTopo(vname, vmset, mode=mode, layer=layer) as vct:
- update_cols(newtab, shpcsv, rstcsv, prefixes, skipshp, skiprst)
+ update_cols(newtab, shpcsv, rstcsv, prefixes, skipshp, skiprst, separator=separator)
if mode == 'rw':
# add the new link
Modified: grass-addons/grass7/vector/v.stats/vstats.py
===================================================================
--- grass-addons/grass7/vector/v.stats/vstats.py 2014-11-09 18:19:12 UTC (rev 62688)
+++ grass-addons/grass7/vector/v.stats/vstats.py 2014-11-09 18:22:26 UTC (rev 62689)
@@ -6,15 +6,15 @@
"""
from __future__ import print_function, division
import os
-from grass.pygrass.modules import Module
+from grass.pygrass.modules import Module, ParallelModuleQueue
-def get_shp_csv(vector, csv=None, overwrite=False):
+def get_shp_csv(vector, csv=None, overwrite=False, separator=';'):
vasts = Module('v.area.stats')
csv = vector + '.csv' if csv is None else csv
if os.path.exists(csv) and overwrite:
os.remove(csv)
- vasts(map=vector, output=csv, overwrite=overwrite)
+ vasts(map=vector, output=csv, overwrite=overwrite, separator=separator)
return csv
@@ -24,13 +24,14 @@
rclr = Module("r.colors", map=zones, color="random")
-def get_rst_csv(rasters, zones, csvfiles, percentile=90., overwrite=False):
- procs = []
+def get_rst_csv(rasters, zones, csvfiles, percentile=90., overwrite=False,
+ nprocs=1, separator=';'):
+ queue = ParallelModuleQueue(nprocs=nprocs)
for rast, csv in zip(rasters, csvfiles):
- procs.append(Module('r.univar2', map=rast, zones=zones,
- percentile=percentile, output=csv,
- overwrite=overwrite, flags='e', finish_=False))
+ print(rast, csv)
+ queue.put(Module('r.univar2', map=rast, zones=zones,
+ percentile=percentile, output=csv, separator=separator,
+ overwrite=overwrite, flags='et', run_=False))
# wait the end of all process
- for proc in procs:
- proc.popen.wait()
+ queue.wait()
return csvfiles
More information about the grass-commit
mailing list