[GRASS-SVN] r66297 - in grass-addons/grass7/raster: r.stream.variables r.stream.watersheds
svn_grass at osgeo.org
svn_grass at osgeo.org
Tue Sep 22 15:24:10 PDT 2015
Author: elselvaje
Date: 2015-09-22 15:24:10 -0700 (Tue, 22 Sep 2015)
New Revision: 66297
Modified:
grass-addons/grass7/raster/r.stream.variables/r.stream.variables
grass-addons/grass7/raster/r.stream.watersheds/r.stream.watersheds
Log:
change echos and other littel impruvement
Modified: grass-addons/grass7/raster/r.stream.variables/r.stream.variables
===================================================================
--- grass-addons/grass7/raster/r.stream.variables/r.stream.variables 2015-09-22 21:52:57 UTC (rev 66296)
+++ grass-addons/grass7/raster/r.stream.variables/r.stream.variables 2015-09-22 22:24:10 UTC (rev 66297)
@@ -49,16 +49,26 @@
#% type: string
#% key_desc: name
#% description: Provide the full folder path (same as for r.stream.watersheds)
-#% required:yes
+#% required:no
+#% answer: GISDBASE/folder_structure
#%end
#%option
+#% key: out_folder
+#% type: string
+#% key_desc: name
+#% description: Provide the full folder path for the output stream-specific variable
+#% required:no
+#% answer: GISDBASE/stream-specific_variables
+#%end
+
+#%option
#% key: output
#% type: string
#% key_desc: method
#% multiple: yes
-#% options: min,max,range,mean,stddev,coeff_var,sum
-#% description: Provide the output aggregation method for the stream-specific variable: upstream minimum, maximum, range, mean, standard deviation, coefficient of variation, sum. Output datatype is Int32
+#% options: cells,min,max,range,mean,stddev,coeff_var,sum
+#% description: Provide the output aggregation method for the stream-specific variable: upstream cells numbers, minimum, maximum, range, mean, standard deviation, coefficient of variation, sum. Output datatype is Int32
#% required:yes
#%end
@@ -137,6 +147,14 @@
export GIS_OPT_FOLDER=$GIS_OPT_FOLDER
fi
+if [ ${GIS_OPT_OUT_FOLDER} = "GISDBASE/stream-specific_variables" ] ; then
+ export GIS_OPT_OUT_FOLDER=$GISDBASE"/stream-specific_variables"
+else
+ export GIS_OPT_OUT_FOLDER=$GIS_OPT_OUT_FOLDER
+fi
+
+mkdir $GIS_OPT_OUT_FOLDER 2> /dev/null
+
# what to do in case of user break:
exitprocedure()
{
@@ -150,9 +168,9 @@
# delete intermediate files
rm -fr $GISDBASE/$LOCATION_NAME/sub_${GIS_OPT_AREA}*
-rm -fr $GIS_OPT_FOLDER/*digit4/*digit3/*digit2/*digit1/*.txt
-rm -fr $GIS_OPT_FOLDER/*digit4/*digit3/*digit2/*digit1/*.tif
-rm -fr $GIS_OPT_FOLDER/blockfile/stat_*.txt $GIS_OPT_FOLDER/blockfile/*.tif
+find $GIS_OPT_FOLDER/ -maxdepth 1 -name '*.txt' -delete
+find $GIS_OPT_FOLDER/ -maxdepth 1 -name '*.tif' -delete
+rm -fr $GIS_OPT_FOLDER/blockfile/stat_*.txt
exit 1
}
@@ -161,11 +179,13 @@
trap "exitprocedure" 2 3 9 15 19
echo ""
-echo "Streams variables calculation based on the sub-watershads and sub-streams."
+echo "##########################################################################"
+echo "Streams variables calculation based on the sub-watershads and sub-streams"
echo ""
-echo "Citation: Domisch, S., Amatulli, G., Jetz, W. (2015)"
+echo "Citation: Domisch, S., Amatulli, G., Jetz, W. (in review)"
echo "Near-global freshwater-specific environmental variables for"
echo "biodiversity analyses in 1km resolution. Scientific Data"
+echo "##########################################################################"
echo ""
export GISRC_def=$GISRC
@@ -252,7 +272,6 @@
echo "Check for missing files due to RAM overload"
-
if ls $DIRNAME/sub_${GIS_OPT_AREA}ID*.tif 1> /dev/null 2>&1 ; then
echo $(ls $DIRNAME/sub_${GIS_OPT_AREA}ID*.tif | wc -l ) files had RAM issues
@@ -262,6 +281,8 @@
file=$1
filename=$(basename $file .tif )
+echo Fix missing file $file using only 1 CPU
+
if [ $GIS_OPT_AREA = "watershed" ] ; then ID=${filename:15:99}; fi
if [ $GIS_OPT_AREA = "stream" ] ; then ID=${filename:12:99}; fi
@@ -324,10 +345,11 @@
echo "ID|non_null_cells|null_cells|min|max|range|mean|mean_of_abs|stddev|variance|coeff_var|sum|sum_abs" > $GIS_OPT_FOLDER/blockfile/stat_${GIS_OPT_VARIABLE}.txt
cat $GIS_OPT_FOLDER/*digit4/*digit3/*digit2/*digit1/stat_${GIS_OPT_VARIABLE}.txt >> $GIS_OPT_FOLDER/blockfile/stat_${GIS_OPT_VARIABLE}.txt
-echo reclass the grid_id for the following output ${GIS_OPT_OUTPUT//","/" "}
+echo Reclass the grid_id for the following output ${GIS_OPT_OUTPUT//","/" "}
echo ${GIS_OPT_OUTPUT//","/" "} | xargs -n 1 -P $GIS_OPT_CPU bash -c $'
+if [ $1 = "cells" ] ; then col=2 ; fi
if [ $1 = "min" ] ; then col=4 ; fi
if [ $1 = "max" ] ; then col=5 ; fi
if [ $1 = "range" ] ; then col=6 ; fi
@@ -342,6 +364,7 @@
echo ${GIS_OPT_OUTPUT//","/" "} | xargs -n 1 -P 1 bash -c $'
+if [ $1 = "cells" ] ; then col=2 ; fi
if [ $1 = "min" ] ; then col=4 ; fi
if [ $1 = "max" ] ; then col=5 ; fi
if [ $1 = "range" ] ; then col=6 ; fi
@@ -355,11 +378,13 @@
r.reclass input=grid_ID output=${GIS_OPT_VARIABLE}_$1 rules=$GIS_OPT_FOLDER/blockfile/stat_${GIS_OPT_VARIABLE}_${1}.txt --overwrite --q
rm -f $GIS_OPT_FOLDER/blockfile/stat_${GIS_OPT_VARIABLE}_${1}.txt
-r.out.gdal -c input=${GIS_OPT_VARIABLE}_${1} nodata=-9999 output=$GIS_OPT_FOLDER/blockfile/${GIS_OPT_VARIABLE}_${1}.tif type=Int32 -c createopt="COMPRESS=LZW,ZLEVEL=9" --o --q 2> /dev/null
+r.out.gdal -c input=${GIS_OPT_VARIABLE}_${1} nodata=-9999 output=$GIS_OPT_OUT_FOLDER/${GIS_OPT_VARIABLE}_${1}.tif type=Int32 -c createopt="COMPRESS=LZW,ZLEVEL=9" --o --q 2> /dev/null
echo "$GIS_OPT_FOLDER/blockfile/${GIS_OPT_VARIABLE}_${1}.tif has been created!"
' _
+exit
+
echo "The full stream variable calculation has been done!"
-echo "To list the stream-specific output variables, use ls $GIS_OPT_FOLDER/blockfile/*.tif"
+echo "To list the stream-specific output variables, use ls $GIS_OPT_OUT_FOLDER/*.tif"
Modified: grass-addons/grass7/raster/r.stream.watersheds/r.stream.watersheds
===================================================================
--- grass-addons/grass7/raster/r.stream.watersheds/r.stream.watersheds 2015-09-22 21:52:57 UTC (rev 66296)
+++ grass-addons/grass7/raster/r.stream.watersheds/r.stream.watersheds 2015-09-22 22:24:10 UTC (rev 66297)
@@ -22,7 +22,7 @@
#############################################################################
#%Module
-#% description: Sub-watershed delineation based on the drainage direction and a gridded stream network. Based on "Domisch, S., Amatulli, G., Jetz, W. (2015) Near-global freshwater-specific environmental variables for biodiversity analyses in 1km resolution. Scientific Data"
+#% description: Sub-watershed and sub-stream delineation based on the drainage direction and a gridded stream network.
#% keywords: drainage, stream, hydrology
#%End
@@ -46,12 +46,11 @@
#% key: folder
#% type: string
#% key_desc: name
-#% description: Provide the full folder path and name where the sub-watersheds and sub-streams will be stored
+#% description: Provide the full folder path and name where the sub-watersheds and sub-streams should be stored
#% required:no
#% answer: GISDBASE/folder_structure
#%end
-
#%option
#% key: cpu
#% type: double
@@ -136,17 +135,20 @@
# shell check for user break (signal list: trap -l)
# trap "exitprocedure" 2 3 15
+
echo ""
+echo "#######################################################################################################"
echo "Sub-watershed and sub-streams delineation based on the drainage direction and a gridded stream network."
echo ""
-echo "Citation: Domisch, S., Amatulli, G., Jetz, W. (2015)"
+echo "Citation: Domisch, S., Amatulli, G., Jetz, W. (in review)"
echo "Near-global freshwater-specific environmental variables for"
echo "biodiversity analyses in 1km resolution. Scientific Data"
+echo "#######################################################################################################"
echo ""
echo Exporting stream grid cell coordinates and saving in $GIS_OPT_FOLDER/stream_coord_lines.txt
-rm -fr $GIS_OPT_FOLDER 2> /dev/null
-mkdir $GIS_OPT_FOLDER 2> /dev/null
+rm -fr $GIS_OPT_FOLDER
+mkdir $GIS_OPT_FOLDER
r.out.xyz --o input=$GIS_OPT_STREAM separator=space output=$GIS_OPT_FOLDER/stream_coord.txt --q
@@ -173,25 +175,25 @@
max_seq4d=${max_line: -8:1} ; if [ -z $max_seq4d ] ; then max_seq4d=0 ; else max_seq3d=9 ; max_seq2d=9 ; max_seq1d=9 ; fi
for dir4d in $(seq 0 $max_seq4d) ; do
-for dir3d in $(seq 0 $max_seq3d) ; do
-for dir2d in $(seq 0 $max_seq2d) ; do
-for dir1d in $(seq 0 $max_seq1d) ; do
-mkdir -p $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1
-done
-done
+ for dir3d in $(seq 0 $max_seq3d) ; do
+ for dir2d in $(seq 0 $max_seq2d) ; do
+ for dir1d in $(seq 0 $max_seq1d) ; do
+ mkdir -p $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1
+ done
+ done
+ done
done
-done
mkdir $GIS_OPT_FOLDER/blockfile
-echo "Split the stream grid cell coordinate file with $( wc -l $GIS_OPT_FOLDER/stream_coord.txt | awk '{ print $1 }' ) grid cells into blocks of 10000 cells"
+echo "Split the stream grid cell coordinate file (X|Y|ID) into subsets of 10000 lines"
awk -v PATH=$GIS_OPT_FOLDER 'NR%10000==1{x=PATH"/blockfile/blockfile"(++i*10000-10000)}{print > x}' $GIS_OPT_FOLDER/stream_coord_lines.txt
awk '{ if(NR>1) print }' $GIS_OPT_FOLDER/blockfile/blockfile0 > $GIS_OPT_FOLDER/blockfile/blockfile0_tmp
mv $GIS_OPT_FOLDER/blockfile/blockfile0_tmp $GIS_OPT_FOLDER/blockfile/blockfile0
-echo Create $( ls $GIS_OPT_FOLDER/blockfile/blockfile* | wc -l ) blocks in $GIS_OPT_FOLDER/blockfile/
+echo Create $( ls $GIS_OPT_FOLDER/blockfile/blockfile* | wc -l ) subsets: $GIS_OPT_FOLDER/blockfile/blockfile*.tif
rm -fr $GISDBASE/$LOCATION_NAME/sub_watershedID*
@@ -216,7 +218,7 @@
g.gisenv set="LOCATION_NAME=$LOCATION_NAME"
g.gisenv set="MAPSET=PERMANENT"
-echo "Start the sub-watershed delineation for block $file"
+echo "Start the sub-watershed delineation for subset $file"
awk '{ print NR , $1 , $2 ,$3 }' $file | xargs -n 4 -P $GIS_OPT_CPU bash -c $'
NR=$1
@@ -224,13 +226,10 @@
Y_coord=$3
ID=$4
+echo -en "\r$(expr $NR / 100 + 1 )% done"
-if [ $( expr $( expr $NR + 1 ) / 100) -eq $( expr $NR / 100 + 1 ) ] ; then
-echo -en "\r$(expr $NR / 100 + 1 )% done"
-fi
+# replicate the start-GISRC in a unique GISRC
-# replicate the start-GISRC in a uniq GISRC
-
cp $GISRC_def $HOME/.grass7/rc$ID
export GISRC=$HOME/.grass7/rc$ID
@@ -256,8 +255,6 @@
r.null map=sub_streamID${ID} setnull=0 --q
r.out.gdal -c type=Byte --o --q nodata=255 createopt="COMPRESS=LZW,ZLEVEL=9" input=sub_streamID${ID}@sub_watershedID${ID} output=$GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/sub_streamID${ID}.tif
-if [ ! -f $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/sub_streamID${ID}.tif ] ; then the file $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/sub_streamID${ID}.tif dose not exist ; fi
-
rm -r $HOME/.grass7/rc$ID $GISDBASE/$LOCATION_NAME/sub_watershedID$ID
' _
@@ -320,30 +317,20 @@
' _
-
-
rm -fr $GISDBASE/$LOCATION_NAME/sub_watershedID*
echo "Compress the sub-watersheds and sub-streams to reduce the number of inodes filling up the hard disk"
-cd $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/
+tar -zcPf $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/${filename}_sub_watershed.tar.gz $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/sub_watershedID*.tif
+tar -zcPf $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/${filename}_sub_stream.tar.gz $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/sub_streamID*.tif
-tar -zcPf ${filename}_sub_watershed.tar.gz sub_watershedID*.tif
-tar -zcPf ${filename}_sub_stream.tar.gz sub_streamID*.tif
+echo list the compressed files by ls $GIS_OPT_FOLDER/*digit4/*digit3/*digit2/*digit1/*.tar.gz
-cd $GIS_OPT_FOLDER
-
g.gisenv set="MAPSET=PERMANENT"
g.gisenv set="LOCATION_NAME=$LOCATION_NAME"
g.gisenv set="GISDBASE=$GISDBASE"
-rm -fr $GISDBASE/$LOCATION_NAME/sub_watershedID* $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/*.tif
+rm -fr $GISDBASE/$LOCATION_NAME/sub_watershedID* $GIS_OPT_FOLDER/${dir4d}digit4/${dir3d}digit3/${dir2d}digit2/${dir1d}digit1/*.tif
done
-cd $GIS_OPT_FOLDER
-
-echo The full sub-watershed delineation process has been done!
-echo To list the compressed files, use "ls $GIS_OPT_FOLDER/*digit4/*digit3/*digit2/*digit1/*.tar.gz"
-echo Now you can use r.stream.variables to compute stream-specific environmental variables.
-
More information about the grass-commit
mailing list