[GRASS-SVN] r50386 - in grass-addons/grass7/general: . g.cloud
g.cloud/test
svn_grass at osgeo.org
svn_grass at osgeo.org
Mon Jan 23 04:14:47 EST 2012
Author: lucadelu
Date: 2012-01-23 01:14:47 -0800 (Mon, 23 Jan 2012)
New Revision: 50386
Added:
grass-addons/grass7/general/g.cloud/
grass-addons/grass7/general/g.cloud/Makefile
grass-addons/grass7/general/g.cloud/cloud_collect.sh
grass-addons/grass7/general/g.cloud/cloud_mail.sh
grass-addons/grass7/general/g.cloud/cloud_ssh.py
grass-addons/grass7/general/g.cloud/cloud_unpack.py
grass-addons/grass7/general/g.cloud/cloud_which.py
grass-addons/grass7/general/g.cloud/g.cloud.html
grass-addons/grass7/general/g.cloud/g.cloud.py
grass-addons/grass7/general/g.cloud/test/
grass-addons/grass7/general/g.cloud/test/launch_SGE_grassjob.sh
grass-addons/grass7/general/g.cloud/test/test_morevariables.sh
grass-addons/grass7/general/g.cloud/test/test_morevariables_raster.sh
grass-addons/grass7/general/g.cloud/test/test_novariables.sh
grass-addons/grass7/general/g.cloud/test/test_onevariable.sh
grass-addons/grass7/general/g.cloud/test/test_onevariable_raster.sh
grass-addons/grass7/general/g.cloud/test/test_onevariable_sun.sh
Log:
add g.cloud to the addons repository
Added: grass-addons/grass7/general/g.cloud/Makefile
===================================================================
--- grass-addons/grass7/general/g.cloud/Makefile (rev 0)
+++ grass-addons/grass7/general/g.cloud/Makefile 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,16 @@
+MODULE_TOPDIR = ../..
+
+PGM = g.cloud
+
+include $(MODULE_TOPDIR)/include/Make/Script.make
+
+SRCFILES = cloud_which.py cloud_ssh.py cloud_unpack.py cloud_collect.sh cloud_mail.sh
+DSTFILES := $(patsubst %,$(ETC)/g.cloud/%,$(SRCFILES))
+
+default: script $(DSTFILES)
+
+$(ETC)/g.cloud/%: % | $(ETC)/g.cloud
+ $(INSTALL) $< $@
+
+$(ETC)/g.cloud:
+ $(MKDIR) $@
Added: grass-addons/grass7/general/g.cloud/cloud_collect.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/cloud_collect.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/cloud_collect.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,145 @@
+#!/bin/sh
+
+############################################################################
+#
+# MODULE: cloud_collect.sh
+# AUTHOR(S): Markus Neteler
+# Modify by Luca Delucchi for g.cloud
+#
+# PURPOSE: Copy all maps from the temporaneous Grid Engine mapsets into target mapset
+#
+# COPYRIGHT: (C) 2011 by Markus Neteler
+#
+# This program is free software under the GNU General Public
+# License (>=v2). Read the file COPYING that comes with GRASS
+# for details.
+#
+#############################################################################
+
+## Grid Engine settings
+# request Bourne shell as shell for job
+#$ -S /bin/sh
+
+# run in current working directory
+#$ -cwd
+
+## DEBUG
+#set -x
+#echo "Using blade: `uname -n`" 1>&2
+
+if [ $# -ne 5 ] ; then
+ echo "Script to move Grid Engine job results to GRASS target mapset"
+ echo ""
+ echo "Usage: $0 targetgrassdbase targetlocation mail_address remove_file"
+ echo ""
+ echo "targetgrassdbase = GRASSDBASE where write files"
+ echo "targetlocation = LOCATION where write files"
+ echo "mail_address = Mail address where send email when jobs finish, NOOO to no mail"
+ echo "remove_file = Remove temporary files, NOOO to no remove"
+ echo ""
+ echo "Example:"
+ echo " $0 /grassdata patUTM32 launch_SGE_grassjob_rsun_energy.sh"
+ exit 0
+fi
+
+#### start of GRASS 7 setup
+# better say where to find libs and bins:
+export PATH=$PATH:/usr/local/bin:$HOME
+export LD_LIBRARY_PATH=$MYLD_LIBRARY_PATH
+MYGRASS_ADDON_PATH=$HOME/.grass7/addons/scripts/
+
+# define location to work in
+GRASSDBROOT=$1
+MYLOC=$2
+MAILADDR=$3
+REMOVE=$4
+CLOUDID=$5
+
+MYMAPSET="PERMANENT"
+
+# path to GRASS binaries and libraries:
+export GISBASE=`grass70 --config path`
+export PATH=$PATH:$GISBASE/bin:$GISBASE/scripts
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib64:$GISBASE/lib
+
+# use process ID (PID) as GRASS lock file number:
+export GIS_LOCK=$$
+export GRASS_MESSAGE_FORMAT=plain
+export TERM=linux
+
+# path to GRASS settings file
+mkdir -p $HOME/.grass7/
+MYGISRC="$HOME/.grass7/rc.gcollectorjob.$MYPID"
+
+#generate GISRCRC
+#echo "GRASS_ADDON_PATH: $MYGRASS_ADDON_PATH" > "$MYGISRC"
+echo "GISDBASE: $GRASSDBROOT" >> "$MYGISRC"
+echo "LOCATION_NAME: $MYLOC" >> "$MYGISRC"
+echo "MAPSET: $MYMAPSET" >> "$MYGISRC"
+echo "GRASS_GUI: text" >> "$MYGISRC"
+
+# path to GRASS settings file
+export GISRC=$MYGISRC
+
+#### end of GRASS 7 setup
+
+echo $MYPID 1>&2
+
+
+TOCOLLECTFILE=to_be_collected_$MYPID.csv
+
+ls -1 $GRASSDBROOT/$MYLOC/sge*/${MYPID} > $GRASSDBROOT/$MYLOC/$TOCOLLECTFILE
+
+rm -f $GRASSDBROOT/$MYLOC/clean_$TOCOLLECTFILE
+for myname in `cat $GRASSDBROOT/$MYLOC/$TOCOLLECTFILE` ; do
+ basename `dirname $myname` >> $GRASSDBROOT/$MYLOC/clean_$TOCOLLECTFILE
+done
+
+LIST=`cat $GRASSDBROOT/$MYLOC/clean_$TOCOLLECTFILE`
+
+echo $LIST 1>&2
+
+
+for mapset in $LIST ; do
+ MAPS=`g.mlist rast mapset=$mapset`
+ for map in $MAPS ; do
+ g.copy rast=$map@$mapset,$map --o
+ done
+done
+
+for mapset in $LIST ; do
+ MAPS=`g.mlist rast3d mapset=$mapset`
+ for map in $MAPS ; do
+ g.copy rast=$map@$mapset,$map --o
+ done
+done
+
+for mapset in $LIST ; do
+ MAPS=`g.mlist vect mapset=$mapset`
+ for map in $MAPS ; do
+ g.copy vect=$map@$mapset,$map --o
+ done
+done
+
+if [ "$REMOVE" != "NOOO" ] ; then
+ rm -f $GRASSDBROOT/$MYLOC/$MYMAPSET/.gislock
+ rm -f $GRASSDBROOT/$MYLOC/clean_$TOCOLLECTFILE
+ rm -f $GRASSDBROOT/$MYLOC/$TOCOLLECTFILE
+
+ if [ ! -z "$GRASSDBROOT" ] ; then
+ if [ ! -z "$MYLOC" ] ; then
+ rm -fr $GRASSDBROOT/$MYLOC/sge*
+ fi
+ fi
+fi
+
+if [ "$MAILADDR" != "NOOO" ] ; then
+ sh -x cloud_mail.sh $MAILADDR
+fi
+
+rm -f $MYGISRC
+
+touch cloud_finish_${CLOUDID}
+
+exit 0
+
Added: grass-addons/grass7/general/g.cloud/cloud_mail.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/cloud_mail.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/cloud_mail.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+FROMMAIL="me at somewhere.com" # TO BE SET
+
+echo "$1 The jobs that you submitted have finished" | mail -s "Jobs finished" $1
Added: grass-addons/grass7/general/g.cloud/cloud_ssh.py
===================================================================
--- grass-addons/grass7/general/g.cloud/cloud_ssh.py (rev 0)
+++ grass-addons/grass7/general/g.cloud/cloud_ssh.py 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+############################################################################
+#
+# MODULE: ssh_session.py
+# AUTHOR(S): Eric S. Raymond
+#
+# Greatly modified by Nigel W. Moriarty, April 2003
+# Modified by Luca Delucchi 2011
+#
+# PURPOSE: establish a SSH session to a remote system
+#
+# COPYRIGHT: (C) 2011 by Eric S. Raymond, Nigel W. Moriarty, Luca Delucchi
+#
+# This program is free software under the GNU General Public
+# License (>=v2). Read the file COPYING that comes with GRASS
+# for details.
+#
+#############################################################################
+
+from pexpect import *
+import os, sys, subprocess
+import getpass
+import time
+
+class ssh_session:
+
+ """Session with extra state including the password to be used"""
+
+ def __init__(self, user, host, gsession, password=None, verbose=0):
+
+ self.user = user
+ self.host = host
+ self.verbose = verbose
+ self.password = password
+ self.gsession = gsession
+ self.openagent = False
+ self.keys = [
+ 'authenticity',
+ 'password:',
+ 'Enter passphrase',
+ '@@@@@@@@@@@@',
+ 'Command not found.',
+ EOF,
+ ]
+ # set the home path
+ home = os.path.expanduser('~')
+ logfile = os.path.join(home, self.gsession,'g.cloud','ssh.log')
+ self.f = open(logfile,'w')
+
+ def __repr__(self):
+ outl = 'class :'+self.__class__.__name__
+ for attr in self.__dict__:
+ if attr == 'password':
+ outl += '\n\t'+attr+' : '+'*'*len(self.password)
+ else:
+ outl += '\n\t'+attr+' : '+str(getattr(self, attr))
+ return outl
+
+ def __exec(self, command):
+ """Execute a command on the remote host. Return the output."""
+
+ child = spawn(command #, timeout=10
+ )
+ if self.verbose:
+ sys.stderr.write("-> " + command + "\n")
+ seen = child.expect(self.keys)
+ self.f.write(str(child.before) + str(child.after)+'\n')
+ if seen == 0:
+ child.sendline('yes')
+ seen = child.expect(self.keys)
+ if seen == 1 or seen == 2:
+ if not self.password:
+ self.password = getpass.getpass('Remote password: ')
+ child.sendline(self.password)
+ child.readline()
+ time.sleep(5)
+ # Added to allow the background running of remote process
+ if not child.isalive():
+ seen = child.expect(self.keys)
+ if seen == 3:
+ lines = child.readlines()
+ self.f.write(lines)
+ if self.verbose:
+ sys.stderr.write("<- " + child.before + "|\n")
+ try:
+ self.f.write(str(child.before) + str(child.after)+'\n')
+ except:
+ pass
+ #self.f.close()
+ return child.before
+
+ def ssh(self, command):
+ """Function to launch command with ssh"""
+ return self.__exec("ssh -Y -l %s %s \"%s\"" % (self.user,self.host,command))
+
+ def scp(self, src, dst):
+ """Function to move data from client to server"""
+ return self.__exec("scp %s %s@%s:%s" % (src, self.user, self.host, dst))
+
+ def pcs(self, src, dst):
+ """Function to move data from server to client"""
+ return self.__exec("scp %s@%s:%s %s" % (self.user, self.host, src, dst))
+
+ def add(self):
+ """Function to launch ssh-add"""
+ sess = self.__exec("ssh-add")
+ if sess.find('Could not open') == -1:
+ return 0
+ else:
+ self.openagent = True
+ proc = subprocess.Popen(['ssh-agent', '-s'], stdout=subprocess.PIPE)
+ output = proc.stdout.read()
+ vari = output.split('\n')
+ sshauth = vari[0].split(';')[0].split('=')
+ sshpid = vari[1].split(';')[0].split('=')
+ os.putenv(sshauth[0],sshauth[1])
+ os.putenv(sshpid[0],sshpid[1])
+ self.add()
+
+ def close(self):
+ """Close connection"""
+ if self.openagent:
+ subprocess.Popen(['ssh-agent', '-k'], stdout=subprocess.PIPE )
+ return self.f.close()
Added: grass-addons/grass7/general/g.cloud/cloud_unpack.py
===================================================================
--- grass-addons/grass7/general/g.cloud/cloud_unpack.py (rev 0)
+++ grass-addons/grass7/general/g.cloud/cloud_unpack.py 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+############################################################################
+#
+# MODULE: cloud_unpack.py
+# AUTHOR(S): Luca Delucchi
+# PURPOSE: It is used to unpack raster and vector maps by g.cloud module
+#
+# COPYRIGHT: (C) 2011 by Luca Delucchi
+#
+# This program is free software under the GNU General Public
+# License (>=v2). Read the file COPYING that comes with GRASS
+# for details.
+#
+#############################################################################
+
+import glob, os, sys, tarfile, logging
+import cloud_which as which
+
+# the home of the user
+homeServer = os.getcwd()
+
+f = open(os.path.join(homeServer, 'unpackwrite.log'), 'w')
+
+LOG_FILENAME = os.path.join(homeServer, 'unpack.log')
+LOGGING_FORMAT='%(asctime)s - %(levelname)s - %(message)s'
+logging.basicConfig(filename=LOG_FILENAME, level=logging.DEBUG, \
+ format=LOGGING_FORMAT)
+
+if len(sys.argv) != 2:
+ logging.error('Usage: python %s GISDBASE' % sys.argv[0])
+ f.write('Usage: python %s GISDBASE' % sys.argv[0])
+
+# the full path to GISDBASE, LOCATION_NAME, MAPSET
+dbaselocatmap = sys.argv[1].strip('').rstrip('\/')
+mapset = os.path.split(dbaselocatmap)[1]
+location = os.path.split(os.path.split(dbaselocatmap)[0])[1]
+dbase = os.path.split(os.path.split(dbaselocatmap)[0])[0]
+
+logging.debug("Unpacking in <%s/%s/%s>" % (dbase, location, mapset))
+f.write("Unpacking in <%s/%s/%s>" % (dbase, location, mapset))
+
+# read for grass70 executable the path to gisbase
+for line in open(which.which('grass70')).readlines():
+ if line.startswith(' gisbase = "'):
+ gisbase = line.split('=')[-1].split('"')[1]
+# look for raster and vector pack
+raster = glob.glob1(homeServer, "rastertarpack")
+vector = glob.glob1(homeServer, "vectortarpack")
+
+logging.debug("Found: %s raster pack, %s vector pack" % (raster, vector))
+f.write("Found: %s raster pack, %s vector pack" % (raster, vector))
+## add some environment variables
+os.environ['GISBASE'] = gisbase
+
+sys.path.append(os.path.join(gisbase, "etc","python"))
+
+import grass.script as grass
+import grass.script.setup as gsetup
+
+gsetup.init(gisbase, dbase, location, mapset)
+
+grass.run_command('db.connect', flags='p')
+kv = grass.db_connection()
+database = kv['database']
+driver = kv['driver']
+logging.debug("db.connect: driver: %s, database %s" % (driver, database))
+
+# unpack raster and vector maps
+if len(raster) != 0:
+ try:
+ tar = tarfile.TarFile.open(name = 'rastertarpack', mode = 'r')
+ tar.extractall()
+ rasters = glob.glob1(homeServer, "*.rasterpack")
+ except:
+ logging.error("Error unpacking rastertarpack")
+ f.write("Error unpacking rastertarpack")
+ for i in rasters:
+ logging.debug("Unpacking raster map <%s>" % os.path.join(homeServer,i))
+ ret = grass.run_command('r.unpack',input=os.path.join(homeServer,i))
+ if ret != 0:
+ logging.error("Error unpacking raster map <%s>" % os.path.join(homeServer,i))
+ #os.remove(os.path.join(homeServer,i)) TO UNCOMMENT WHEN ALL WILL BE OK
+
+if len(vector) != 0:
+ try:
+ tar = tarfile.TarFile.open(name = 'vectortarpack', mode = 'r')
+ tar.extractall()
+ vectors = glob.glob1(homeServer, "*.vectorpack")
+ except:
+ logging.error("Error unpacking vectortarpack")
+ f.write("Error unpacking vectortarpack")
+
+ for i in vectors:
+ logging.debug("Unpacking vector map <%s>" % os.path.join(homeServer,i))
+ ret = grass.run_command('v.unpack',input=os.path.join(homeServer,i))
+ if ret != 0:
+ logging.error("Error unpacking raster map <%s>" % os.path.join(homeServer,i))
+ #os.remove(os.path.join(homeServer,i)) TO UNCOMMENT WHEN ALL WILL BE OK
Added: grass-addons/grass7/general/g.cloud/cloud_which.py
===================================================================
--- grass-addons/grass7/general/g.cloud/cloud_which.py (rev 0)
+++ grass-addons/grass7/general/g.cloud/cloud_which.py 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,225 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2002-2005 ActiveState Corp.
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# http://code.google.com/p/which/
+
+import os
+import sys
+import getopt
+import stat
+#---- exceptions
+
+class WhichError(Exception):
+ pass
+
+
+
+#---- internal support stuff
+
+def _getRegisteredExecutable(exeName):
+ """Windows allow application paths to be registered in the registry."""
+ registered = None
+ if sys.platform.startswith('win'):
+ if os.path.splitext(exeName)[1].lower() != '.exe':
+ exeName += '.exe'
+ import _winreg
+ try:
+ key = "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\App Paths\\" +\
+ exeName
+ value = _winreg.QueryValue(_winreg.HKEY_LOCAL_MACHINE, key)
+ registered = (value, "from HKLM\\"+key)
+ except _winreg.error:
+ pass
+ if registered and not os.path.exists(registered[0]):
+ registered = None
+ return registered
+
+def _samefile(fname1, fname2):
+ if sys.platform.startswith('win'):
+ return ( os.path.normpath(os.path.normcase(fname1)) ==\
+ os.path.normpath(os.path.normcase(fname2)) )
+ else:
+ return os.path.samefile(fname1, fname2)
+
+def _cull(potential, matches, verbose=0):
+ """Cull inappropriate matches. Possible reasons:
+ - a duplicate of a previous match
+ - not a disk file
+ - not executable (non-Windows)
+ If 'potential' is approved it is returned and added to 'matches'.
+ Otherwise, None is returned.
+ """
+ for match in matches: # don't yield duplicates
+ if _samefile(potential[0], match[0]):
+ if verbose:
+ sys.stderr.write("duplicate: %s (%s)\n" % potential)
+ return None
+ else:
+ if not stat.S_ISREG(os.stat(potential[0]).st_mode):
+ if verbose:
+ sys.stderr.write("not a regular file: %s (%s)\n" % potential)
+ elif sys.platform != "win32" \
+ and not os.access(potential[0], os.X_OK):
+ if verbose:
+ sys.stderr.write("no executable access: %s (%s)\n"\
+ % potential)
+ else:
+ matches.append(potential)
+ return potential
+
+
+#---- module API
+
+def whichgen(command, path=None, verbose=0, exts=None):
+ """Return a generator of full paths to the given command.
+
+ "command" is a the name of the executable to search for.
+ "path" is an optional alternate path list to search. The default it
+ to use the PATH environment variable.
+ "verbose", if true, will cause a 2-tuple to be returned for each
+ match. The second element is a textual description of where the
+ match was found.
+ "exts" optionally allows one to specify a list of extensions to use
+ instead of the standard list for this system. This can
+ effectively be used as an optimization to, for example, avoid
+ stat's of "foo.vbs" when searching for "foo" and you know it is
+ not a VisualBasic script but ".vbs" is on PATHEXT. This option
+ is only supported on Windows.
+
+ This method returns a generator which yields either full paths to
+ the given command or, if verbose, tuples of the form (<path to
+ command>, <where path found>).
+ """
+ matches = []
+ if path is None:
+ usingGivenPath = 0
+ path = os.environ.get("PATH", "").split(os.pathsep)
+ if sys.platform.startswith("win"):
+ path.insert(0, os.curdir) # implied by Windows shell
+ else:
+ usingGivenPath = 1
+
+ # Windows has the concept of a list of extensions (PATHEXT env var).
+ if sys.platform.startswith("win"):
+ if exts is None:
+ exts = os.environ.get("PATHEXT", "").split(os.pathsep)
+ # If '.exe' is not in exts then obviously this is Win9x and
+ # or a bogus PATHEXT, then use a reasonable default.
+ for ext in exts:
+ if ext.lower() == ".exe":
+ break
+ else:
+ exts = ['.COM', '.EXE', '.BAT']
+ elif not isinstance(exts, list):
+ raise TypeError("'exts' argument must be a list or None")
+ else:
+ if exts is not None:
+ raise WhichError("'exts' argument is not supported on "\
+ "platform '%s'" % sys.platform)
+ exts = []
+
+ # File name cannot have path separators because PATH lookup does not
+ # work that way.
+ if os.sep in command or os.altsep and os.altsep in command:
+ if os.path.exists(command):
+ match = _cull((command, "explicit path given"), matches, verbose)
+ if verbose:
+ yield match
+ else:
+ yield match[0]
+ else:
+ for i in range(len(path)):
+ dirName = path[i]
+ # On windows the dirName *could* be quoted, drop the quotes
+ if sys.platform.startswith("win") and len(dirName) >= 2\
+ and dirName[0] == '"' and dirName[-1] == '"':
+ dirName = dirName[1:-1]
+ for ext in ['']+exts:
+ absName = os.path.abspath(
+ os.path.normpath(os.path.join(dirName, command+ext)))
+ if os.path.isfile(absName):
+ if usingGivenPath:
+ fromWhere = "from given path element %d" % i
+ elif not sys.platform.startswith("win"):
+ fromWhere = "from PATH element %d" % i
+ elif i == 0:
+ fromWhere = "from current directory"
+ else:
+ fromWhere = "from PATH element %d" % (i-1)
+ match = _cull((absName, fromWhere), matches, verbose)
+ if match:
+ if verbose:
+ yield match
+ else:
+ yield match[0]
+ match = _getRegisteredExecutable(command)
+ if match is not None:
+ match = _cull(match, matches, verbose)
+ if match:
+ if verbose:
+ yield match
+ else:
+ yield match[0]
+
+
+def which(command, path=None, verbose=0, exts=None):
+ """Return the full path to the first match of the given command on
+ the path.
+
+ "command" is a the name of the executable to search for.
+ "path" is an optional alternate path list to search. The default it
+ to use the PATH environment variable.
+ "verbose", if true, will cause a 2-tuple to be returned. The second
+ element is a textual description of where the match was found.
+ "exts" optionally allows one to specify a list of extensions to use
+ instead of the standard list for this system. This can
+ effectively be used as an optimization to, for example, avoid
+ stat's of "foo.vbs" when searching for "foo" and you know it is
+ not a VisualBasic script but ".vbs" is on PATHEXT. This option
+ is only supported on Windows.
+
+ If no match is found for the command, a WhichError is raised.
+ """
+ try:
+ match = whichgen(command, path, verbose, exts).next()
+ except StopIteration:
+ raise WhichError("Could not find '%s' on the path." % command)
+ return match
+
+def whichall(command, path=None, verbose=0, exts=None):
+ """Return a list of full paths to all matches of the given command
+ on the path.
+
+ "command" is a the name of the executable to search for.
+ "path" is an optional alternate path list to search. The default it
+ to use the PATH environment variable.
+ "verbose", if true, will cause a 2-tuple to be returned for each
+ match. The second element is a textual description of where the
+ match was found.
+ "exts" optionally allows one to specify a list of extensions to use
+ instead of the standard list for this system. This can
+ effectively be used as an optimization to, for example, avoid
+ stat's of "foo.vbs" when searching for "foo" and you know it is
+ not a VisualBasic script but ".vbs" is on PATHEXT. This option
+ is only supported on Windows.
+ """
+ return list( whichgen(command, path, verbose, exts) )
Added: grass-addons/grass7/general/g.cloud/g.cloud.html
===================================================================
--- grass-addons/grass7/general/g.cloud/g.cloud.html (rev 0)
+++ grass-addons/grass7/general/g.cloud/g.cloud.html 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,75 @@
+<h2>DESCRIPTION</h2>
+
+<em>g.cloud</em> is a module to connect a personal GRASS GIS session to a remote
+GRASS GIS installation on a much faster cluster facility (high performance computing
+system). Actually it support only Grid Engine system. The general aim is to offer
+the possibility to a user to perform high speed calculations, which are hard to
+compute on a personal computer by connecting to an external, powerful server or
+cloud computing system.
+
+<p>
+The <em>g.cloud</em> environment consists of a Python module. It checks if all
+program requirements are fullfilled on both the client and the server side.
+If all it is ok, it connects the current session to the cloud server; it transfers
+the user and other required scripts as well as the GRASS data to the server and
+eventually executes the remote GRASS GIS job. Subsequently, it transfers back
+the resulting maps and data and cleans up the server session.
+
+<p>
+For a job execution, the script requirements are two: on the client-side the user
+prepared GRASS GIS script which performs the calculations and on the server-side
+the <em>qsub</em> control script for Grid Engine to launch the GRASS GIS script job.
+
+<h2>NOTES</h2>
+
+It is highly recommended to use "ssh-add" in order to avoid the authentication
+to the cluster via password file.
+
+<h2>EXAMPLE</h2>
+
+North Carolina example, calculation of texture maps:
+<p>
+<div class="code"><pre>
+g.cloud config=$HOME/.gc_loginfile.txt server=giscluster \
+ grass_script=$HOME/g.cloud/test/test_onevariable_raster.sh \
+ qsub_script=$HOME/g.cloud/test/launch_SGE_grassjob.sh \
+ variables="{'TEXTURE' : ['asm','corr','entr','se','var']}" \
+ raster=lsat7_2002_40
+</pre></div>
+<p>
+
+North Carolina example, calculation of daily sun radiation:
+<p>
+<div class="code"><pre>
+g.cloud config=$HOME/.gc_loginfile.txt server=giscluster \
+ grass_script=$HOME/g.cloud/test/test_onevariable_sun.sh \
+ qsub_script=$HOME/test/launch_SGE_grassjob.sh \
+ variables="{'DOY' : [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215,
216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325]}" \
+ raster=elevation
+</pre></div>
+
+<h2>REQUIREMENTS</h2>
+
+The software requirements are GRASS GIS 7 and <em>ssh/scp</em> on the client side; while
+on the server side the Grid Engine software or compatible software needs to be present
+to launch the GRASS jobs. Furthermore, <em>python-pexpect</em> is required to be installed
+on the client system.
+<p>
+
+<!--
+To facilitate the usage, the <em>g.cloud.prep</em> module,
+if it does not find a GRASS GIS 7 installation on the server, it will download and
+install it automatically.
+-->
+
+<h2>SEE ALSO</h2>
+
+<a href="http://www.slideshare.net/lucadelu/grass-cloud">FOSS4G 2011 presentation</a>
+
+<h2>AUTHORS</h2>
+
+Luca Delucchi, Markus Neteler, Fondazione Edmund Mach, Research and Innovation Centre,
+Department of Biodiversity and Molecular Ecology,
+<a href="http://gis.cri.fmach.it">GIS and Remote Sensing Unit</a>
+
+<p><i>Last changed: $Date: 2012-01-10 14:03:09 +0100 (Tue, 10 Jan 2012) $</i>
Added: grass-addons/grass7/general/g.cloud/g.cloud.py
===================================================================
--- grass-addons/grass7/general/g.cloud/g.cloud.py (rev 0)
+++ grass-addons/grass7/general/g.cloud/g.cloud.py 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,433 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+############################################################################
+#
+# MODULE: g.cloud
+# AUTHOR(S): Luca Delucchi, Markus Neteler
+# PURPOSE: g.cloud give the possibility to connect a local GRASS session with
+# another one in a cluster system (it use qsub)
+#
+# COPYRIGHT: (C) 2011 by Luca Delucchi
+#
+# This program is free software under the GNU General Public
+# License (>=v2). Read the file COPYING that comes with GRASS
+# for details.
+#
+#############################################################################
+#%module
+#% description: Connect your GRASS session with another one in a cluster system
+#% keywords: general
+#%end
+#%flag
+#% key: c
+#% description: Cycle through all variables
+#%end
+#%flag
+#% key: k
+#% description: Keep temporal files and mapsets
+#%end
+#%flag
+#% key: a
+#% description: Use ssh-add for faster access (requires to launch ssh-agent beforehand)
+#%end
+#%option
+#% key: config
+#% type: string
+#% gisprompt: old,file,input
+#% label: Path to ASCII file containing authentication parameters
+#% description: "-" to pass the parameters interactively
+#% required: yes
+#% guisection: Define
+#%end
+#%option
+#% key: server
+#% type: string
+#% key_desc: name
+#% description: Name or IP of server to be connected
+#% required: yes
+#%end
+#%option
+#% key: grass_script
+#% type: string
+#% key_desc: name
+#% description: Path to the input GRASS script
+#% required: no
+#% gisprompt: old,file,input
+#%end
+#%option
+#% key: qsub_script
+#% type: string
+#% key_desc: name
+#% description: Path to the input qsub script
+#% required: no
+#% gisprompt: old,file,input
+#%end
+#%option
+#% key: variables
+#% type: string
+#% key_desc: name
+#% label: Python dictionary with the variables to pass to the GRASS script via qsub
+#% description: Example: "{'year':[2010,2011]}" will call 'qsub -v year=2010 ...' and 'qsub -v year=2011 ...'
+#% required: no
+#%end
+#%option G_OPT_R_INPUTS
+#% key: raster
+#% description: Name of input raster map(s) used by GRASS script
+#% required: no
+#%end
+#%option G_OPT_V_INPUTS
+#% key: vector
+#% label: {NULL}
+#% description: Name of input vector map(s) used by GRASS script
+#% required: no
+#%end
+#%option
+#% key: mail
+#% type: string
+#% required: no
+#% key_desc: name
+#% description: Mail address to send emails at the end of jobs
+#%end
+#%option
+#% key: reconnect
+#% type: string
+#% required: no
+#% key_desc: name
+#% description: Reconnect with old job to see if it is finished
+#%end
+#%option
+#% key: path
+#% type: string
+#% key_desc: name
+#% label: Path for temporal for g.cloud operations
+#% description: The directory must be visible on the network to the blades
+#% required: no
+#% answer: $HOME
+#%end
+
+# import library
+import os, sys, pexpect, tarfile, ast, tempfile, getpass, itertools, collections, stat
+from types import *
+import grass.script as grass
+
+python = os.getenv('GRASS_PYTHON', 'python')
+cloudpath = os.path.join(os.getenv('GISBASE'), 'etc', 'g.cloud')
+sys.path.append(cloudpath)
+
+try:
+ import cloud_ssh as sshs
+except ImportError:
+ pass
+
+def transposed(lists):
+ """ Function to transpose list of variables """
+ if not lists: return []
+ return map(lambda *row: list(row), *lists)
+
+def _iteration(lis):
+ """Interation inside the lists of values"""
+ if len(lis) == 1:
+ return lis[0]
+ else:
+ out = [i for i in itertools.product(lis[0], _iteration(lis[1:]))]
+ return out
+
+def _flatten(lis):
+ """Used after iteration to create a good list"""
+ for el in lis:
+ if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
+ for sub in _flatten(el):
+ #return but keep the state of the function
+ yield sub
+ else:
+ yield el
+
+def serverCheck(conn, grassV):
+ """ Function to check if the server has GRASS installed and return the home
+ of user used to connect to the server
+ """
+ test_grass = conn.ssh('which %s' % grassV)
+ test_qsub = conn.ssh('which %s' % 'qsub')
+ if test_qsub == '' or test_qsub.find('no %s in' % 'qsub') != -1:
+ grass.fatal(_("On the server '%s' is not installed. Please install it first." % 'qsub'))
+ if test_grass == '' or test_grass.find('no %s in' % grassV) != -1:
+ grass.warning(_("On the server '%s' is not installed. Please install it first" % grassV ))
+ # return the home path of the user on the cluster
+ loc = conn.ssh('pwd')
+ return loc.strip()
+
+def copyMaps(conn, infiles, typ, home):
+ """ Function to copy maps to the cluster server using r.pack or v.pack"""
+ if typ == 'raster':
+ ele = 'cell'
+ com = 'r.pack'
+ elif typ == 'vector':
+ ele = 'vector'
+ com = 'v.pack'
+ tar = tarfile.TarFile.open(name = typ + 'tarpack', mode = 'w')
+ for i in infiles:
+ gfile = grass.find_file(name = i, element = ele)
+ if not gfile['name']:
+ grass.fatal(_("%s map <%s> not found") % (typ,i))
+ else:
+ outputpack = "%s.%spack" % (i, typ)
+ grass.run_command(com, input = i, output = outputpack, overwrite = True)
+ tar.add(outputpack)
+ tar.close()
+ conn.scp(typ + 'tarpack', home)
+ for i in infiles:
+ os.remove("%s.%spack" % (i, typ))
+ os.remove(typ + 'tarpack')
+
+def variablesCheck(listValue):
+ """Function to check if all variables as the same length and
+ return the values in a useful list"""
+ if type(listValue[0]) == ListType:
+ oldlen = len(listValue[0])
+ else:
+ grass.fatal(_('Values must be a Python list'))
+ # check if all variables have the same lenght
+ for i in listValue:
+ if oldlen != len(i):
+ grass.fatal(_('Attention: the lists of values have different length\n' \
+ 'All the variables have to have the same number of values'))
+ return transposed(listValue)
+
+def variablesCheckCicle(listValue):
+ """Return the values in a useful list"""
+ vals = _iteration(listValue)
+ out = []
+ for v in vals:
+ v = list(_flatten(v))
+ out.append(v)
+ return out
+
+def reconnect(conn, pid, path, vari, home):
+ collect_file_name = "cloud_finish_%s" % options['reconnect']
+ gcloud_home = os.path.join(home, 'gcloud%s' % options['reconnect'])
+ collect_file = conn.ssh('"cd %s; ls %s"'% (gcloud_home, collect_file_name))
+ output_file = os.path.join(home, "gcloud_result_%s.tar.gz" % options['reconnect'])
+ if collect_file.strip() == collect_file_name:
+ location_name = conn.ssh('"cd %s; ls"'%
+ os.path.join(home, 'grassdata%s' % options['reconnect']))
+ location_name = location_name.strip()
+ mapset_name = os.path.join(home,"grassdata%s" % options['reconnect'],
+ location_name,"PERMANENT")
+ grass.message(_("Job %s terminated, now coping the result data..." % options['reconnect']))
+ conn.ssh('"cd %s; tar --exclude=DEFAULT_WIND --exclude=PROJ_INFO --exclude=PROJ_UNITS -czf %s *;"' \
+ % (mapset_name, output_file))
+ conn.pcs(output_file, path)
+ new_mapset = os.path.join(vari['GISDBASE'],vari['LOCATION_NAME'],'gcloud%s' % options['reconnect'])
+ #os.mkdir(new_mapset)
+ outtar = tarfile.open(os.path.join(path,"gcloud_result_%s.tar.gz" % options['reconnect']), "r:gz")
+ outtar.extractall(path=new_mapset)
+ grass.message(_("To see the new data launch\n g.mapsets add=gcloud%s" % options['reconnect']))
+ else:
+ grass.message(_("Job %s it is not terminated yet" % options['reconnect']))
+ return
+
+# main function
+def main():
+ # set the home path
+ home = os.path.expanduser('~')
+ # check if user is in GRASS
+ gisbase = os.getenv('GISBASE')
+ if not gisbase:
+ grass.fatal(_('$GISBASE not defined'))
+ return 0
+ # check ssh
+ if not grass.find_program('ssh', ['-V']):
+ grass.fatal(_("%s required. Please install '%s' first.") % ('ssh', 'ssh'))
+ return 0
+ # parse the grassdata, location e mapset
+ variables = grass.core.gisenv()
+ # check the version
+ version = grass.core.version()
+ # this is would be set automatically
+ if version['version'].find('7.') != -1:
+ grassVersion = 'grass%s%s' % (version['version'][0],version['version'][2])
+ session_path = '.grass%s' % version['version'][0]
+ else:
+ grass.fatal(_('You are not in GRASS GIS version 7'))
+ return 0
+ # set the path of grassdata/location/mapset
+ # set to .grass7 folder
+ path = os.path.join(home,session_path,'g.cloud')
+ if not os.path.exists(path):
+ os.makedirs(path)
+ # set username, password and folder if settings are inserted by stdin
+ if options['config'] == '-':
+ user = raw_input(_('Insert username: '))
+ passwd = getpass.getpass()
+ # set username, password and folder by file
+ else:
+ # open the file and read the the user and password:
+ # first line is username
+ # second line is password
+ if not os.path.exists(options['config']):
+ grass.fatal(_('The file %s doesn\'t exist' % options['config']))
+ if stat.S_IMODE(os.stat(options['config']).st_mode) == int('0600',8):
+ filesett = open(options['config'],'r')
+ fileread = filesett.readlines()
+ user = fileread[0].strip()
+ passwd = fileread[1].strip()
+ filesett.close()
+ else:
+ err = 'The file permissions of %s are considered insecure.\n' % options['config']
+ err = 'Please correct permissions to read/write only for user (mode 600)'
+ grass.fatal(_(err))
+ return 0
+ # server option
+ server = options['server']
+ # create the sshs session
+ ssh_conn = sshs.ssh_session(user, server, session_path, passwd)
+ if flags['a']:
+ ssh_conn.add()
+ # check if the server has grass and qsub installed, and return the home
+ if options['path'] == '$HOME':
+ serverHome = serverCheck(ssh_conn,grassVersion)
+ else:
+ serverHome = options['path']
+ if options['reconnect']:
+ reconnect(ssh_conn,options['reconnect'],path,variables,serverHome)
+ else:
+ if options['grass_script']:
+ if not os.path.exists(options['grass_script']):
+ grass.fatal(_("File %s does not exists" % options['grass_script']))
+ else:
+ grassScript = options['grass_script']
+ nameGrassScript = os.path.split(grassScript)[-1]
+ else:
+ grass.fatal(_("You have to set %s option") % 'grass_script')
+ if options['qsub_script']:
+ if not os.path.exists(options['qsub_script']):
+ grass.fatal(_("File %s does not exists" % options['qsub_script']))
+ else:
+ qsubScript = options['qsub_script']
+ nameQsubScript = os.path.split(qsubScript)[-1]
+ else:
+ grass.fatal(_("You have to set %s option") % 'qsub_script')
+ # the pid of process to have unique value
+ pid = os.path.split(tempfile.mkstemp()[1])[-1]
+ # name for the unique folder
+ serverFolder = os.path.join(serverHome, 'gcloud%s' % pid)
+ ssh_conn.ssh('mkdir %s' % serverFolder)
+ serverGISDBASE = os.path.split(variables['GISDBASE'])[-1] + str(pid)
+ permanent = os.path.join(variables['GISDBASE'],variables['LOCATION_NAME'],
+ 'PERMANENT')
+ # create the new path for $home/GRASSDATA/LOCATION/PERMANENT on the server
+ new_perm = os.path.join(serverHome, serverGISDBASE,
+ variables['LOCATION_NAME'], 'PERMANENT')
+ ssh_conn.ssh('mkdir -p %s' % new_perm)
+ tar = tarfile.open("PERMANENT.tar.gz", "w:gz")
+ tar.add(os.path.join(permanent,'PROJ_INFO'),'PROJ_INFO')
+ tar.add(os.path.join(permanent,'DEFAULT_WIND'),'DEFAULT_WIND')
+ tar.add(os.path.join(permanent,'PROJ_UNITS'),'PROJ_UNITS')
+ tar.add(os.path.join(permanent,'WIND'),'WIND')
+ if os.path.isfile(os.path.join(permanent,'VAR')):
+ tar.add(os.path.join(permanent,'VAR'),'VAR')
+ tar.close()
+ ssh_conn.scp('PERMANENT.tar.gz', serverHome)
+ ssh_conn.ssh('tar -C %s -xzf PERMANENT.tar.gz' % new_perm)
+ ssh_conn.ssh('rm -f PERMANENT.tar.gz')
+ os.remove('PERMANENT.tar.gz')
+
+ if options['raster'] != '':
+ rasters = options['raster'].split(',')
+ copyMaps(ssh_conn, rasters, 'raster', serverFolder)
+ if options['vector'] != '':
+ rasters = options['vector'].split(',')
+ copyMaps(ssh_conn, rasters, 'vector', serverFolder)
+ # copy the scripts to the server
+ tar = tarfile.open("script_gcloud.tar.gz", "w:gz")
+ if options['raster'] != '' or options['vector'] != '':
+ tar.add(os.path.join(cloudpath,'cloud_unpack.py'), 'cloud_unpack.py')
+ tar.add(os.path.join(cloudpath,'cloud_which.py'), 'cloud_which.py')
+ tar.add(os.path.join(cloudpath,'cloud_collect.sh'), 'cloud_collect.sh')
+ tar.add(os.path.join(cloudpath,'cloud_mail.sh'),'cloud_mail.sh')
+ tar.add(grassScript,nameGrassScript)
+ tar.add(qsubScript,nameQsubScript)
+ tar.close()
+ ssh_conn.scp("script_gcloud.tar.gz", serverHome)
+ ssh_conn.ssh('tar -C %s -xzf script_gcloud.tar.gz' % serverFolder)
+ ssh_conn.ssh('rm -f script_gcloud.tar.gz')
+ os.remove('script_gcloud.tar.gz')
+ if options['raster'] != '' or options['vector'] != '':
+ grass.debug("Launching cloud_unpack.py with this parameters: %s, %s, %s" %
+ (serverFolder, python, new_perm), debug = 2)
+ ssh_conn.ssh('"cd %s ; %s cloud_unpack.py %s"' % (serverFolder, python, new_perm))
+ qsubid = os.path.join(serverFolder, 'tmpqsub')
+ grass.debug("The pid of job is %s" % (str(pid)), debug = 2)
+ if options['variables'] != '':
+ vari = ast.literal_eval(options['variables'])
+ values = vari.values()
+ keys = vari.keys()
+ if flags['c']:
+ values = variablesCheckCicle(values)
+ else:
+ values = variablesCheck(values)
+ njobs = 0
+ for val in range(len(values)):
+ launchstr = '"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH ' \
+ '-v GRASSDBASE=%s -v MYLOC=%s -v GRASSCRIPT=%s' % (
+ serverFolder, pid, os.path.join(serverHome, serverGISDBASE),
+ variables['LOCATION_NAME'], os.path.join(serverFolder,nameGrassScript)
+ )
+ for k in range(len(keys)):
+ launchstr += ' -v %s=%s' % (str(keys[k]),str(values[val][k]))
+ launchstr += ' %s >> %s' % (os.path.join(serverFolder, nameQsubScript),
+ qsubid)
+ ssh_conn.ssh(launchstr)
+ njobs += 1
+ grass.message(_('Launching %i jobs...' % njobs))
+ else:
+ launchstr = 'cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH ' \
+ '-v GRASSDBASE=%s -v MYLOC=%s -v GRASSCRIPT=%s %s > %s' % (
+ serverFolder, pid, os.path.join(serverHome, serverGISDBASE),
+ variables['LOCATION_NAME'], os.path.join(serverFolder,nameGrassScript),
+ os.path.join(serverFolder, nameQsubScript), qsubid
+ )
+ ssh_conn.ssh(launchstr)
+ grass.message(_('Launching a single job...'))
+
+ #if options['mail']:
+ #strin = "\"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH -hold_jid "
+ #strin += "-hold_jid `cat %s | tr '\n' ',' | sed 's+,$++g'` %s %s %s %s\""
+ #ssh_conn.ssh(strin % ( serverFolder, pid, qsubid,
+ #os.path.join(serverFolder, 'cloud_collect.sh'),
+ #os.path.join(serverHome, serverGISDBASE),
+ #variables['LOCATION_NAME'], options['mail'])
+ #)
+ #else:
+ #strin = "\"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH -hold_jid "
+ #strin += "-hold_jid `cat %s | tr '\n' ',' | sed 's+,$++g'` %s %s %s\""
+ #ssh_conn.ssh(strin % ( serverFolder, pid, qsubid,
+ #os.path.join(serverFolder, 'cloud_collect.sh'),
+ #os.path.join(serverHome, serverGISDBASE),
+ #variables['LOCATION_NAME'])
+ #)
+ if options['mail']:
+ mail = options['mail']
+ else:
+ mail = "NOOO"
+ if flags['k']:
+ remove = "NOOO"
+ else:
+ remove = "yes"
+ ids = ssh_conn.ssh("cat %s | cut -d' ' -f3 | tr '\n' ',' | sed 's+,$++g'" % qsubid)
+ #'string %(s)s' % {'s':1}
+ collectstr = "\"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH " % (
+ serverFolder, pid)
+ collectstr += "-hold_jid %s %s %s %s %s %s %s\"" % (
+ ids, os.path.join(serverFolder, 'cloud_collect.sh'),
+ os.path.join(serverHome, serverGISDBASE), variables['LOCATION_NAME'],
+ mail,remove,pid)
+ ssh_conn.ssh(collectstr)
+ grass.message(_('If you want to reconnect to this job to see its status please use the reconnect options with this value: %s' % pid))
+ grass.message(_(' g.cloud config=path|- server=host reconnect=%s' % pid))
+ ssh_conn.close()
+
+if __name__ == "__main__":
+ options, flags = grass.parser()
+ sys.exit(main())
Added: grass-addons/grass7/general/g.cloud/test/launch_SGE_grassjob.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/test/launch_SGE_grassjob.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/test/launch_SGE_grassjob.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,97 @@
+#!/bin/sh
+
+############################################################################
+#
+# MODULE: launch_SGE_grassjob.sh.template
+# AUTHOR(S): Markus Neteler
+# Modify by Luca Delucchi for g.cloud
+#
+# PURPOSE: Launch GRASS GIS job on Grid Engine
+#
+# COPYRIGHT: (C) 2008-2011 by Markus Neteler
+#
+# This program is free software under the GNU General Public
+# License (>=v2). Read the file COPYING that comes with GRASS
+# for details.
+#
+#############################################################################
+
+## Grid Engine settings
+# request Bourne shell as shell for job
+#$ -S /bin/sh
+#
+# run in current working directory
+#$ -cwd
+#
+#$ -l mem_free=1700M
+#
+####################################
+
+## DEBUG
+#set -x
+
+# better say where to find libs and bins:
+export PATH=$PATH:/usr/local/bin:$HOME
+export LD_LIBRARY_PATH=$MYLD_LIBRARY_PATH
+MYGRASS_ADDON_PATH=$HOME/.grass7/addons/scripts/
+
+#### DON'T TOUCH THE VARIABLES BELOW
+# generate machine (blade) unique TMP string
+UNIQUE=`mktemp`
+MYTMP=`basename $UNIQUE`
+# use Grid Engine jobid + unique string as MAPSET to avoid GRASS lock
+MYMAPSET=sge.$JOB_ID.$MYTMP
+MYUSER=$MYMAPSET
+# The target mapset where output data will be stored
+TARGETMAPSET=PERMANENT
+# the job
+GRASS_BATCH_JOB=$GRASSCRIPT
+
+# path to GRASS binaries and libraries:
+export GISBASE=`grass70 --config path`
+export PATH=$PATH:$GISBASE/bin:$GISBASE/scripts
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GISBASE/lib
+
+# use process ID (PID) as GRASS lock file number:
+export GIS_LOCK=$$
+export GRASS_MESSAGE_FORMAT=plain
+export TERM=linux
+
+################ nothing to change below ############
+
+# Set the global GRASS settings file to individual file name
+mkdir -p $HOME/.grass7/
+MYGISRC="$HOME/.grass7/rc.$MYUSER.`uname -n`.$MYTMP"
+
+#generate GISRCRC
+echo "GRASS_ADDON_PATH: $MYGRASS_ADDON_PATH" > "$MYGISRC"
+echo "GISDBASE: $GRASSDBASE" >> "$MYGISRC"
+echo "LOCATION_NAME: $MYLOC" >> "$MYGISRC"
+echo "MAPSET: $MYMAPSET" >> "$MYGISRC"
+echo "GRASS_GUI: text" >> "$MYGISRC"
+
+# path to GRASS settings file
+export GISRC=$MYGISRC
+
+mkdir $GRASSDBASE/$MYLOC/$MYMAPSET
+# fix WIND in the newly created mapset
+cp "$GRASSDBASE/$MYLOC/PERMANENT/DEFAULT_WIND" "$GRASSDBASE/$MYLOC/$MYMAPSET/WIND"
+db.connect -c --quiet
+
+# run the GRASS job:
+. $GRASS_BATCH_JOB
+
+# cleaning up temporary files
+$GISBASE/etc/clean_temp > /dev/null
+rm -f ${MYGISRC}
+rm -rf $HOME/tmp/grass7-$USER-$GIS_LOCK
+
+# leave breadcrumb to find related mapsets back when moving results
+# into final mapset:
+echo "touch $GRASSDBASE/$MYLOC/$MYMAPSET/$MYPID" 1>&2
+
+touch $GRASSDBASE/$MYLOC/$MYMAPSET/$MYPID
+
+echo "Hopefully successfully finished at `date` *************"
+
+exit 0
Added: grass-addons/grass7/general/g.cloud/test/test_morevariables.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/test/test_morevariables.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/test/test_morevariables.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# Sample script to test g.cloud without user defined variables
+# How to use test:
+# - start GRASS with North Carolina dataset location
+#
+# TEST 1: pairwise: use number point 10 with buffer distance 100; and number point 20 with buffer distance 500
+# - g.cloud conf=~/.diss server=giscluster grass=/PATH/TO/g.cloud/test/test_morevariables.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh variables="{'NPOINT' : [10,20], 'BUFFERDIST' : [100,500]}"
+# TEST 2: all-to-all: use number point 10 with buffer distance 100 and 500; number point 20 with buffer distance 100 and 500
+# - g.cloud -c conf=~/.diss server=giscluster grass=/PATH/TO/g.cloud/test/test_morevariables.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh variables="{'NPOINT' : [10,20], 'BUFFERDIST' : [100,500]}"
+
+# using PID as name suffix
+v.random output=random$$_$NPOINT n=$NPOINT
+g.region vect=random$$_$NPOINT
+v.buffer input=random$$_$NPOINT output=buffer$$_$NPOINT_$BUFFERDIST distance=$BUFFERDIST
Added: grass-addons/grass7/general/g.cloud/test/test_morevariables_raster.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/test/test_morevariables_raster.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/test/test_morevariables_raster.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# Sample script to test g.cloud with more user defined variables and raster
+# How to use this test:
+# - start GRASS with North Carolina dataset location
+#
+# TEST 1: pairwise: use var with size 3; and corr with size 7
+# - g.cloud conf=~/.diss server=giscluster grass=/PATH/TO/g.cloud/test/test_morevariables_raster.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh variables="{'TEXTURE' : ['var','corr'], 'SIZE' : [3,7]}" raster=lsat7_2002_40
+#
+# TEST 2: all-to-all: use var with size 3 and 7; use corr with size 3 and 7
+# - g.cloud -c conf=~/.diss server=giscluster grass=/PATH/TO/g.cloud/test/test_morevariables_raster.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh variables="{'TEXTURE' : ['var','corr'], 'SIZE' : [3,7]}" raster=lsat7_2002_40
+
+g.region rast=lsat7_2002_40
+r.texture input=lsat7_2002_40 prefix=lsat7_2002_40_$SIZE method=$TEXTURE size=$SIZE
+
Added: grass-addons/grass7/general/g.cloud/test/test_novariables.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/test/test_novariables.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/test/test_novariables.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Sample script to test g.cloud without user defined variables
+# How to use test:
+# - start GRASS with North Carolina dataset location
+#
+# - g.cloud conf=~/.gc_loginfile.txt server=giscluster grass=/PATH/TO/g.cloud/test/test_novariables_raster.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh
+
+# using PID ($$) as name suffix
+v.random output=random$$ n=123
+g.region vect=random$$
+v.buffer input=random$$ output=buffer$$ distance=200
Added: grass-addons/grass7/general/g.cloud/test/test_onevariable.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/test/test_onevariable.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/test/test_onevariable.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# Sample script to test g.cloud without user defined variables
+# How to use test:
+# - start GRASS with North Carolina dataset location
+#
+# - g.cloud conf=~/.gc_loginfile.txt server=giscluster grass=/PATH/TO/g.cloud/test/test_onevariable.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh variables="{'NPOINT' : [20,50,100]}" raster=lsat7_2002_40
+
+# using PID ($$) as name suffix
+v.random output=random$$_$NPOINT n=$NPOINT
+g.region vect=random$$_$NPOINT
+v.buffer input=random$$_$NPOINT output=buffer$$ distance=1
Added: grass-addons/grass7/general/g.cloud/test/test_onevariable_raster.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/test/test_onevariable_raster.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/test/test_onevariable_raster.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Sample script to test g.cloud with one user defined variable and raster
+# How to use test:
+# - start GRASS with North Carolina dataset location
+#
+# - g.cloud conf=~/.gc_loginfile.txt server=giscluster grass=/PATH/TO/g.cloud/test/test_onevariable_raster.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh variables="{'TEXTURE' : ['var','corr']}" raster=lsat7_2002_40
+
+g.region rast=lsat7_2002_40
+r.texture input=lsat7_2002_40 prefix=lsat7_2002_40 method=$TEXTURE
Added: grass-addons/grass7/general/g.cloud/test/test_onevariable_sun.sh
===================================================================
--- grass-addons/grass7/general/g.cloud/test/test_onevariable_sun.sh (rev 0)
+++ grass-addons/grass7/general/g.cloud/test/test_onevariable_sun.sh 2012-01-23 09:14:47 UTC (rev 50386)
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# Sample script to test g.cloud with one user defined variable and raster
+# How to use test:
+# - start GRASS with North Carolina dataset location
+#
+# - g.cloud conf=~/.gc_loginfile.txt server=giscluster grass=/PATH/TO/g.cloud/test/test_onevariable_sun.sh qsub=/PATH/TO/g.cloud/test/test_launch_SGE_grassjob.sh variables="{'DOY' : [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,22
9,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365]}" raster=elevation
+
+export GRASS_OVERWRITE=1
+
+D=`echo $DOY | awk '{printf "%03d\n", $1}'`
+
+ELEV=elevation
+
+g.region rast=$ELEV
+
+# r.sun memory requirements: 4 bytes per raster cell
+# rows,cols: rows and columns of current region (find out with g.region)
+# IR: number of input raster maps without horizon maps
+# OR: number of output raster maps
+IR=1
+OR=5
+eval `g.region -g`
+horizonsteps=1
+echo "memory in bytes being used by r.sun: `echo \"$rows * $cols * ($IR * 4 + $horizonsteps + $OR * 4)\" | bc `"
+
+r.sun -s $ELEV day=$DOY numpartitions=1 \
+ horizonstep=30 step=0.05 \
+ insol_time=pat_insol_time$D beam_rad=pat_beam_rad$D \
+ diff_rad=pat_diff_rad$D refl_rad=pat_refl_rad$D glob_rad=pat_glob_rad$D
+
More information about the grass-commit
mailing list