[Tilecache] make dir error
Stefan Zweig
stefanzweig1881 at web.de
Fri Apr 11 04:09:02 EDT 2008
hi fredrik,
first of all thanks for your quick reply.
but i am afraid it does not.
i still get the same error in Disk.py on line 81:
An error occurred: [Errno 2] No such file or directory: '/var/_apps/tilecache/tiles/56559/00/000/000/002/-01/999/999.png.k674.25149.tmp'
File "/var/www/tilecache/TileCache/Service.py", line 264,
in cgiHandler File "/var/www/tilecache/TileCache/Service.py", line 182,
in dispatchRequest File "/var/www/tilecache/TileCache/Service.py", line 120,
in renderTile File "/var/gisapp/_apps/tilecache/TileCache/Caches/Disk.py", line 81, in set output = file(tmpfile, "wb")
i have just checked my tilecache and there really is no file in this directory. even the path /var/_apps/tilecache/tiles/56559/00/000/000/002/-01/999/ does not exist, whereas /var/_apps/tilecache/tiles/56559/00/000/000/002/000/000/ does exist and the latter one has 3 png (000.png, 001.png, 002.png) inside.
actually i do not think, that the above error has something to do with pid number crashings. the problem occurs too often (1 or 2 times within a test of only 20 tiles). i guess it is more a side effect of changing the makedirs() function.
here is the source code i used to make the test. i hope could manage to introduce your patch correctly into my code:
# BSD Licensed, Copyright (c) 2006-2008 MetaCarta, Inc.
from TileCache.Cache import Cache
import sys, os, time, errno, socket
hostname = socket.gethostname()
class Disk (Cache):
def __init__ (self, base = None, umask = '002', **kwargs):
Cache.__init__(self, **kwargs)
self.basedir = base
self.umask = int(umask, 0)
if sys.platform.startswith("java"):
from java.io import File
self.file_module = File
self.platform = "jython"
else:
self.platform = "cpython"
if not self.access(base, 'read'):
self.makedirs(base)
def makedirs(self, path):
old_umask = os.umask(self.umask)
try:
os.makedirs(path)
except OSError, e:
# os.makedirs can suffer a race condition because it doesn't check
# that the directory doesn't exist at each step, nor does it
# catch errors. This lets 'directory exists' errors pass through,
# since they mean that as far as we're concerned, os.makedirs
# has 'worked'
if e.errno != errno.EEXIST:
raise e
os.umask(old_umask)
def access(self, path, type='read'):
if self.platform == "jython":
if type == "read":
return self.file_module(path).canRead()
else:
return self.file_module(path).canWrite()
else:
if type =="read":
return os.access(path, os.R_OK)
else:
return os.access(path, os.W_OK)
def getKey (self, tile):
components = ( self.basedir,
tile.layer.name,
"%02d" % tile.z,
"%03d" % int(tile.x / 1000000),
"%03d" % (int(tile.x / 1000) % 1000),
"%03d" % (int(tile.x) % 1000),
"%03d" % int(tile.y / 1000000),
"%03d" % (int(tile.y / 1000) % 1000),
"%03d.%s" % (int(tile.y) % 1000, tile.layer.extension)
)
filename = os.path.join( *components )
return filename
def get (self, tile):
filename = self.getKey(tile)
if self.access(filename, 'read'):
tile.data = file(filename, "rb").read()
return tile.data
else:
return None
def set (self, tile, data):
if self.readonly: return data
filename = self.getKey(tile)
dirname = os.path.dirname(filename)
if not self.access(dirname, 'write'):
self.makedirs(dirname)
tmpfile = filename + ".%s.%d.tmp" % (hostname, os.getpid())
#tmpfile = filename + ".%d.tmp" % os.getpid()
old_umask = os.umask(self.umask)
output = file(tmpfile, "wb")
output.write(data)
output.close()
os.umask( old_umask );
try:
os.rename(tmpfile, filename)
except OSError:
os.unlink(filename)
os.rename(tmpfile, filename)
tile.data = data
return data
def delete (self, tile):
filename = self.getKey(tile)
if self.access(filename, 'read'):
os.unlink(filename)
def attemptLock (self, tile):
name = self.getLockName(tile)
try:
self.makedirs(name)
return True
except OSError:
pass
try:
st = os.stat(name)
if st.st_ctime + self.stale < time.time():
warnings.warn("removing stale lock %s" % name)
# remove stale lock
self.unlock()
self.makedirs(name)
return True
except OSError:
pass
return False
def unlock (self, tile):
name = self.getLockName(tile)
try:
os.rmdir(name)
except OSError, E:
print >>sys.stderr, "unlock %s failed: %s" % (name, str(E))
regards,
stefan
> -----Ursprüngliche Nachricht-----
> Von: "Fredrik Lundh" <fredrik at pythonware.com>
> Gesendet: 10.04.08 18:09:19
> An: tilecache at openlayers.org
> Betreff: Re: [Tilecache] make dir error
>
> Christopher Schmidt <crschmidt at metacarta.com> wrote:
>
> > And they both get called at the same time, you could get a condition
> > where:
> >
> > 1: /0 is created
> > 2: tries to make /0, bails
> > 1. Successfully makes /0/0/0/0/0/0 whatever
> > 2. Has no /1 directory, writing file fails
> >
> > Am I wrong here, or is this a possibility?
>
> makedirs starts by attempting to create the full directory, and then
> works backwards if that doesn't work -- so the one that gets to the
> topmost directory first will create it, and the other one will get an
> EEXIST error, which is handled internally for intermediate
> directories.
>
> but maybe I'm missing something.
>
> there's another potential collision here as well, since you're using
> pid numbers to distinguish multiple processes, which isn't reliable in
> Stefan scenario (multiple machines sharing a cache over NFS).
> intuitively, PID collisions should be pretty unlikely, but I've
> learned not to trust my intuition when it comes to race problems ;-)
>
> Stefan, does the attached patch (untested) change the behaviour in any way?
>
> </F>
>
> --- Disk.py.bak Thu Apr 10 17:27:09 2008
> +++ Disk.py Thu Apr 10 18:05:32 2008
> @@ -1,7 +1,9 @@
> # BSD Licensed, Copyright (c) 2006-2007 MetaCarta, Inc.
>
> from TileCache.Cache import Cache
> -import sys, os, time
> +import sys, os, time, errno, socket
> +
> +hostname = socket.gethostname()
>
> class Disk (Cache):
> def __init__ (self, base = None, umask = '002', **kwargs):
>
> @@ -64,7 +70,7 @@
> dirname = os.path.dirname(filename)
> if not self.access(dirname, 'write'):
> self.makedirs(dirname)
> - tmpfile = filename + ".%d.tmp" % os.getpid()
> + tmpfile = filename + ".%s.%d.tmp" % (hostname, os.getpid())
> old_umask = os.umask(self.umask)
> output = file(tmpfile, "wb")
> output.write(data)
> _______________________________________________
> Tilecache mailing list
> Tilecache at openlayers.org
> http://openlayers.org/mailman/listinfo/tilecache
>
_______________________________________________________________
Schon gehört? Der neue WEB.DE MultiMessenger kann`s mit allen:
http://www.produkte.web.de/messenger/?did=3016
More information about the Tilecache
mailing list