[Tilecache] Reduce the size of the PNGs (1/2)
Guillaume Lathoud
glathoud at yahoo.fr
Tue Jul 15 09:22:48 EDT 2008
Hello,
Here at Alpstein Tourismus GmbH we would like to share some options we
have added to TileCache 2.04, mainly following this presentation:
http://blog.projectxtech.com/2008/07/02/awesome-slide-deck-on-image-optimisation/
Our main modifications:
* Permit to significantly reduce the size of the PNG files using:
http://www.imagemagick.org
http://pmt.sourceforge.net/pngcrush/
* Permit to do antialiasing on top of a WMSLayer.
We appreciate a lot the work done by MetaCarta Labs on TileCache, and
we hope that our modifications can be useful to anyone.
Best regards,
Guillaume Lathoud
http://www.alpstein-tourismus.de/
Following:
* Examples of use in tilecache.cfg
* List of modifications
* Modified TileCache/Service.py
* Modified TileCache/Layer.py
* Modified TileCache/Client.py
* Modified TileCache/Layers/WMS.py
* Modified TileCache/Layers/Image.py
##################################################
# Examples of use in tilecache.cfg
##################################################
[alpsteinpc43_wege_sbg]
type=WMSLayer
layers=alpsteinpc43:wege_sbg
url=http://localhost:9090/geoserver/wms?
levels=20
tms_type=google
spherical_mercator=true
#
# Ask the WMS server for a larger resolution, then downsample.
antialias=true
#
# Convert 24-bit PNGs to 8-bit PNG (indexed colors).
palette="f:/guillaume.lathoud/Programme/ImageMagick-6.4.1-Q8/convert.exe"
#
# Reduce a bit more the size of the PNG data.
pngcrush="f:/guillaume.lathoud/Programme/pngcrush-1.6.4-win32/pngcrush.exe"
[tropical_beach]
type=ImageLayer
file=e:/guillaume.lathoud/DATA/co.pics/tropical-beach.jpg
filebounds=0,-1016,728,0
scaling=antialias
maxresolution=16
levels=10
tms_type=google
#
# Fill the rest of the ImageLayer with yellow.
fillcolor=yellow
extension=jpg
#
# Select the JPEG quality.
jpegquality=65
[alpsteinpc43_wege_sbg_jpg]
type=WMSLayer
layers=alpsteinpc43:wege_sbg
url=http://localhost:9090/geoserver/wms?
extension=jpg
levels=20
tms_type=google
spherical_mercator=true
#
# Select the JPEG quality.
jpegquality=50
[tropical_beach_png]
type=ImageLayer
file=e:/guillaume.lathoud/DATA/co.pics/tropical-beach.jpg
filebounds=0,-1016,728,0
scaling=antialias
maxresolution=16
levels=10
tms_type=google
fillcolor=yellow
extension=png
palette="f:/guillaume.lathoud/Programme/ImageMagick-6.4.1-Q8/convert.exe"
pngcrush="f:/guillaume.lathoud/Programme/pngcrush-1.6.4-win32/pngcrush.exe"
##################################################
# List of modifications
##################################################
All modifications are marked in the code below with "modified_XXX"
comments (e.g. "modified_palette").
Proposed modifications:
* (all layers, extension=png) palette
Converts 24-bit PNG to indexed 8-bit PNG
Requires Python Imaging Library 1.1.6
Requires Image Magick 6.4.1-Q8
* (all layers, extension=png) pngcrush
Reduces the size of a PNG.
Requires pngcrush 1.6.4
* (all layers, extension=jpg) jpegquality
Selects the target JPEG quality.
Requires Python Imaging Library 1.1.6
* (WMSLayer only) antialias
Asks the WMS for a larger tile, then downsamples it.
Requires Python Imaging Library 1.1.6
* (ImageLayer only) fillcolor
Selects the fill color around the image.
Requires Python Imaging Library 1.1.6
##################################################
# Modified TileCache/Service.py
##################################################
#!/usr/bin/python
# BSD Licensed, Copyright (c) 2006-2008 MetaCarta, Inc.
class TileCacheException(Exception): pass
import sys, cgi, time, os, traceback, ConfigParser
import Cache, Caches
import Layer, Layers
# Windows doesn't always do the 'working directory' check correctly.
if sys.platform == 'win32':
workingdir = os.path.abspath(os.path.join(os.getcwd(), os.path.dirname(sys.argv[0])))
cfgfiles = (os.path.join(workingdir, "tilecache.cfg"), os.path.join(workingdir,"..","tilecache.cfg"))
else:
cfgfiles = ("/etc/tilecache.cfg", os.path.join("..", "tilecache.cfg"), "tilecache.cfg")
class Capabilities (object):
def __init__ (self, format, data):
self.format = format
self.data = data
class Request (object):
def __init__ (self, service):
self.service = service
def getLayer(self, layername):
try:
return self.service.layers[layername]
except:
raise TileCacheException("The requested layer (%s) does not exist. Available layers are: \n * %s" % (layername, "\n * ".join(self.service.layers.keys())))
def import_module(name):
"""Helper module to import any module based on a name, and return the module."""
mod = __import__(name)
components = name.split('.')
for comp in components[1:]:
mod = getattr(mod, comp)
return mod
class Service (object):
__slots__ = ("layers", "cache", "metadata", "tilecache_options", "config", "files")
def __init__ (self, cache, layers, metadata = {}):
self.cache = cache
self.layers = layers
self.metadata = metadata
def _loadFromSection (cls, config, section, module, **objargs):
type = config.get(section, "type")
for opt in config.options(section):
if opt not in ["type", "module"]:
objargs[opt] = config.get(section, opt)
object_module = None
if config.has_option(section, "module"):
object_module = import_module(config.get(section, "module"))
else:
if module is Layer:
type = type.replace("Layer", "")
object_module = import_module("TileCache.Layers.%s" % type)
else:
type = type.replace("Cache", "")
object_module = import_module("TileCache.Caches.%s" % type)
if object_module == None:
raise TileCacheException("Attempt to load %s failed." % type)
section_object = getattr(object_module, type)
if module is Layer:
return section_object(section, **objargs)
else:
return section_object(**objargs)
loadFromSection = classmethod(_loadFromSection)
def _load (cls, *files):
cache = None
metadata = {}
layers = {}
config = None
try:
config = ConfigParser.ConfigParser()
config.read(files)
if config.has_section("metadata"):
for key in config.options("metadata"):
metadata[key] = config.get("metadata", key)
if config.has_section("tilecache_options"):
if 'path' in config.options("tilecache_options"):
for path in config.get("tilecache_options", "path").split(","):
sys.path.insert(0, path)
cache = cls.loadFromSection(config, "cache", Cache)
layers = {}
for section in config.sections():
if section in cls.__slots__: continue
layers[section] = cls.loadFromSection(
config, section, Layer,
cache = cache)
except Exception, E:
metadata['exception'] = E
metadata['traceback'] = "".join(traceback.format_tb(sys.exc_traceback))
service = cls(cache, layers, metadata)
service.files = files
service.config = config
return service
load = classmethod(_load)
def generate_crossdomain_xml(self):
"""Helper method for generating the XML content for a crossdomain.xml
file, to be used to allow remote sites to access this content."""
xml = ["""<?xml version="1.0"?>
<!DOCTYPE cross-domain-policy SYSTEM
"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd">
<cross-domain-policy>
"""]
if self.metadata.has_key('crossdomain_sites'):
sites = self.metadata['crossdomain_sites'].split(',')
for site in sites:
xml.append(' <allow-access-from domain="%s" />' % site)
xml.append("</cross-domain-policy>")
return ('text/xml', "\n".join(xml))
def renderTile (self, tile, force = False):
from warnings import warn
start = time.time()
# do more cache checking here: SRS, width, height, layers
layer = tile.layer
image = None
if not force: image = self.cache.get(tile)
if not image:
data = layer.render(tile)
if (data):
# modified_palette begin
if layer.palette:
# xxx debug
open( 'c:/tmp/data.png', 'wb' ).write( data )
import PIL.Image as PILImage
import StringIO
pilimage = PILImage.open( StringIO.StringIO( data ) )
if pilimage.format.lower() in ( 'png', ):
pilcolors = pilimage.getcolors()
# Sometimes (PIL 1.1.6) getcolors() "fails" and returns None
if pilcolors == None:
pilcolors = []
pilpixels = pilimage.load()
for a in range( pilimage.size[ 0 ] ):
for b in range( pilimage.size[ 1 ] ):
pilc = pilpixels[ a, b ]
if pilc not in pilcolors:
pilcolors.append( pilc )
if len( pilcolors ) > 1:
break
if len( pilcolors ) > 1:
break
if len( pilcolors ) < 2:
# Work around a bug in Image Magick 6.4.1-Q8:
# "convert.exe <src.png> -colors 256 png8:<dest.png>"
# ...when all pixels in src.png have the same color.
pilimage = PILImage.open( StringIO.StringIO( data ) )
newpilimage = pilimage.convert( 'P' )
palette_out = StringIO.StringIO()
newpilimage.save( palette_out, layer.extension )
data = palette_out.getvalue()
else: # Call Image Magick's convert on non-empty images
import tempfile, os, datetime, subprocess
tmp_convert_exefullpath = layer.palette
if tmp_convert_exefullpath[ 0 ] == tmp_convert_exefullpath[ -1 ]:
if tmp_convert_exefullpath[ 0 ] in ( '"', "'", ):
tmp_convert_exefullpath = tmp_convert_exefullpath[ 1:-1 ]
tmp_orig_imagefile = tempfile.NamedTemporaryFile( suffix = '.' + layer.extension )
tmp_orig_imagefilename = tmp_orig_imagefile.name
tmp_orig_imagefile.close()
open( tmp_orig_imagefilename, 'wb' ).write( data )
tmp_dest_imagefile = tempfile.NamedTemporaryFile( suffix = '.' + layer.extension )
tmp_dest_imagefilename = tmp_dest_imagefile.name
tmp_dest_imagefile.close()
tmp_format_str = 'png8:' if ( layer.extension.lower() == 'png' ) else ''
tmp_pipe = subprocess.Popen( tmp_convert_exefullpath +
" %s -colors 256 %s%s" % ( tmp_orig_imagefilename,
tmp_format_str,
tmp_dest_imagefilename, ),
shell=False,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE )
tmp_stdout = tmp_pipe.stdout.read()
tmp_stderr = tmp_pipe.stderr.read()
tmp_exit_code = tmp_pipe.wait()
assert tmp_exit_code == 0
data = open( tmp_dest_imagefilename, 'rb' ).read()
f_debug = open( 'c:/tmp/Client_py_debug.txt', 'at' )
f_debug.write( os.linesep * 2 )
f_debug.write( "palette: tmp_orig_imagefilename:'%s'" % tmp_orig_imagefilename )
f_debug.write( os.linesep )
f_debug.write( "palette: tmp_dest_imagefilename:'%s'" % tmp_dest_imagefilename )
f_debug.write( os.linesep )
os.remove( tmp_orig_imagefilename )
os.remove( tmp_dest_imagefilename )
# modified_palette end
# modified_pngcrush begin
if layer.pngcrush:
import StringIO
import PIL.Image as PILImage
pilimage = PILImage.open( StringIO.StringIO( data ) )
if pilimage.format.lower() in ( 'png', ):
import tempfile, os, datetime, subprocess
tmp_pngcrush_exefullpath = layer.pngcrush
if tmp_pngcrush_exefullpath[ 0 ] == tmp_pngcrush_exefullpath[ -1 ]:
if tmp_pngcrush_exefullpath[ 0 ] in ( '"', "'", ):
tmp_pngcrush_exefullpath = tmp_pngcrush_exefullpath[ 1:-1 ]
tmp_orig_imagefile = tempfile.NamedTemporaryFile( suffix = '.' + layer.extension )
tmp_orig_imagefilename = tmp_orig_imagefile.name
tmp_orig_imagefile.close()
open( tmp_orig_imagefilename, 'wb' ).write( data )
tmp_dest_imagefile = tempfile.NamedTemporaryFile( suffix = '.' + layer.extension )
tmp_dest_imagefilename = tmp_dest_imagefile.name
tmp_dest_imagefile.close()
tmp_pipe = subprocess.Popen( tmp_pngcrush_exefullpath +
" %s %s" % ( tmp_orig_imagefilename,
tmp_dest_imagefilename, ),
shell=False,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE )
tmp_stdout = tmp_pipe.stdout.read()
tmp_stderr = tmp_pipe.stderr.read()
tmp_exit_code = tmp_pipe.wait()
assert tmp_exit_code == 0
data = open( tmp_dest_imagefilename, 'rb' ).read()
os.remove( tmp_orig_imagefilename )
os.remove( tmp_dest_imagefilename )
# modified_pngcrush end
# modified_jpegquality begin
if layer.jpegquality:
import StringIO
import PIL.Image as PILImage
pilimage = PILImage.open( StringIO.StringIO( data ) )
if pilimage.format.lower() in ( 'jpg', 'jpeg', ):
jpegquality_out = StringIO.StringIO()
jpegquality_opt = { 'quality': layer.jpegquality, }
pilimage.save( jpegquality_out, layer.extension,
**jpegquality_opt )
data = jpegquality_out.getvalue()
# modified_jpegquality end
image = self.cache.set(tile, data)
else:
raise Exception("Zero length data returned from layer.")
if layer.debug:
sys.stderr.write(
"Cache miss: %s, Tile: x: %s, y: %s, z: %s, time: %s\n" % (
tile.bbox(), tile.x, tile.y, tile.z, (time.time() - start)) )
else:
if layer.debug:
sys.stderr.write(
"Cache hit: %s, Tile: x: %s, y: %s, z: %s, time: %s, debug: %s\n" % (
tile.bbox(), tile.x, tile.y, tile.z, (time.time() - start), layer.debug) )
return (layer.mime_type, image)
def expireTile (self, tile):
bbox = tile.bounds()
layer = tile.layer
for z in range(len(layer.resolutions)):
bottomleft = layer.getClosestCell(z, bbox[0:2])
topright = layer.getClosestCell(z, bbox[2:4])
for y in range(bottomleft[1], topright[1] + 1):
for x in range(bottomleft[0], topright[0] + 1):
coverage = Tile(layer,x,y,z)
self.cache.delete(coverage)
def dispatchRequest (self, params, path_info="/", req_method="GET", host="http://example.com/"):
if self.metadata.has_key('exception'):
raise TileCacheException("%s\n%s" % (self.metadata['exception'], self.metadata['traceback']))
if path_info.find("crossdomain.xml") != -1:
return self.generate_crossdomain_xml()
if path_info.split(".")[-1] == "kml":
from TileCache.Services.KML import KML
return KML(self).parse(params, path_info, host)
if params.has_key("scale") or params.has_key("SCALE"):
from TileCache.Services.WMTS import WMTS
tile = WMTS(self).parse(params, path_info, host)
elif params.has_key("service") or params.has_key("SERVICE") or \
params.has_key("REQUEST") and params['REQUEST'] == "GetMap" or \
params.has_key("request") and params['request'] == "GetMap":
from TileCache.Services.WMS import WMS
tile = WMS(self).parse(params, path_info, host)
elif params.has_key("L") or params.has_key("l") or \
params.has_key("request") and params['request'] == "metadata":
from TileCache.Services.WorldWind import WorldWind
tile = WorldWind(self).parse(params, path_info, host)
elif params.has_key("interface"):
from TileCache.Services.TileService import TileService
tile = TileService(self).parse(params, path_info, host)
elif params.has_key("v") and \
(params['v'] == "mgm" or params['v'] == "mgmaps"):
from TileCache.Services.MGMaps import MGMaps
tile = MGMaps(self).parse(params, path_info, host)
elif params.has_key("tile"):
from TileCache.Services.VETMS import VETMS
tile = VETMS(self).parse(params, path_info, host)
elif params.has_key("format") and params['format'].lower() == "json":
from TileCache.Services.JSON import JSON
return JSON(self).parse(params, path_info, host)
else:
from TileCache.Services.TMS import TMS
tile = TMS(self).parse(params, path_info, host)
if isinstance(tile, Layer.Tile):
if req_method == 'DELETE':
self.expireTile(tile)
return ('text/plain', 'OK')
else:
return self.renderTile(tile, params.has_key('FORCE'))
else:
return (tile.format, tile.data)
def modPythonHandler (apacheReq, service):
from mod_python import apache, util
try:
if apacheReq.headers_in.has_key("X-Forwarded-Host"):
host = "http://" + apacheReq.headers_in["X-Forwarded-Host"]
else:
host = "http://" + apacheReq.headers_in["Host"]
host += apacheReq.uri[:-len(apacheReq.path_info)]
format, image = service.dispatchRequest(
util.FieldStorage(apacheReq),
apacheReq.path_info,
apacheReq.method,
host )
apacheReq.content_type = format
apacheReq.send_http_header()
apacheReq.write(image)
except Layer.TileCacheException, E:
apacheReq.content_type = "text/plain"
apacheReq.status = apache.HTTP_NOT_FOUND
apacheReq.send_http_header()
apacheReq.write("An error occurred: %s\n" % (str(E)))
except Exception, E:
apacheReq.content_type = "text/plain"
apacheReq.status = apache.HTTP_INTERNAL_SERVER_ERROR
apacheReq.send_http_header()
apacheReq.write("An error occurred: %s\n%s\n" % (
str(E),
"".join(traceback.format_tb(sys.exc_traceback))))
return apache.OK
def wsgiHandler (environ, start_response, service):
from paste.request import parse_formvars
try:
path_info = host = ""
if "PATH_INFO" in environ:
path_info = environ["PATH_INFO"]
if "HTTP_X_FORWARDED_HOST" in environ:
host = "http://" + environ["HTTP_X_FORWARDED_HOST"]
elif "HTTP_HOST" in environ:
host = "http://" + environ["HTTP_HOST"]
host += environ["SCRIPT_NAME"]
req_method = environ["REQUEST_METHOD"]
fields = parse_formvars(environ)
format, image = service.dispatchRequest( fields, path_info, req_method, host )
start_response("200 OK", [('Content-Type',format)])
return [image]
except TileCacheException, E:
start_response("404 Tile Not Found", [('Content-Type','text/plain')])
return ["An error occurred: %s" % (str(E))]
except Exception, E:
start_response("500 Internal Server Error", [('Content-Type','text/plain')])
return ["An error occurred: %s\n%s\n" % (
str(E),
"".join(traceback.format_tb(sys.exc_traceback)))]
def cgiHandler (service):
try:
params = {}
input = cgi.FieldStorage()
for key in input.keys(): params[key] = input[key].value
path_info = host = ""
if "PATH_INFO" in os.environ:
path_info = os.environ["PATH_INFO"]
if "HTTP_X_FORWARDED_HOST" in os.environ:
host = "http://" + os.environ["HTTP_X_FORWARDED_HOST"]
elif "HTTP_HOST" in os.environ:
host = "http://" + os.environ["HTTP_HOST"]
host += os.environ["SCRIPT_NAME"]
req_method = os.environ["REQUEST_METHOD"]
format, image = service.dispatchRequest( params, path_info, req_method, host )
print "Content-type: %s\n" % format
if sys.platform == "win32":
binaryPrint(image)
else:
print image
except TileCacheException, E:
print "Cache-Control: max-age=10, must-revalidate" # make the client reload
print "Content-type: text/plain\n"
print "An error occurred: %s\n" % (str(E))
except Exception, E:
print "Cache-Control: max-age=10, must-revalidate" # make the client reload
print "Content-type: text/plain\n"
print "An error occurred: %s\n%s\n" % (
str(E),
"".join(traceback.format_tb(sys.exc_traceback)))
theService = None
lastRead = None
def handler (apacheReq):
global theService, lastRead
options = apacheReq.get_options()
cfgs = cfgfiles
cfgTime = None
if options.has_key("TileCacheConfig"):
cfgs = cfgs + (options["TileCacheConfig"],)
try:
cfgTime = os.stat(options['TileCacheConfig'])[8]
except:
pass
if not theService or (lastRead and cfgTime and lastRead < cfgTime):
theService = Service.load(*cfgs)
lastRead = time.time()
return modPythonHandler(apacheReq, theService)
def wsgiApp (environ, start_response):
global theService
cfgs = cfgfiles
if not theService:
theService = Service.load(*cfgs)
return wsgiHandler(environ, start_response, theService)
def binaryPrint(binary_data):
"""This function is designed to work around the fact that Python
in Windows does not handle binary output correctly. This function
will set the output to binary, and then write to stdout directly
rather than using print."""
try:
import msvcrt
msvcrt.setmode(sys.__stdout__.fileno(), os.O_BINARY)
except:
pass
sys.stdout.write(binary_data)
if __name__ == '__main__':
svc = Service.load(*cfgfiles)
cgiHandler(svc)
_____________________________________________________________________________
Envoyez avec Yahoo! Mail. Une boite mail plus intelligente http://mail.yahoo.fr
More information about the Tilecache
mailing list