[postgis-tickets] [SCM] PostGIS branch master updated. 3.1.0rc1-113-gd758f50
git at osgeo.org
git at osgeo.org
Fri Mar 5 11:39:10 PST 2021
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "PostGIS".
The branch, master has been updated
via d758f50258f36905e9fbfcdece276060ec8b3762 (commit)
from ec53e3b15f25f42399aa61e6555e1606286a87d8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit d758f50258f36905e9fbfcdece276060ec8b3762
Author: Paul Ramsey <pramsey at cleverelephant.ca>
Date: Fri Mar 5 11:38:52 2021 -0800
Allow open options to be passed to GDAL drivers (in particular the /vsi drivers for cloud buckets. Closes #4870
diff --git a/NEWS b/NEWS
index bcb4d2a..7fcff87 100644
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,7 @@ PostGIS 3.2.0
- #4830, ValidateTopology check for edges side face containment
(Sandro Santilli)
- #4827, Allow NaN coordinates in WKT input (Paul Ramsey)
+ - #4870, Allow open options to be passed to GDAL drivers (Paul Ramsey)
* New features*
- #4841, FindTopology to quickly get a topology record (Sandro Santilli)
diff --git a/doc/reference_raster.xml b/doc/reference_raster.xml
index 6bdf1b6..4d40733 100644
--- a/doc/reference_raster.xml
+++ b/doc/reference_raster.xml
@@ -1321,6 +1321,64 @@ WHERE short_name = 'GTiff') As g;
</refsection>
</refentry>
+
+ <refentry id="RT_ST_GDALOpenOptions">
+ <refnamediv>
+ <refname>ST_GDALOpenOptions</refname>
+
+ <refpurpose>Sets the GDAL driver open options for this session. When using "out db" rasters, it may be necessary to alter the default open options in order to improve performance, or supply authentication information.</refpurpose>
+ </refnamediv>
+
+ <refsynopsisdiv>
+ <funcsynopsis>
+ <funcprototype>
+ <funcdef>text[] <function>ST_GDALOpenOptions</function></funcdef>
+ <paramdef>
+ <type>text[] </type><parameter>options</parameter>
+ </paramdef>
+ </funcprototype>
+ </funcsynopsis>
+ </refsynopsisdiv>
+
+ <refsection>
+ <title>Description</title>
+
+ <para>
+ Sets the "open options" to be used by GDAL for this session (until the current user disconnects). Returns the list of options the function could parse.
+ </para>
+ <para>
+ Options are expressed in the <ulink url="https://gdal.org/user/configoptions.html">standard GDAL options format</ulink>, "NAME=VALUE". Options remain set for the lifetime of the databaes connection, but can be altered at any time by re-running the function.
+ </para>
+ <para>
+ For cloud-stored "out db" rasters, setting authentication options is neccesary to accessing private buckets.
+ </para>
+ <para>Availability: 3.2.0</para>
+
+ </refsection>
+
+ <refsection>
+ <title>Example</title>
+ <para>To access a raster stored in an S3 bucket, us a id/key pair that has permission for the object.</para>
+
+ <programlisting>SELECT st_gdalopenoptions(ARRAY[
+ 'AWS_ACCESS_KEY_ID=08080808080808080808 ',
+ 'AWS_SECRET_ACCESS_KEY=2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x']
+)</programlisting>
+ <para>To create an "out db" table that references an authenticated cloud raster, pass in the options to the environment of the loader.</para>
+ <programlisting>AWS_ACCESS_KEY_ID=08080808080808080808 \
+AWS_SECRET_ACCESS_KEY=2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x2x \
+raster2pgsql \
+ -s 26910 \
+ -t 256x256 \
+ -I \
+ -R \
+ /vsis3/your.bucket.com/your_raster_file.tif \
+ your_raster_table \
+ | psql your_db</programlisting>
+
+</refsection>
+</refentry>
+
<refentry id="RT_UpdateRasterSRID">
<refnamediv>
<refname>UpdateRasterSRID</refname>
diff --git a/doc/using_raster_dataman.xml b/doc/using_raster_dataman.xml
index 85a5c02..0466f20 100644
--- a/doc/using_raster_dataman.xml
+++ b/doc/using_raster_dataman.xml
@@ -449,6 +449,45 @@ Available GDAL raster formats:
<listitem><para>Apply raster constraints using <xref linkend="RT_AddRasterConstraints" /></para></listitem>
</orderedlist>
</sect2>
+
+ <sect2 id="RT_Cloud_Rasters">
+ <title>Using "out db" cloud rasters</title>
+ <para>
+ The <varname>raster2pgsql</varname> tool uses GDAL to access raster data, and can take advantage of a key GDAL feature: the ability to read
+ from rasters that are <ulink url="https://gdal.org/user/virtual_file_systems.html#network-based-file-systems">stored remotely</ulink> in cloud "object stores" (e.g. AWS S3, Google Cloud Storage).
+ </para>
+ <para>
+ Efficient use of cloud stored rasters requires the use of a "cloud optimized" format. The most well-known and widely used is the "<ulink url="https://www.cogeo.org/">cloud optimized GeoTIFF</ulink>" format. Using a non-cloud format, like a JPEG, or an un-tiled TIFF will result in very poor performance, as the system will have to download the entire raster each time it needs to access a subset.
+ </para>
+ <para>
+ First, load your raster into the cloud storage of your choice. Once it is loaded, you will have a URI to access it with, either an "http" URI, or sometimes a URI specific to the service. (e.g., "s3://bucket/object"). To access non-public buckets, you will need to supply GDAL config options to authenticate your connection. Note that this command is <emphasis>reading</emphasis> from the cloud raster and <emphasis>writing</emphasis> to the database.
+ </para>
+ <programlisting>AWS_ACCESS_KEY_ID=xxxxxxxxxxxxxxxxxxxx \
+AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \
+raster2pgsql \
+ -s 990000 \
+ -t 256x256 \
+ -I \
+ -R \
+ /vsis3/your.bucket.com/your_file.tif \
+ your_table \
+ | psql your_db</programlisting>
+ <para>
+ Once the table is loaded, you need to give the database permission to read from remote rasters, by setting two permissions.
+ </para>
+ <programlisting>SET postgis.enable_outdb_rasters = true;
+SET postgis.gdal_enabled_drivers TO 'ENABLE_ALL';
+ </programlisting>
+ <para>
+ To make the changes sticky, set them directly on your database. You will need to re-connect to experience the new settings.
+ </para>
+ <programlisting>ALTER DATABASE your_db SET postgis.enable_outdb_rasters = true;
+ALTER DATABASE your_db SET postgis.gdal_enabled_drivers TO 'ENABLE_ALL';
+ </programlisting>
+ <para>
+ Once you have the data loaded and permissions set you can interact with the raster table like any other raster table, using the same functions. The database will handle all the mechanics of connecting to the cloud data when it needs to read pixel data.
+ </para>
+ </sect2>
</sect1>
<sect1 id="RT_Raster_Catalog">
<title>Raster Catalogs</title>
diff --git a/raster/rt_core/librtcore.h b/raster/rt_core/librtcore.h
index 1ed99ef..c004da6 100644
--- a/raster/rt_core/librtcore.h
+++ b/raster/rt_core/librtcore.h
@@ -241,7 +241,6 @@ typedef void (*rt_message_handler)(const char* string, va_list ap)
*/
extern void rt_install_default_allocators(void);
-
/**
* Wrappers used for managing memory. They simply call the functions defined by
* the caller
diff --git a/raster/rt_core/rt_util.c b/raster/rt_core/rt_util.c
index f616a56..4c9f999 100644
--- a/raster/rt_core/rt_util.c
+++ b/raster/rt_core/rt_util.c
@@ -376,11 +376,14 @@ rt_util_gdal_driver_registered(const char *drv) {
/* variable for PostgreSQL GUC: postgis.gdal_enabled_drivers */
char *gdal_enabled_drivers = NULL;
+char ** gdal_open_options = NULL;
+
/*
wrapper for GDALOpen and GDALOpenShared
*/
GDALDatasetH
rt_util_gdal_open(const char *fn, GDALAccess fn_access, int shared) {
+ unsigned int open_flags;
assert(NULL != fn);
if (gdal_enabled_drivers != NULL) {
@@ -401,10 +404,12 @@ rt_util_gdal_open(const char *fn, GDALAccess fn_access, int shared) {
}
}
- if (shared)
- return GDALOpenShared(fn, fn_access);
- else
- return GDALOpen(fn, fn_access);
+ open_flags = GDAL_OF_RASTER
+ | GDAL_OF_VERBOSE_ERROR
+ | (fn_access == GA_Update ? GDAL_OF_UPDATE : 0)
+ | (shared ? GDAL_OF_SHARED : 0);
+
+ return GDALOpenEx(fn, open_flags, NULL, (const char **)gdal_open_options, NULL);
}
void
diff --git a/raster/rt_pg/rtpg_gdal.c b/raster/rt_pg/rtpg_gdal.c
index a0653c8..2c26b9c 100644
--- a/raster/rt_pg/rtpg_gdal.c
+++ b/raster/rt_pg/rtpg_gdal.c
@@ -34,6 +34,7 @@
#include "utils/lsyscache.h" /* for get_typlenbyvalalign */
#include "utils/array.h" /* for ArrayType */
#include "catalog/pg_type.h" /* for INT2OID, INT4OID, FLOAT4OID, FLOAT8OID and TEXTOID */
+#include "utils/memutils.h" /* For TopMemoryContext */
#include "../../postgis_config.h"
@@ -50,6 +51,7 @@ Datum RASTER_fromGDALRaster(PG_FUNCTION_ARGS);
/* convert raster to GDAL raster */
Datum RASTER_asGDALRaster(PG_FUNCTION_ARGS);
Datum RASTER_getGDALDrivers(PG_FUNCTION_ARGS);
+Datum RASTER_setGDALOpenOptions(PG_FUNCTION_ARGS);
/* warp a raster using GDAL Warp API */
Datum RASTER_GDALWarp(PG_FUNCTION_ARGS);
@@ -441,6 +443,98 @@ Datum RASTER_getGDALDrivers(PG_FUNCTION_ARGS)
}
}
+
+/* variable declared in rt_util.c */
+extern char ** gdal_open_options;
+
+PG_FUNCTION_INFO_V1(RASTER_setGDALOpenOptions);
+Datum RASTER_setGDALOpenOptions(PG_FUNCTION_ARGS)
+{
+//xxxxx
+ int16 elmlen;
+ bool elmbyval;
+ char elmalign;
+ int dims[1];
+ int lbs[1] = {1};
+
+ Datum value;
+ bool isnull;
+
+ MemoryContext oldcontext;
+
+ Datum *elems;
+ bool *nulls;
+# define GDAL_OPEN_MAXOPTS 64
+ char * opts[GDAL_OPEN_MAXOPTS];
+ size_t nopts = 0;
+ size_t i;
+
+ ArrayType *array = PG_GETARG_ARRAYTYPE_P(0);
+ ArrayIterator iterator = array_create_iterator(array, 0, NULL);
+ while (array_iterate(iterator, &value, &isnull))
+ {
+ char* namevalue;
+ /* Skip Null */
+ if (isnull)
+ continue;
+
+ /* Skip anything that is not 'NAME=VALUE' */
+ namevalue = text_to_cstring(DatumGetTextP(value));
+ if (!strstr(namevalue, "="))
+ continue;
+
+ /* Don't overshoot our buffer */
+ if (nopts == GDAL_OPEN_MAXOPTS)
+ {
+ elog(NOTICE, "%s: maximum number (%u) of open options reached", __func__, GDAL_OPEN_MAXOPTS);
+ break;
+ }
+
+ /* Remember this option */
+ opts[nopts++] = namevalue;
+ }
+ array_free_iterator(iterator);
+
+ /* Clean up any pre-existing global options */
+ if (gdal_open_options)
+ {
+ char** opts = gdal_open_options;
+ while(*opts)
+ {
+ pfree(*opts);
+ ++opts;
+ }
+ pfree(gdal_open_options);
+ gdal_open_options = NULL;
+ }
+
+ /* Copy the options to the global location (for this backend) */
+ oldcontext = MemoryContextSwitchTo(TopMemoryContext);
+ gdal_open_options = palloc(sizeof(char*) * (nopts+1));
+ for (i = 0; i < nopts; i++)
+ {
+ gdal_open_options[i] = pstrdup(opts[i]);
+ }
+ gdal_open_options[nopts] = NULL;
+ MemoryContextSwitchTo(oldcontext);
+
+ /* Fill-in array data arrays */
+ elems = palloc(nopts * sizeof(Datum));
+ nulls = palloc(nopts * sizeof(bool));
+ for (i = 0; i < nopts; i++)
+ {
+ elems[i] = PointerGetDatum(cstring_to_text(opts[i]));
+ nulls[i] = false;
+ }
+
+ /* Construct output array */
+ dims[0] = nopts;
+ get_typlenbyvalalign(TEXTOID, &elmlen, &elmbyval, &elmalign);
+ PG_RETURN_POINTER(construct_md_array(
+ elems, nulls, 1, dims, lbs,
+ TEXTOID, elmlen, elmbyval, elmalign));
+}
+
/**
* warp a raster using GDAL Warp API
*/
diff --git a/raster/rt_pg/rtpostgis.sql.in b/raster/rt_pg/rtpostgis.sql.in
index e7e491f..c6e33ff 100644
--- a/raster/rt_pg/rtpostgis.sql.in
+++ b/raster/rt_pg/rtpostgis.sql.in
@@ -1495,6 +1495,11 @@ CREATE OR REPLACE FUNCTION st_gdaldrivers(
AS 'MODULE_PATHNAME', 'RASTER_getGDALDrivers'
LANGUAGE 'c' IMMUTABLE STRICT PARALLEL SAFE;
+CREATE OR REPLACE FUNCTION st_gdalopenoptions(opts text[])
+ RETURNS text[]
+ AS 'MODULE_PATHNAME', 'RASTER_setGDALOpenOptions'
+ LANGUAGE 'c' STRICT;
+
-- Cannot be strict as "options" and "srid" can be NULL
CREATE OR REPLACE FUNCTION st_asgdalraster(rast raster, format text, options text[] DEFAULT NULL, srid integer DEFAULT NULL)
RETURNS bytea
-----------------------------------------------------------------------
Summary of changes:
NEWS | 1 +
doc/reference_raster.xml | 58 ++++++++++++++++++++++++++
doc/using_raster_dataman.xml | 39 ++++++++++++++++++
raster/rt_core/librtcore.h | 1 -
raster/rt_core/rt_util.c | 13 ++++--
raster/rt_pg/rtpg_gdal.c | 94 +++++++++++++++++++++++++++++++++++++++++++
raster/rt_pg/rtpostgis.sql.in | 5 +++
7 files changed, 206 insertions(+), 5 deletions(-)
hooks/post-receive
--
PostGIS
More information about the postgis-tickets
mailing list