[Liblas-commits] hg: 3 new changesets
liblas-commits at liblas.org
liblas-commits at liblas.org
Thu Oct 14 19:44:40 EDT 2010
changeset 21d31fe1baa7 in /Volumes/Data/www/liblas.org/hg
details: http://hg.liblas.orghg?cmd=changeset;node=21d31fe1baa7
summary: New index file revision. Extra data stored to support IndexIterator class functionality.
changeset a77554a00add in /Volumes/Data/www/liblas.org/hg
details: http://hg.liblas.orghg?cmd=changeset;node=a77554a00add
summary: IndexIterator added. Retrieve filtered points in batches instead of in one large array.
changeset bb131e655373 in /Volumes/Data/www/liblas.org/hg
details: http://hg.liblas.orghg?cmd=changeset;node=bb131e655373
summary: Test code for IndexIterator functionality.
diffstat:
apps/lasindex_test.cpp | 132 ++++++-
include/liblas/detail/index/indexoutput.hpp | 4 +-
include/liblas/lasindex.hpp | 93 +++-
src/detail/index/indexoutput.cpp | 37 +-
src/lasindex.cpp | 583 ++++++++++++++++++++++-----
5 files changed, 698 insertions(+), 151 deletions(-)
diffs (truncated from 1450 to 300 lines):
diff -r 6e16ffeea5f4 -r bb131e655373 apps/lasindex_test.cpp
--- a/apps/lasindex_test.cpp Tue Oct 12 20:37:08 2010 -0500
+++ b/apps/lasindex_test.cpp Thu Oct 14 17:40:34 2010 -0600
@@ -99,6 +99,10 @@
fprintf(debugger," follow with low and high values for Z filter\n");
fprintf(debugger,"\n");
+ fprintf(debugger,"-it (use iterator for filtering):\n");
+ fprintf(debugger," follow with number of points to be returned in each iteration\n");
+ fprintf(debugger,"\n");
+
fprintf(debugger, "\nFor more information, see the full documentation for lasindex_test at:\n"
" http://liblas.org/browser/trunk/doc/lasindex_test.txt\n");
fprintf(debugger,"----------------------------------------------------------\n");
@@ -121,6 +125,7 @@
if (!istrm->good())
{
delete istrm;
+ istrm = 0;
throw std::runtime_error("Reading stream was not able to be created.");
}
return istrm;
@@ -142,6 +147,7 @@
if (!ostrm->good())
{
delete ostrm;
+ ostrm = 0;
throw std::runtime_error("Writing stream was not able to be created.");
}
return ostrm;
@@ -162,6 +168,26 @@
fprintf(debugger, "Unable to initialize index filter. Invalid values.\n");
}
+bool ReportIteratorResults(FILE *debugger, uint32_t resultSize, uint32_t pointCount, uint32_t step)
+{
+ if (resultSize)
+ {
+ // do something with the list of points
+ #ifdef VISUAL_8
+ fprintf(debugger, "Points within filter area %d of %d, step %d, %s\n", resultSize,
+ pointCount, step, "Using iterator");
+ #else // VISUAL_8
+ fprintf(debugger, "Points within filter area %zu of %d, step %d, %s\n", resultSize,
+ pointCount, step, "Using iterator");
+ #endif // VISUAL_8
+ return true;
+ }
+ else
+ {
+ IndexFilterNoPoints(debugger);
+ return false;
+ } // else
+} // ReportIteratorResults
int main(int argc, char* argv[])
{
@@ -174,12 +200,14 @@
char *datefield = 0;
double zbinheight = 0.0;
double oLowFilterX = 0.0, oHighFilterX = 0.0, oLowFilterY = 0.0, oHighFilterY = 0.0, oLowFilterZ = 0.0, oHighFilterZ = 0.0;
- uint32_t maxmem = 0;
+ boost::uint32_t maxmem = 0;
+ boost::uint32_t chunkSize = 100;
int debuglevel = 3;
bool readonly = 0;
bool forcenewindex = 0;
bool boundssetbyuser = 0;
bool writestandaloneindex = 0;
+ bool useiterator = 0;
FILE *debugger = stderr;
// temporary until argv[] starts to work
@@ -214,6 +242,11 @@
"-x", "1443000.00", "1444000.00", "-y", "376000.02", "379000.00", "-z", "850.00", "950.00"};
argc = 22;
*/
+ /*------------filter with embedded index using iterator---------------------
+ const char* arggv[] = {"foo",
+ "-i", "C:\\LibLAS\\Samples\\N1440375_idx.las", "-r", "-it", "20000"};
+ argc = 6;
+ */
/*---------------------Serpent Mound Model LAS Data-----------------------*/
/*----------------------------build index-----------------------------------
const char* arggv[] = {"foo", "-t", "C:\\LibLAS\\Samples\\Serpent Mound Model LAS Data.tmp",
@@ -251,17 +284,22 @@
argc = 4;
*/
/*------------------------------flatDataset-------------------------------*/
- ///*------------------------------build index---------------------------------
+ /*------------------------------build index---------------------------------
const char* arggv[] = {"foo", "-t", "C:\\LibLAS\\flatDataset.tmp",
"-i", "C:\\LibLAS\\Samples\\flatDataset.las", "-a", SAMPLE_AUTHOR, "-c", SAMPLE_COMMENT, "-d", SAMPLE_DATE,
"-o", "C:\\LibLAS\\Samples\\flatDataset_idx.las"};
argc = 13;
- //*/
+ */
/*------------------filter with embedded index------------------------------
const char* arggv[] = {"foo",
"-i", "C:\\LibLAS\\Samples\\flatDataset_idx.las", "-r"};
argc = 4;
*/
+ ///*------------filter with embedded index using iterator---------------------
+ const char* arggv[] = {"foo",
+ "-i", "C:\\LibLAS\\Samples\\flatDataset_idx.las", "-r", "-it", "5"};
+ argc = 6;
+ //*/
/*------------------------------------------------------------------------*/
for (int i = 1; i < argc; i++)
@@ -375,11 +413,19 @@
oHighFilterZ = atof((const char *)arggv[i]);
boundssetbyuser = true;
}
+ else if ( strcmp((const char *)arggv[i],"-it") == 0
+ )
+ {
+ i++;
+ chunkSize = atoi((const char *)arggv[i]);
+ useiterator = true;
+ }
} // for
fprintf(debugger, "%s\n", lasinfilenme);
IndexData ParamSrc;
+ IndexData ParamSrc2;
if (lasinfilenme)
{
@@ -419,7 +465,78 @@
if (index.IndexReady())
{
// for testing filter bounds set by user
- if (boundssetbyuser)
+ if (useiterator)
+ {
+ ParamSrc = IndexData(index);
+ ParamSrc2 = IndexData(index);
+ if (ParamSrc.SetFilterValues(index.GetBounds(), index)
+ && ParamSrc2.SetFilterValues((index.GetMinX() + index.GetMaxX()) * .5, index.GetMaxX(),
+ (index.GetMinY() + index.GetMaxY()) * .5, index.GetMaxY(), index.GetMinZ(), index.GetMaxZ(), index))
+ {
+ // two iterators with different filter bounds
+ IndexIterator *indexIt = index.Filter(ParamSrc, chunkSize);
+ IndexIterator *indexIt2 = index.Filter(ParamSrc2, chunkSize);
+ if (indexIt)
+ {
+ for (boost::uint32_t step = 0; step < 3; ++step)
+ {
+ // use any of these to begin the vector with the next point that fits the filter criteria
+ // that hasn't been scanned yet.
+ // ex: indexIt->advance(1) starts the vector on the first point that fits the criteria if no points have
+ // been scanned previously. If there have been previous points scanned then the vector returned
+ // will start with the next point that fits the criteria beyond those previously scanned.
+ // ex: indexIt->advance(5) starts the vector on the fifth point that fits the criteria if no points have
+ // been scanned previously. If there have been previous points scanned then the vector returned
+ // will start with the fifth point that fits the criteria beyond those previously scanned.
+ // The following three methods may be used to advance by any positive number X.
+ //const std::vector<boost::uint32_t>& FilterResult = indexIt->advance(X);
+ //const std::vector<boost::uint32_t>& FilterResult = (*indexIt)+=X;
+ //const std::vector<boost::uint32_t>& FilterResult = (*indexIt)+X;
+ // The following two methods may be used to advance as though X were 1.
+ // There is no functional difference between pre and post increment;
+ //const std::vector<boost::uint32_t>& FilterResult = ++(*indexIt);
+ //const std::vector<boost::uint32_t>& FilterResult = (*indexIt)++;
+ // Random access to any number point in the file that fits the filter criteria is
+ // provided with the overloaded [] operator or () operator.
+ // ex: (*indexIt)[32] would start the array at the 32nd point in the file that fits the criteria
+ // Any number of non-conforming points may have been skipped to get to the start point
+ //const std::vector<boost::uint32_t>& FilterResult = (*indexIt)[32];
+ //const std::vector<boost::uint32_t>& FilterResult = (*indexIt)(0);
+
+ // get first or next consecutive range on first iterator
+ const std::vector<boost::uint32_t>& FilterResult = (*indexIt)++;
+ if (!ReportIteratorResults(debugger, FilterResult.size(), index.GetPointRecordsCount(), step))
+ break;
+
+ // get first or next consecutive range on 2nd iterator
+ const std::vector<boost::uint32_t>& FilterResult2 = (*indexIt2)++;
+ if (!ReportIteratorResults(debugger, FilterResult2.size(), index.GetPointRecordsCount(), step))
+ break;
+
+ // get next range on first iterator
+ const std::vector<boost::uint32_t>& FilterResult3 = (*indexIt)++;
+ if (!ReportIteratorResults(debugger, FilterResult3.size(), index.GetPointRecordsCount(), step))
+ break;
+
+ // get a range beginning with the 5th compliant point on 2nd iterator
+ const std::vector<boost::uint32_t>& FilterResult4 = (*indexIt2)[5];
+ if (!ReportIteratorResults(debugger, FilterResult4.size(), index.GetPointRecordsCount(), step))
+ break;
+
+ // get a range on first iterator beginning 6 compliant pts beyond last range (skipping 5)
+ const std::vector<boost::uint32_t>& FilterResult5 = (*indexIt)+6;
+ if (!ReportIteratorResults(debugger, FilterResult5.size(), index.GetPointRecordsCount(), step))
+ break;
+
+ } // for
+ } // if
+ else
+ IndexFilterInitError(debugger);
+ }
+ else
+ IndexFilterInitError(debugger);
+ } // if useiterator
+ else if (boundssetbyuser)
{
Bounds<double> filterBounds(oLowFilterX, oLowFilterY, oLowFilterZ,
oHighFilterX, oHighFilterY, oHighFilterZ);
@@ -429,8 +546,13 @@
if (FilterResult.size())
{
// do something with the list of points
+ #ifdef VISUAL_8
+ fprintf(debugger, "Points within filter area %d of %d, %s\n", FilterResult.size(),
+ index.GetPointRecordsCount(), "User-defined filter bounds");
+ #else // VISUAL_8
fprintf(debugger, "Points within filter area %zu of %d, %s\n", FilterResult.size(),
- index.GetPointRecordsCount(), "User-defined gilter bounds");
+ index.GetPointRecordsCount(), "User-defined filter bounds");
+ #endif // VISUAL_8
}
else
IndexFilterNoPoints(debugger);
diff -r 6e16ffeea5f4 -r bb131e655373 include/liblas/detail/index/indexoutput.hpp
--- a/include/liblas/detail/index/indexoutput.hpp Tue Oct 12 20:37:08 2010 -0500
+++ b/include/liblas/detail/index/indexoutput.hpp Thu Oct 14 17:40:34 2010 -0600
@@ -58,8 +58,8 @@
liblas::Index *m_index;
liblas::VariableRecord m_indexVLRHeaderRecord, m_indexVLRCellRecord;
IndexVLRData m_indexVLRHeaderData, m_indexVLRCellPointData, m_indexVLRTempData;
- boost::uint32_t m_VLRCommonDataSize, m_VLRDataSizeLocation, m_FirstCellLocation, m_LastCellLocation;
- boost::uint32_t m_DataRecordSize, m_TempWritePos;
+ boost::uint32_t m_VLRCommonDataSize, m_VLRDataSizeLocation, m_FirstCellLocation, m_LastCellLocation, m_VLRPointCountLocation;
+ boost::uint32_t m_DataRecordSize, m_TempWritePos, m_DataPointsThisVLR;
bool m_FirstCellInVLR, m_SomeDataReadyToWrite;
protected:
diff -r 6e16ffeea5f4 -r bb131e655373 include/liblas/lasindex.hpp
--- a/include/liblas/lasindex.hpp Tue Oct 12 20:37:08 2010 -0500
+++ b/include/liblas/lasindex.hpp Thu Oct 14 17:40:34 2010 -0600
@@ -58,13 +58,14 @@
namespace liblas {
#define LIBLAS_INDEX_MAXMEMDEFAULT 10000000 // 10 megs default
-#define LIBLAS_INDEX_MINMEMDEFAULT 100000 // 1 meg at least has to be allowed
+#define LIBLAS_INDEX_MINMEMDEFAULT 1000000 // 1 meg at least has to be allowed
#define LIBLAS_INDEX_VERSIONMAJOR 1
-#define LIBLAS_INDEX_VERSIONMINOR 0
+#define LIBLAS_INDEX_VERSIONMINOR 1 // minor version 1 begins 10/12/10
#define LIBLAS_INDEX_MAXSTRLEN 512
#define LIBLAS_INDEX_MAXCELLS 250000
#define LIBLAS_INDEX_OPTPTSPERCELL 100
#define LIBLAS_INDEX_MAXPTSPERCELL 1000
+#define LIBLAS_INDEX_RESERVEFILTERDEFAULT 1000000 // 1 million points will be reserved on large files for filter result
// define this in order to fix problem with last bytes of last VLR getting corrupted
// when saved and reloaded from index or las file.
@@ -76,6 +77,7 @@
class liblas::detail::IndexCell;
class IndexData;
+class IndexIterator;
// Index class is the fundamental object for building and filtering a spatial index of points in an LAS file.
// An Index class doesn't do anything until it is configured with an IndexData object (see below).
@@ -136,15 +138,10 @@
Bounds<double> m_bounds;
bool m_indexBuilt, m_tempFileStarted, m_readerCreated, m_readOnly, m_writestandaloneindex, m_forceNewIndex;
int m_debugOutputLevel;
+ boost::uint8_t m_versionMajor, m_versionMinor;
boost::uint32_t m_pointRecordsCount, m_maxMemoryUsage, m_cellsX, m_cellsY, m_cellsZ, m_totalCells,
m_tempFileWrittenBytes, m_DataVLR_ID;
- boost::int32_t m_LowXCellCompletelyIn, m_HighXCellCompletelyIn, m_LowYCellCompletelyIn, m_HighYCellCompletelyIn,
- m_LowZCellCompletelyIn, m_HighZCellCompletelyIn;
- boost::int32_t m_LowXBorderCell, m_HighXBorderCell, m_LowYBorderCell, m_HighYBorderCell,
- m_LowZBorderCell, m_HighZBorderCell;
double m_rangeX, m_rangeY, m_rangeZ, m_cellSizeZ, m_cellSizeX, m_cellSizeY;
- double m_filterMinXCell, m_filterMaxXCell, m_filterMinYCell, m_filterMaxYCell, m_filterMinZCell, m_filterMaxZCell,
- m_LowXBorderPartCell, m_HighXBorderPartCell, m_LowYBorderPartCell, m_HighYBorderPartCell;
std::string m_tempFileName;
std::string m_indexAuthor;
std::string m_indexComment;
@@ -159,9 +156,14 @@
void ClearOldIndex(void);
bool BuildIndex(void);
bool Validate(void);
+ boost::uint32_t GetDefaultReserve(void);
bool LoadIndexVLR(VariableRecord const& vlr);
- void SetCellFilterBounds(IndexData const& ParamSrc);
- bool FilterOneVLR(VariableRecord const& vlr, boost::uint32_t& i, IndexData const& ParamSrc);
+ void SetCellFilterBounds(IndexData & ParamSrc);
+ bool FilterOneVLR(VariableRecord const& vlr, boost::uint32_t& i, IndexData & ParamSrc, bool & VLRDone);
+ bool FilterPointSeries(boost::uint32_t & PointID, boost::uint32_t & PointsScanned,
+ boost::uint32_t const PointsToIgnore, boost::uint32_t const x, boost::uint32_t const y, boost::uint32_t const z,
+ liblas::detail::ConsecPtAccumulator const ConsecutivePts, IndexIterator *Iterator,
+ IndexData const& ParamSrc);
bool VLRInteresting(boost::int32_t MinCellX, boost::int32_t MinCellY, boost::int32_t MaxCellX, boost::int32_t MaxCellY,
IndexData const& ParamSrc);
More information about the Liblas-commits
mailing list