[Liblas-commits] hg: 3 new changesets

liblas-commits at liblas.org liblas-commits at liblas.org
Thu Oct 14 19:44:40 EDT 2010


changeset 21d31fe1baa7 in /Volumes/Data/www/liblas.org/hg
details: http://hg.liblas.orghg?cmd=changeset;node=21d31fe1baa7
summary: New index file revision. Extra data stored to support IndexIterator class functionality.

changeset a77554a00add in /Volumes/Data/www/liblas.org/hg
details: http://hg.liblas.orghg?cmd=changeset;node=a77554a00add
summary: IndexIterator added. Retrieve filtered points in batches instead of in one large array.

changeset bb131e655373 in /Volumes/Data/www/liblas.org/hg
details: http://hg.liblas.orghg?cmd=changeset;node=bb131e655373
summary: Test code for IndexIterator functionality.

diffstat:

 apps/lasindex_test.cpp                      |  132 ++++++-
 include/liblas/detail/index/indexoutput.hpp |    4 +-
 include/liblas/lasindex.hpp                 |   93 +++-
 src/detail/index/indexoutput.cpp            |   37 +-
 src/lasindex.cpp                            |  583 ++++++++++++++++++++++-----
 5 files changed, 698 insertions(+), 151 deletions(-)

diffs (truncated from 1450 to 300 lines):

diff -r 6e16ffeea5f4 -r bb131e655373 apps/lasindex_test.cpp
--- a/apps/lasindex_test.cpp	Tue Oct 12 20:37:08 2010 -0500
+++ b/apps/lasindex_test.cpp	Thu Oct 14 17:40:34 2010 -0600
@@ -99,6 +99,10 @@
     fprintf(debugger,"  follow with low and high values for Z filter\n");
     fprintf(debugger,"\n");
     
+    fprintf(debugger,"-it (use iterator for filtering):\n");
+    fprintf(debugger,"  follow with number of points to be returned in each iteration\n");
+    fprintf(debugger,"\n");
+    
     fprintf(debugger, "\nFor more information, see the full documentation for lasindex_test at:\n"
                     " http://liblas.org/browser/trunk/doc/lasindex_test.txt\n");
     fprintf(debugger,"----------------------------------------------------------\n");
@@ -121,6 +125,7 @@
     if (!istrm->good())
     {
         delete istrm;
+        istrm = 0;
         throw std::runtime_error("Reading stream was not able to be created.");
     }
     return istrm;
@@ -142,6 +147,7 @@
     if (!ostrm->good())
     {
         delete ostrm;
+        ostrm = 0;
         throw std::runtime_error("Writing stream was not able to be created.");
     }
     return ostrm;
@@ -162,6 +168,26 @@
     fprintf(debugger, "Unable to initialize index filter. Invalid values.\n");
 }
 
+bool ReportIteratorResults(FILE *debugger, uint32_t resultSize, uint32_t pointCount, uint32_t step)
+{
+	if (resultSize)
+	{
+		// do something with the list of points
+		#ifdef VISUAL_8
+		fprintf(debugger, "Points within filter area %d of %d, step %d, %s\n", resultSize, 
+			pointCount, step, "Using iterator");
+		#else // VISUAL_8
+		fprintf(debugger, "Points within filter area %zu of %d, step %d, %s\n", resultSize, 
+			pointCount, step, "Using iterator");
+		#endif // VISUAL_8
+		return true;
+	}
+	else
+	{
+		IndexFilterNoPoints(debugger);
+		return false;
+	} // else
+} // ReportIteratorResults
 
 int main(int argc, char* argv[])
 {
@@ -174,12 +200,14 @@
 	char *datefield = 0;
 	double zbinheight = 0.0;
 	double oLowFilterX = 0.0, oHighFilterX = 0.0, oLowFilterY = 0.0, oHighFilterY = 0.0, oLowFilterZ = 0.0, oHighFilterZ = 0.0;
-	uint32_t maxmem = 0;
+	boost::uint32_t maxmem = 0;
+	boost::uint32_t chunkSize = 100;
 	int debuglevel = 3;
 	bool readonly = 0;
 	bool forcenewindex = 0;
 	bool boundssetbyuser = 0;
 	bool writestandaloneindex = 0;
+	bool useiterator = 0;
 	FILE *debugger = stderr;
 	
 	// temporary until argv[] starts to work
@@ -214,6 +242,11 @@
 		 "-x", "1443000.00", "1444000.00", "-y", "376000.02", "379000.00", "-z", "850.00", "950.00"};
 	argc = 22;
 	*/	
+	/*------------filter with embedded index using iterator---------------------
+	const char* arggv[] = {"foo",
+		 "-i", "C:\\LibLAS\\Samples\\N1440375_idx.las", "-r", "-it", "20000"};
+	argc = 6;
+	*/
 	/*---------------------Serpent Mound Model LAS Data-----------------------*/
 	/*----------------------------build index-----------------------------------
 	const char* arggv[] = {"foo", "-t", "C:\\LibLAS\\Samples\\Serpent Mound Model LAS Data.tmp",
@@ -251,17 +284,22 @@
 	argc = 4;
 	*/
 	/*------------------------------flatDataset-------------------------------*/
-	///*------------------------------build index---------------------------------
+	/*------------------------------build index---------------------------------
 	const char* arggv[] = {"foo", "-t", "C:\\LibLAS\\flatDataset.tmp",
 		 "-i", "C:\\LibLAS\\Samples\\flatDataset.las", "-a", SAMPLE_AUTHOR, "-c", SAMPLE_COMMENT, "-d", SAMPLE_DATE,
 		 "-o", "C:\\LibLAS\\Samples\\flatDataset_idx.las"};
 	argc = 13;
-	//*/
+	*/
 	/*------------------filter with embedded index------------------------------
 	const char* arggv[] = {"foo",
 		 "-i", "C:\\LibLAS\\Samples\\flatDataset_idx.las", "-r"};
 	argc = 4;
 	*/
+	///*------------filter with embedded index using iterator---------------------
+	const char* arggv[] = {"foo",
+		 "-i", "C:\\LibLAS\\Samples\\flatDataset_idx.las", "-r", "-it", "5"};
+	argc = 6;
+	//*/
 	/*------------------------------------------------------------------------*/
 	
 	for (int i = 1; i < argc; i++)
@@ -375,11 +413,19 @@
             oHighFilterZ = atof((const char *)arggv[i]);
             boundssetbyuser = true;
         }
+        else if (   strcmp((const char *)arggv[i],"-it") == 0
+            )
+        {
+            i++;
+            chunkSize = atoi((const char *)arggv[i]);
+            useiterator = true;
+        }
 	} // for
 
 	fprintf(debugger, "%s\n", lasinfilenme);
 	
 	IndexData ParamSrc;
+	IndexData ParamSrc2;
 	
 	if (lasinfilenme)
 	{
@@ -419,7 +465,78 @@
 							if (index.IndexReady())
 							{
 								// for testing filter bounds set by user
-								if (boundssetbyuser)
+								if (useiterator)
+								{
+									ParamSrc = IndexData(index);
+									ParamSrc2 = IndexData(index);
+									if (ParamSrc.SetFilterValues(index.GetBounds(), index)
+										&& ParamSrc2.SetFilterValues((index.GetMinX() + index.GetMaxX()) * .5, index.GetMaxX(), 
+										(index.GetMinY() + index.GetMaxY()) * .5, index.GetMaxY(), index.GetMinZ(), index.GetMaxZ(), index))
+									{
+										// two iterators with different filter bounds
+										IndexIterator *indexIt = index.Filter(ParamSrc, chunkSize);
+										IndexIterator *indexIt2 = index.Filter(ParamSrc2, chunkSize);
+										if (indexIt)
+										{
+											for (boost::uint32_t step = 0; step < 3; ++step)
+											{
+												// use any of these to begin the vector with the next point that fits the filter criteria 
+												// that hasn't been scanned yet.
+												// ex: indexIt->advance(1) starts the vector on the first point that fits the criteria if no points have 
+												// been scanned previously. If there have been previous points scanned then the vector returned
+												// will start with the next point that fits the criteria beyond those previously scanned.
+												// ex: indexIt->advance(5) starts the vector on the fifth point that fits the criteria if no points have 
+												// been scanned previously. If there have been previous points scanned then the vector returned
+												// will start with the fifth point that fits the criteria beyond those previously scanned.
+												// The following three methods may be used to advance by any positive number X.
+												//const std::vector<boost::uint32_t>& FilterResult = indexIt->advance(X);
+												//const std::vector<boost::uint32_t>& FilterResult = (*indexIt)+=X;
+												//const std::vector<boost::uint32_t>& FilterResult = (*indexIt)+X;
+												// The following two methods may be used to advance as though X were 1.
+												// There is no functional difference between pre and post increment;
+												//const std::vector<boost::uint32_t>& FilterResult = ++(*indexIt);
+												//const std::vector<boost::uint32_t>& FilterResult = (*indexIt)++;
+												// Random access to any number point in the file that fits the filter criteria is 
+												// provided with the overloaded [] operator or () operator.
+												// ex: (*indexIt)[32] would start the array at the 32nd point in the file that fits the criteria
+												// Any number of non-conforming points may have been skipped to get to the start point
+												//const std::vector<boost::uint32_t>& FilterResult = (*indexIt)[32];
+												//const std::vector<boost::uint32_t>& FilterResult = (*indexIt)(0);
+												
+												// get first or next consecutive range on first iterator
+												const std::vector<boost::uint32_t>& FilterResult = (*indexIt)++;
+												if (!ReportIteratorResults(debugger, FilterResult.size(), index.GetPointRecordsCount(), step))
+													break;
+
+												// get first or next consecutive range on 2nd iterator
+												const std::vector<boost::uint32_t>& FilterResult2 = (*indexIt2)++;
+												if (!ReportIteratorResults(debugger, FilterResult2.size(), index.GetPointRecordsCount(), step))
+													break;
+
+												// get next range on first iterator
+												const std::vector<boost::uint32_t>& FilterResult3 = (*indexIt)++;
+												if (!ReportIteratorResults(debugger, FilterResult3.size(), index.GetPointRecordsCount(), step))
+													break;
+												
+												// get a range beginning with the 5th compliant point on 2nd iterator
+												const std::vector<boost::uint32_t>& FilterResult4 = (*indexIt2)[5];
+												if (!ReportIteratorResults(debugger, FilterResult4.size(), index.GetPointRecordsCount(), step))
+													break;
+
+												// get a range on first iterator beginning 6 compliant pts beyond last range (skipping 5)
+												const std::vector<boost::uint32_t>& FilterResult5 = (*indexIt)+6;
+												if (!ReportIteratorResults(debugger, FilterResult5.size(), index.GetPointRecordsCount(), step))
+													break;
+												
+											} // for
+										} // if
+										else
+											IndexFilterInitError(debugger);
+									}
+									else
+										IndexFilterInitError(debugger);
+								} // if useiterator
+								else if (boundssetbyuser)
 								{
 									Bounds<double> filterBounds(oLowFilterX, oLowFilterY, oLowFilterZ,
 										oHighFilterX, oHighFilterY, oHighFilterZ);
@@ -429,8 +546,13 @@
 										if (FilterResult.size())
 										{
 											// do something with the list of points
+											#ifdef VISUAL_8
+											fprintf(debugger, "Points within filter area %d of %d, %s\n", FilterResult.size(), 
+												index.GetPointRecordsCount(), "User-defined filter bounds");
+											#else // VISUAL_8
 											fprintf(debugger, "Points within filter area %zu of %d, %s\n", FilterResult.size(), 
-												index.GetPointRecordsCount(), "User-defined gilter bounds");
+												index.GetPointRecordsCount(), "User-defined filter bounds");
+											#endif // VISUAL_8
 										}
 										else
 											IndexFilterNoPoints(debugger);
diff -r 6e16ffeea5f4 -r bb131e655373 include/liblas/detail/index/indexoutput.hpp
--- a/include/liblas/detail/index/indexoutput.hpp	Tue Oct 12 20:37:08 2010 -0500
+++ b/include/liblas/detail/index/indexoutput.hpp	Thu Oct 14 17:40:34 2010 -0600
@@ -58,8 +58,8 @@
 	liblas::Index *m_index;
 	liblas::VariableRecord m_indexVLRHeaderRecord, m_indexVLRCellRecord;
 	IndexVLRData m_indexVLRHeaderData, m_indexVLRCellPointData, m_indexVLRTempData;
-	boost::uint32_t m_VLRCommonDataSize, m_VLRDataSizeLocation, m_FirstCellLocation, m_LastCellLocation;
-	boost::uint32_t  m_DataRecordSize, m_TempWritePos;
+	boost::uint32_t m_VLRCommonDataSize, m_VLRDataSizeLocation, m_FirstCellLocation, m_LastCellLocation, m_VLRPointCountLocation;
+	boost::uint32_t  m_DataRecordSize, m_TempWritePos, m_DataPointsThisVLR;
 	bool m_FirstCellInVLR, m_SomeDataReadyToWrite;
 	
 protected:
diff -r 6e16ffeea5f4 -r bb131e655373 include/liblas/lasindex.hpp
--- a/include/liblas/lasindex.hpp	Tue Oct 12 20:37:08 2010 -0500
+++ b/include/liblas/lasindex.hpp	Thu Oct 14 17:40:34 2010 -0600
@@ -58,13 +58,14 @@
 namespace liblas {
 
 #define LIBLAS_INDEX_MAXMEMDEFAULT	10000000	// 10 megs default
-#define LIBLAS_INDEX_MINMEMDEFAULT	100000	// 1 meg at least has to be allowed
+#define LIBLAS_INDEX_MINMEMDEFAULT	1000000	// 1 meg at least has to be allowed
 #define LIBLAS_INDEX_VERSIONMAJOR	1
-#define LIBLAS_INDEX_VERSIONMINOR	0
+#define LIBLAS_INDEX_VERSIONMINOR	1	// minor version 1 begins 10/12/10
 #define LIBLAS_INDEX_MAXSTRLEN	512
 #define LIBLAS_INDEX_MAXCELLS	250000
 #define LIBLAS_INDEX_OPTPTSPERCELL	100
 #define LIBLAS_INDEX_MAXPTSPERCELL	1000
+#define LIBLAS_INDEX_RESERVEFILTERDEFAULT	1000000	// 1 million points will be reserved on large files for filter result
 
 // define this in order to fix problem with last bytes of last VLR getting corrupted
 // when saved and reloaded from index or las file.
@@ -76,6 +77,7 @@
 
 class liblas::detail::IndexCell;
 class IndexData;
+class IndexIterator;
 
 // Index class is the fundamental object for building and filtering a spatial index of points in an LAS file.
 // An Index class doesn't do anything until it is configured with an IndexData object (see below).
@@ -136,15 +138,10 @@
 	Bounds<double> m_bounds;
 	bool m_indexBuilt, m_tempFileStarted, m_readerCreated, m_readOnly, m_writestandaloneindex, m_forceNewIndex;
 	int m_debugOutputLevel;
+	boost::uint8_t m_versionMajor, m_versionMinor;
     boost::uint32_t m_pointRecordsCount, m_maxMemoryUsage, m_cellsX, m_cellsY, m_cellsZ, m_totalCells, 
 		m_tempFileWrittenBytes, m_DataVLR_ID;
-    boost::int32_t m_LowXCellCompletelyIn, m_HighXCellCompletelyIn, m_LowYCellCompletelyIn, m_HighYCellCompletelyIn,
-		m_LowZCellCompletelyIn, m_HighZCellCompletelyIn;
-    boost::int32_t m_LowXBorderCell, m_HighXBorderCell, m_LowYBorderCell, m_HighYBorderCell,
-		m_LowZBorderCell, m_HighZBorderCell;
     double m_rangeX, m_rangeY, m_rangeZ, m_cellSizeZ, m_cellSizeX, m_cellSizeY;
-	double m_filterMinXCell, m_filterMaxXCell, m_filterMinYCell, m_filterMaxYCell, m_filterMinZCell, m_filterMaxZCell,
-		m_LowXBorderPartCell, m_HighXBorderPartCell, m_LowYBorderPartCell, m_HighYBorderPartCell;
 	std::string m_tempFileName;	
 	std::string m_indexAuthor;
 	std::string m_indexComment;
@@ -159,9 +156,14 @@
     void ClearOldIndex(void);
 	bool BuildIndex(void);
 	bool Validate(void);
+	boost::uint32_t GetDefaultReserve(void);
 	bool LoadIndexVLR(VariableRecord const& vlr);
-	void SetCellFilterBounds(IndexData const& ParamSrc);
-	bool FilterOneVLR(VariableRecord const& vlr, boost::uint32_t& i, IndexData const& ParamSrc);
+	void SetCellFilterBounds(IndexData & ParamSrc);
+	bool FilterOneVLR(VariableRecord const& vlr, boost::uint32_t& i, IndexData & ParamSrc, bool & VLRDone);
+	bool FilterPointSeries(boost::uint32_t & PointID, boost::uint32_t & PointsScanned, 
+		boost::uint32_t const PointsToIgnore, boost::uint32_t const x, boost::uint32_t const y, boost::uint32_t const z, 
+		liblas::detail::ConsecPtAccumulator const ConsecutivePts, IndexIterator *Iterator, 
+		IndexData const& ParamSrc);
 	bool VLRInteresting(boost::int32_t MinCellX, boost::int32_t MinCellY, boost::int32_t MaxCellX, boost::int32_t MaxCellY, 
 		IndexData const& ParamSrc);


More information about the Liblas-commits mailing list