[Liblas-commits] libpc: add stats collection; fix small seek bug;
add clear cache...
liblas-commits at liblas.org
liblas-commits at liblas.org
Fri Mar 11 18:31:33 EST 2011
details: http://hg.liblas.orglibpc/rev/07c553f349ea
changeset: 217:07c553f349ea
user: Michael P. Gerlek <mpg at flaxen.com>
date: Fri Mar 11 15:31:13 2011 -0800
description:
add stats collection; fix small seek bug; add clear cache function; add cache size configurability
Subject: libpc: merge
details: http://hg.liblas.orglibpc/rev/f338384ff966
changeset: 218:f338384ff966
user: Michael P. Gerlek <mpg at flaxen.com>
date: Fri Mar 11 15:31:26 2011 -0800
description:
merge
diffstat:
include/libpc/CacheFilter.hpp | 25 ++++++++++---
include/libpc/PointDataCache.hpp | 28 ++++++++++++--
src/CacheFilter.cpp | 49 ++++++++++++++++++++-----
src/drivers/oci/writer.cpp | 75 ++++++++++++++++++++++++++++++++++++++++
src/drivers/oci/writer.hpp | 2 +
test/unit/CacheFilterTest.cpp | 17 +-------
test/unit/PointDataCacheTest.cpp | 31 ++++++++++------
7 files changed, 181 insertions(+), 46 deletions(-)
diffs (truncated from 432 to 300 lines):
diff -r 44076c9385ab -r f338384ff966 include/libpc/CacheFilter.hpp
--- a/include/libpc/CacheFilter.hpp Fri Mar 11 14:56:11 2011 -0800
+++ b/include/libpc/CacheFilter.hpp Fri Mar 11 15:31:26 2011 -0800
@@ -50,22 +50,33 @@
class LIBPC_DLL CacheFilter : public Filter
{
public:
- CacheFilter(Stage& prevStage);
+ CacheFilter(Stage& prevStage, boost::uint32_t numBlocks, boost::uint32_t blockSize);
~CacheFilter();
const std::string& getName() const;
+ // override
+ boost::uint64_t getCurrentPointIndex() const;
+
+ // override
+ void seekToPoint(boost::uint64_t index);
+
+ // clear cache (but leave cache params unchanged)
+ void resetCache();
+
+ // clear cache, and change cache params too
+ void resetCache(boost::uint32_t numBlocks, boost::uint32_t blockSize);
+
// number of points requested from this filter via read()
boost::uint64_t getNumPointsRequested() const;
// num points this filter read from the previous stage
boost::uint64_t getNumPointsRead() const;
- // override
- boost::uint64_t getCurrentPointIndex() const;
-
- // override
- void seekToPoint(boost::uint64_t index);
+ void getCacheStats(boost::uint64_t& numCacheLookupMisses,
+ boost::uint64_t& numCacheLookupHits,
+ boost::uint64_t& numCacheInsertMisses,
+ boost::uint64_t& numCacheInsertHits) const;
private:
boost::uint32_t readBuffer(PointData&);
@@ -77,6 +88,8 @@
boost::uint64_t m_numPointsRead;
PointDataCache* m_cache;
+ boost::uint32_t m_maxCacheBlocks;
+ boost::uint32_t m_cacheBlockSize;
CacheFilter& operator=(const CacheFilter&); // not implemented
CacheFilter(const CacheFilter&); // not implemented
diff -r 44076c9385ab -r f338384ff966 include/libpc/PointDataCache.hpp
--- a/include/libpc/PointDataCache.hpp Fri Mar 11 14:56:11 2011 -0800
+++ b/include/libpc/PointDataCache.hpp Fri Mar 11 15:31:26 2011 -0800
@@ -60,6 +60,10 @@
// the maximum number of records to be stored.
PointDataCache(size_t c)
:_capacity(c)
+ , m_numCacheLookupMisses(0)
+ , m_numCacheLookupHits(0)
+ , m_numCacheInsertMisses(0)
+ , m_numCacheInsertHits(0)
{
assert(_capacity!=0);
}
@@ -82,17 +86,17 @@
if (it==_cache.left.end())
{
// We don't have it:
+ ++m_numCacheLookupMisses;
return NULL;
}
else
{
-
// We do have it: update the access record view and return it
_cache.right.relocate(
_cache.right.end(),
_cache.project_right(it)
);
-
+ ++m_numCacheLookupHits;
return it->info;
}
}
@@ -108,6 +112,7 @@
{
// We don't have it: insert it
insertx(k,v);
+ ++m_numCacheInsertMisses;
return v;
}
else
@@ -118,7 +123,8 @@
_cache.right.end(),
_cache.project_right(it)
);
- assert(it->info == v);
+ assert(it->info == v);
+ ++m_numCacheInsertHits;
return it->info;
}
}
@@ -138,7 +144,16 @@
}
}
- void foo();
+ void getCacheStats(boost::uint64_t& numCacheLookupMisses,
+ boost::uint64_t& numCacheLookupHits,
+ boost::uint64_t& numCacheInsertMisses,
+ boost::uint64_t& numCacheInsertHits) const
+ {
+ numCacheLookupMisses = m_numCacheLookupMisses;
+ numCacheLookupHits = m_numCacheLookupHits;
+ numCacheInsertMisses = m_numCacheInsertMisses;
+ numCacheInsertHits = m_numCacheInsertHits;
+ }
private:
@@ -169,6 +184,11 @@
const size_t _capacity;
cache_type _cache;
+ boost::uint64_t m_numCacheLookupMisses;
+ boost::uint64_t m_numCacheLookupHits;
+ boost::uint64_t m_numCacheInsertMisses;
+ boost::uint64_t m_numCacheInsertHits;
+
PointDataCache& operator=(const PointDataCache&); // not implemented
PointDataCache(const PointDataCache&); // not implemented
};
diff -r 44076c9385ab -r f338384ff966 src/CacheFilter.cpp
--- a/src/CacheFilter.cpp Fri Mar 11 14:56:11 2011 -0800
+++ b/src/CacheFilter.cpp Fri Mar 11 15:31:26 2011 -0800
@@ -40,7 +40,8 @@
{
-CacheFilter::CacheFilter(Stage& prevStage)
+// cache block size is measured in Points, not bytes
+CacheFilter::CacheFilter(Stage& prevStage, boost::uint32_t maxCacheBlocks, boost::uint32_t cacheBlockSize)
: Filter(prevStage)
, m_currentPointIndex(0)
, m_storedPointIndex(0)
@@ -48,8 +49,10 @@
, m_numPointsRequested(0)
, m_numPointsRead(0)
, m_cache(NULL)
+ , m_maxCacheBlocks(maxCacheBlocks)
+ , m_cacheBlockSize(cacheBlockSize)
{
- m_cache = new PointDataCache(2);
+ resetCache();
return;
}
@@ -69,6 +72,33 @@
}
+void CacheFilter::getCacheStats(boost::uint64_t& numCacheLookupMisses,
+ boost::uint64_t& numCacheLookupHits,
+ boost::uint64_t& numCacheInsertMisses,
+ boost::uint64_t& numCacheInsertHits) const
+{
+ m_cache->getCacheStats(numCacheLookupMisses,
+ numCacheLookupHits,
+ numCacheInsertMisses,
+ numCacheInsertHits);
+}
+
+
+void CacheFilter::resetCache(boost::uint32_t maxCacheBlocks, boost::uint32_t cacheBlockSize)
+{
+ m_maxCacheBlocks = maxCacheBlocks;
+ m_cacheBlockSize = cacheBlockSize;
+ resetCache();
+}
+
+
+void CacheFilter::resetCache()
+{
+ delete m_cache;
+ m_cache = new PointDataCache(m_maxCacheBlocks);
+}
+
+
boost::uint64_t CacheFilter::getCurrentPointIndex() const
{
return m_currentPointIndex;
@@ -96,8 +126,6 @@
boost::uint32_t CacheFilter::readBuffer(PointData& data)
{
- const int chunkSize = 1024;
-
// for now, we only read from the cache if they are asking for one point
// (this avoids the problem of an N-point request needing more than one
// cached block to satisfy it)
@@ -108,12 +136,12 @@
// if they asked for a full block and we got a full block,
// and the block we got is properly aligned and not already cached,
// then let's cache it!
- if (data.getNumPoints() == chunkSize && numRead == chunkSize &&
- (m_currentPointIndex % chunkSize == 0) &&
+ if (data.getNumPoints() == m_cacheBlockSize && numRead == m_cacheBlockSize &&
+ (m_currentPointIndex % m_cacheBlockSize == 0) &&
m_cache->lookup(m_currentPointIndex) == NULL)
{
- PointData* block = new PointData(data.getSchemaLayout(), chunkSize);
- block->copyPointsFast(0, 0, data, chunkSize);
+ PointData* block = new PointData(data.getSchemaLayout(), m_cacheBlockSize);
+ block->copyPointsFast(0, 0, data, m_cacheBlockSize);
m_cache->insert(m_currentPointIndex, block);
}
@@ -126,17 +154,18 @@
}
// they asked for just one point -- first, check Mister Cache
- boost::uint64_t blockNum = m_currentPointIndex / chunkSize;
+ boost::uint64_t blockNum = m_currentPointIndex / m_cacheBlockSize;
PointData* block = m_cache->lookup(blockNum);
if (block != NULL)
{
// A hit! A palpable hit!
- data.copyPointFast(0, m_currentPointIndex % chunkSize, *block);
+ data.copyPointFast(0, m_currentPointIndex % m_cacheBlockSize, *block);
m_numPointsRead += 0;
m_numPointsRequested += 1;
m_currentPointIndex += 1;
+ m_prevStage.seekToPoint(m_currentPointIndex);
return 1;
}
diff -r 44076c9385ab -r f338384ff966 src/drivers/oci/writer.cpp
--- a/src/drivers/oci/writer.cpp Fri Mar 11 14:56:11 2011 -0800
+++ b/src/drivers/oci/writer.cpp Fri Mar 11 15:31:26 2011 -0800
@@ -685,10 +685,85 @@
return;
}
+bool Writer::FillOraclePointData(PointData const& buffer, std::vector<boost::uint8_t>& point_data)
+{
+
+
+ libpc::Schema const& schema = buffer.getSchema();
+
+ bool hasTimeData = schema.hasDimension(Dimension::Field_GpsTime);
+
+ boost::uint64_t count = buffer.getNumPoints();
+
+ const int indexX = schema.getDimensionIndex(Dimension::Field_X);
+ const int indexY = schema.getDimensionIndex(Dimension::Field_Y);
+ const int indexZ = schema.getDimensionIndex(Dimension::Field_Z);
+ const int indexClassification = schema.getDimensionIndex(Dimension::Field_Classification);
+ const int indexTime = schema.getDimensionIndex(Dimension::Field_GpsTime);
+
+ Dimension const& dimX = schema.getDimension(indexX);
+ Dimension const& dimY = schema.getDimension(indexY);
+ Dimension const& dimZ = schema.getDimension(indexZ);
+
+ double xscale = dimX.getNumericScale();
+ double yscale = dimY.getNumericScale();
+ double zscale = dimY.getNumericScale();
+
+ double xoffset = dimX.getNumericOffset();
+ double yoffset = dimY.getNumericOffset();
+ double zoffset = dimZ.getNumericOffset();
+
+ boost::uint32_t counter = 0;
+ while (counter < count)
+ {
+
+ const double x = (buffer.getField<boost::int32_t>(counter, indexX) * xscale) + xoffset;
+ const double y = (buffer.getField<boost::int32_t>(counter, indexY) * yscale) + yoffset;
+ const double z = (buffer.getField<boost::int32_t>(counter, indexZ) * zscale) + zoffset;
+
+ double t = 0.0;
+ if (hasTimeData)
+ t = buffer.getField<double>(counter, indexTime);
+
+ counter++;
+ }
+
+ return true;
+}
+
boost::uint32_t Writer::writeBuffer(const PointData& pointData)
{
boost::uint32_t numPoints = pointData.getNumPoints();
+
+ libpc::Header const& header = m_stage.getHeader();
+ libpc::Schema const& schema = header.getSchema();
More information about the Liblas-commits
mailing list