SparseFile.cpp
Go to the documentation of this file.
//----------------------------------------------------------------------------//
/*
* Copyright (c) 2009 Sony Pictures Imageworks Inc
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the
* distribution. Neither the name of Sony Pictures Imageworks nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
*/
//----------------------------------------------------------------------------//
//----------------------------------------------------------------------------//
// SparseField.h includes SparseFile.h, but we need the definition of
// SparseBlock from SparseField.h, so just include that to get both
// files
#include "SparseField.h"
#include "OgIO.h"
#include "OgSparseDataReader.h"
//----------------------------------------------------------------------------//
FIELD3D_NAMESPACE_OPEN
//----------------------------------------------------------------------------//
// Static instances
//----------------------------------------------------------------------------//
boost::scoped_ptr<SparseFileManager> SparseFileManager::ms_singleton;
//----------------------------------------------------------------------------//
// SparseFileManager
//----------------------------------------------------------------------------//
SparseFileManager & SparseFileManager::singleton()
{
if (ms_singleton.get() == NULL) {
ms_singleton.reset(new SparseFileManager);
}
return *ms_singleton;
}
//----------------------------------------------------------------------------//
void SparseFileManager::setLimitMemUse(bool enabled)
{
m_limitMemUse = enabled;
}
//----------------------------------------------------------------------------//
bool SparseFileManager::doLimitMemUse() const
{
return m_limitMemUse;
}
//----------------------------------------------------------------------------//
void SparseFileManager::setMaxMemUse(float maxMemUse)
{
m_maxMemUse = maxMemUse;
m_maxMemUseInBytes = static_cast<int64_t>(m_maxMemUse * 1024 * 1024);
}
//----------------------------------------------------------------------------//
template <class Data_T>
int64_t SparseFileManager::deallocateBlock(const SparseFile::CacheBlock &cb)
{
int64_t bytesFreed = 0;
SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(cb.refIdx);
// Note: we don't need to lock the block's mutex because
// deallocateBlock() is only called while the SparseFileManager's
// mutex is also locked (in flushCache() or deallocateBlocks()).
// Don't lock the block, to make sure we don't have a deadlock by
// holding two locks at the same time. (Because addBlockToCache()
// locks the manager but is also in a block-specific lock.)
// lock the current block to make sure its blockUsed flag and ref
// counts don't change
// Note: this lock order is made consistent w/ allocate to prevent
// deadlocks and crashes.
#if F3D_SHORT_MUTEX_ARRAY
boost::mutex::scoped_lock
lock_B(reference->blockMutex[cb.blockIdx % reference->blockMutexSize]);
#else
boost::mutex::scoped_lock lock_B(reference->blockMutex[cb.blockIdx]);
#endif
// check whether the block is still in use
if (reference->refCounts[cb.blockIdx] > 0)
return bytesFreed;
if (reference->blockUsed[cb.blockIdx]) {
// the block was recently used according to Second-chance paging
// algorithm, so skip it
reference->blockUsed[cb.blockIdx] = false;
}
else {
// the block wasn't in use, so free it
reference->unloadBlock(cb.blockIdx);
bytesFreed = reference->blockSize(cb.blockIdx);
m_memUse -= bytesFreed;
CacheList::iterator toRemove = m_nextBlock;
++m_nextBlock;
m_blockCacheList.erase(toRemove);
}
return bytesFreed;
}
//----------------------------------------------------------------------------//
template <class Data_T>
void SparseFileManager::deallocateBlock(CacheList::iterator &it)
{
SparseFile::CacheBlock &cb = *it;
SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(cb.refIdx);
int64_t bytesFreed = reference->blockSize(cb.blockIdx);
m_memUse -= bytesFreed;
reference->unloadBlock(cb.blockIdx);
it = m_blockCacheList.erase(it);
}
//----------------------------------------------------------------------------//
void SparseFileManager::deallocateBlocks(int64_t bytesNeeded)
{
boost::mutex::scoped_lock lock_A(m_mutex);
while (m_blockCacheList.begin() != m_blockCacheList.end() &&
m_maxMemUseInBytes-m_memUse < bytesNeeded) {
if (m_nextBlock == m_blockCacheList.end())
m_nextBlock = m_blockCacheList.begin();
SparseFile::CacheBlock &cb = *m_nextBlock;
// if bytesFreed is set to >0, then we've already freed a block
// and advanced the "clock hand" iterator
int64_t bytesFreed = 0;
switch(cb.blockType) {
case DataTypeHalf:
bytesFreed = deallocateBlock<half>(cb);
if (bytesFreed > 0) {
continue;
}
break;
case DataTypeFloat:
bytesFreed = deallocateBlock<float>(cb);
if (bytesFreed > 0) {
continue;
}
break;
case DataTypeDouble:
bytesFreed = deallocateBlock<double>(cb);
if (bytesFreed > 0) {
continue;
}
break;
case DataTypeVecHalf:
bytesFreed = deallocateBlock<V3h>(cb);
if (bytesFreed > 0) {
continue;
}
break;
case DataTypeVecFloat:
bytesFreed = deallocateBlock<V3f>(cb);
if (bytesFreed > 0) {
continue;
}
break;
case DataTypeVecDouble:
bytesFreed = deallocateBlock<V3d>(cb);
if (bytesFreed > 0) {
continue;
}
break;
case DataTypeUnknown:
default:
break;
}
++m_nextBlock;
}
}
//----------------------------------------------------------------------------//
void SparseFileManager::flushCache()
{
boost::mutex::scoped_lock lock(m_mutex);
CacheList::iterator it = m_blockCacheList.begin();
while (it != m_blockCacheList.end()) {
SparseFile::CacheBlock &cb = *it;
switch(cb.blockType) {
case DataTypeHalf:
deallocateBlock<half>(it);
break;
case DataTypeFloat:
deallocateBlock<float>(it);
break;
case DataTypeDouble:
deallocateBlock<double>(it);
break;
case DataTypeVecHalf:
deallocateBlock<V3h>(it);
break;
case DataTypeVecFloat:
deallocateBlock<V3f>(it);
break;
case DataTypeVecDouble:
deallocateBlock<V3d>(it);
break;
case DataTypeUnknown:
default:
break;
}
}
m_nextBlock = m_blockCacheList.begin();
}
//----------------------------------------------------------------------------//
void SparseFileManager::addBlockToCache(DataTypeEnum blockType,
int fileId, int blockIdx)
{
// Note: this lock is obtained while we also have a lock on the
// specific block (in activateBlock()), so we should make sure we
// never lock the SparseFileManager and *then* a block, to ensure we
// don't have a deadlock.
//
// Note: this was changed so the order was consistent w/ dealloc
// again, see activateBlock()
// boost::mutex::scoped_lock lock(m_mutex);
SparseFile::CacheBlock block(blockType, fileId, blockIdx);
if (m_nextBlock == m_blockCacheList.end()) {
m_blockCacheList.push_back(block);
} else {
m_blockCacheList.insert(m_nextBlock, block);
}
}
//----------------------------------------------------------------------------//
SparseFileManager::SparseFileManager()
: m_memUse(0),
m_limitMemUse(false)
{
setMaxMemUse(1000.0);
m_nextBlock = m_blockCacheList.begin();
}
//----------------------------------------------------------------------------//
long long SparseFileManager::totalLoads()
{
long long int numLoads = 0;
for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
numLoads += m_fileData.ref<half>(i)->totalLoads();
}
for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
numLoads += m_fileData.ref<V3h>(i)->totalLoads();
}
for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
numLoads += m_fileData.ref<float>(i)->totalLoads();
}
for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
numLoads += m_fileData.ref<V3f>(i)->totalLoads();
}
for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
numLoads += m_fileData.ref<double>(i)->totalLoads();
}
for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
numLoads += m_fileData.ref<V3d>(i)->totalLoads();
}
return numLoads;
}
//----------------------------------------------------------------------------//
long long SparseFileManager::numLoadedBlocks()
{
long long int numBlocks = 0;
for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
numBlocks += m_fileData.ref<half>(i)->numLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
numBlocks += m_fileData.ref<V3h>(i)->numLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
numBlocks += m_fileData.ref<float>(i)->numLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
numBlocks += m_fileData.ref<V3f>(i)->numLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
numBlocks += m_fileData.ref<double>(i)->numLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
numBlocks += m_fileData.ref<V3d>(i)->numLoadedBlocks();
}
return numBlocks;
}
//----------------------------------------------------------------------------//
long long SparseFileManager::totalLoadedBlocks()
{
long long int numBlocks = 0;
for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
numBlocks += m_fileData.ref<half>(i)->totalLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
numBlocks += m_fileData.ref<V3h>(i)->totalLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
numBlocks += m_fileData.ref<float>(i)->totalLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
numBlocks += m_fileData.ref<V3f>(i)->totalLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
numBlocks += m_fileData.ref<double>(i)->totalLoadedBlocks();
}
for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
numBlocks += m_fileData.ref<V3d>(i)->totalLoadedBlocks();
}
return numBlocks;
}
//----------------------------------------------------------------------------//
float SparseFileManager::cacheFractionLoaded()
{
return ((double)numLoadedBlocks())/std::max(1.0, ((double)totalLoadedBlocks()));
}
//----------------------------------------------------------------------------//
float SparseFileManager::cacheLoadsPerBlock()
{
return ((double)totalLoads())/std::max(1.0, ((double)totalLoadedBlocks()));
}
//----------------------------------------------------------------------------//
float SparseFileManager::cacheEfficiency()
{
return ((double)totalLoadedBlocks())/std::max(1.0, ((double)totalLoads()));
}
//----------------------------------------------------------------------------//
void SparseFileManager::resetCacheStatistics()
{
for (size_t i=0; i<m_fileData.numRefs<half>(); i++) {
m_fileData.ref<half>(i)->resetCacheStatistics();
}
for (size_t i=0; i<m_fileData.numRefs<V3h>(); i++) {
m_fileData.ref<V3h>(i)->resetCacheStatistics();
}
for (size_t i=0; i<m_fileData.numRefs<float>(); i++) {
m_fileData.ref<float>(i)->resetCacheStatistics();
}
for (size_t i=0; i<m_fileData.numRefs<V3f>(); i++) {
m_fileData.ref<V3f>(i)->resetCacheStatistics();
}
for (size_t i=0; i<m_fileData.numRefs<double>(); i++) {
m_fileData.ref<double>(i)->resetCacheStatistics();
}
for (size_t i=0; i<m_fileData.numRefs<V3d>(); i++) {
m_fileData.ref<V3d>(i)->resetCacheStatistics();
}
}
//----------------------------------------------------------------------------//
long long int SparseFileManager::memSize() const
{
boost::mutex::scoped_lock lock(m_mutex);
return sizeof(*this) + m_fileData.memSize() +
m_blockCacheList.size() * sizeof(SparseFile::CacheBlock);
}
//----------------------------------------------------------------------------//
long long int SparseFile::FileReferences::memSize() const
{
Mutex::scoped_lock lock(m_mutex);
long long int size = 0;
// Size of the std::deque's
size += m_hRefs.size() * sizeof(Reference<half>::Ptr);
size += m_vhRefs.size() * sizeof(Reference<V3h>::Ptr);
size += m_fRefs.size() * sizeof(Reference<float>::Ptr);
size += m_vfRefs.size() * sizeof(Reference<V3f>::Ptr);
size += m_dRefs.size() * sizeof(Reference<double>::Ptr);
size += m_vdRefs.size() * sizeof(Reference<V3d>::Ptr);
// Size of the references themselves
for (size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
size += m_hRefs[i]->memSize();
}
for (size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
size += m_vhRefs[i]->memSize();
}
for (size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
size += m_fRefs[i]->memSize();
}
for (size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
size += m_vfRefs[i]->memSize();
}
for (size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
size += m_dRefs[i]->memSize();
}
for (size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
size += m_vdRefs[i]->memSize();
}
return size;
}
//----------------------------------------------------------------------------//
// Template implementations
//----------------------------------------------------------------------------//
namespace SparseFile {
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::loadBlock(int blockIdx)
{
boost::mutex::scoped_lock lock(m_mutex);
// Allocate the block
#if F3D_NO_BLOCKS_ARRAY
blocks[blockIdx].resize(numVoxels);
assert(blocks[blockIdx].data != NULL);
// Read the data
assert(m_reader || m_ogReader);
if (m_reader) {
m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx].data);
} else {
m_ogReader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx].data);
}
// Mark block as loaded
blockLoaded[blockIdx] = 1;
// Track count
m_numActiveBlocks++;
#else
blocks[blockIdx]->resize(numVoxels);
assert(blocks[blockIdx]->data != NULL);
// Read the data
assert(m_reader || m_ogReader);
if (m_reader) {
m_reader->readBlock(fileBlockIndices[blockIdx], *blocks[blockIdx]->data);
} else {
m_ogReader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->data);
}
// Mark block as loaded
blockLoaded[blockIdx] = 1;
// Track count
m_numActiveBlocks++;
#endif
}
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::openFile()
{
using namespace Exc;
using namespace Hdf5Util;
boost::mutex::scoped_lock lock_A(m_mutex);
// check that the file wasn't already opened before obtaining the lock
if (fileIsOpen()) {
return;
}
// First try Ogawa ---
m_ogArchive.reset(new Alembic::Ogawa::IArchive(filename));
if (m_ogArchive->isValid()) {
m_ogRoot.reset(new OgIGroup(*m_ogArchive));
m_ogLayerGroup.reset(new OgIGroup(m_ogRoot->findGroup(layerPath)));
if (m_ogLayerGroup->isValid()) {
// Allocate the reader
m_ogReaderPtr.reset(new OgSparseDataReader<Data_T>(*m_ogLayerGroup,
numVoxels,
occupiedBlocks,
true));
m_ogReader = m_ogReaderPtr.get();
// Done
return;
}
}
// Then, try HDF5 ---
{
// Hold the global lock
GlobalLock lock(g_hdf5Mutex);
// Open the file
m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
if (m_fileHandle >= 0) {
// Open the layer group
m_layerGroup.open(m_fileHandle, layerPath.c_str());
if (m_layerGroup.id() < 0) {
Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
"Couldn't find layer group " + layerPath +
" in .f3d file ");
throw FileIntegrityException(filename);
}
} else {
Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
"Couldn't open HDF5 file ");
throw NoSuchFileException(filename);
}
}
// Re-allocate reader
if (m_reader) {
delete m_reader;
}
m_reader = new SparseDataReader<Data_T>(m_layerGroup.id(),
valuesPerBlock,
occupiedBlocks);
}
//----------------------------------------------------------------------------//
#define FIELD3D_INSTANTIATION_LOADBLOCK(type) \
template \
void Reference<type>::loadBlock(int blockIdx); \
FIELD3D_INSTANTIATION_LOADBLOCK(float16_t);
FIELD3D_INSTANTIATION_LOADBLOCK(float32_t);
FIELD3D_INSTANTIATION_LOADBLOCK(float64_t);
FIELD3D_INSTANTIATION_LOADBLOCK(vec16_t);
FIELD3D_INSTANTIATION_LOADBLOCK(vec32_t);
FIELD3D_INSTANTIATION_LOADBLOCK(vec64_t);
//----------------------------------------------------------------------------//
#define FIELD3D_INSTANTIATION_OPENFILE(type) \
template \
void Reference<type>::openFile(); \
FIELD3D_INSTANTIATION_OPENFILE(float16_t);
FIELD3D_INSTANTIATION_OPENFILE(float32_t);
FIELD3D_INSTANTIATION_OPENFILE(float64_t);
FIELD3D_INSTANTIATION_OPENFILE(vec16_t);
FIELD3D_INSTANTIATION_OPENFILE(vec32_t);
FIELD3D_INSTANTIATION_OPENFILE(vec64_t);
} // namespace SparseFile
//----------------------------------------------------------------------------//
FIELD3D_NAMESPACE_HEADER_CLOSE
//----------------------------------------------------------------------------//
g_hdf5Mutex
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
Definition Hdf5Util.cpp:67
GlobalLock
boost::recursive_mutex::scoped_lock GlobalLock
Definition Hdf5Util.h:78
SparseField.h
Contains the SparseField class.
FIELD3D_INSTANTIATION_OPENFILE
#define FIELD3D_INSTANTIATION_OPENFILE(type)
Definition SparseFile.cpp:614
FIELD3D_INSTANTIATION_LOADBLOCK
#define FIELD3D_INSTANTIATION_LOADBLOCK(type)
Definition SparseFile.cpp:601
V3d
Imath::V3d V3d
Definition SpiMathLib.h:74
half
FIELD3D_NAMESPACE_OPENtypedef ::half half
Definition SpiMathLib.h:64
V3h
Imath::Vec3< half > V3h
Definition SpiMathLib.h:72
V3f
Imath::V3f V3f
Definition SpiMathLib.h:73
vec32_t
Field3D::V3f vec32_t
Definition Traits.h:92
float32_t
float float32_t
Definition Traits.h:87
vec16_t
Field3D::V3h vec16_t
Definition Traits.h:91
vec64_t
Field3D::V3d vec64_t
Definition Traits.h:93
DataTypeEnum
DataTypeEnum
Definition Traits.h:108
DataTypeFloat
@ DataTypeFloat
Definition Traits.h:112
DataTypeHalf
@ DataTypeHalf
Definition Traits.h:109
DataTypeVecHalf
@ DataTypeVecHalf
Definition Traits.h:114
DataTypeVecDouble
@ DataTypeVecDouble
Definition Traits.h:116
DataTypeUnknown
@ DataTypeUnknown
Definition Traits.h:117
DataTypeVecFloat
@ DataTypeVecFloat
Definition Traits.h:115
DataTypeDouble
@ DataTypeDouble
Definition Traits.h:113
float64_t
double float64_t
Definition Traits.h:88
float16_t
half float16_t
Definition Traits.h:86
OgSparseDataReader
Definition SparseFile.h:87
SparseDataReader
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
Definition SparseDataReader.h:71
SparseFileManager::setMaxMemUse
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
Definition SparseFile.cpp:90
SparseFileManager::cacheFractionLoaded
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
Definition SparseFile.cpp:391
SparseFileManager::resetCacheStatistics
void resetCacheStatistics()
Resets block load.
Definition SparseFile.cpp:412
SparseFileManager::m_limitMemUse
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition SparseFile.h:527
SparseFileManager::m_blockCacheList
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
Definition SparseFile.h:538
SparseFileManager::deallocateBlock
int64_t deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
Definition SparseFile.cpp:99
SparseFileManager::cacheLoadsPerBlock
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
Definition SparseFile.cpp:398
SparseFileManager::m_memUse
int64_t m_memUse
Current amount of memory in use in bytes.
Definition SparseFile.h:523
SparseFileManager::m_nextBlock
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition SparseFile.h:542
SparseFileManager::setLimitMemUse
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
Definition SparseFile.cpp:76
SparseFileManager::doLimitMemUse
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Definition SparseFile.cpp:83
SparseFileManager::m_mutex
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition SparseFile.h:546
SparseFileManager::m_fileData
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition SparseFile.h:531
SparseFileManager::addBlockToCache
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
Definition SparseFile.cpp:260
SparseFileManager::cacheEfficiency
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads....
Definition SparseFile.cpp:405
SparseFileManager::singleton
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Definition SparseFile.cpp:66
SparseFileManager::reference
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
Definition SparseFile.h:1248
SparseFileManager::ms_singleton
static boost::scoped_ptr< SparseFileManager > ms_singleton
Pointer to singleton.
Definition SparseFile.h:498
SparseFileManager::memSize
long long int memSize() const
Returns the number of bytes used by the SparseFileManager itself.
Definition SparseFile.cpp:442
SparseFileManager::flushCache
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
Definition SparseFile.cpp:223
SparseFileManager::m_maxMemUse
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition SparseFile.h:517
SparseFileManager::deallocateBlocks
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
Definition SparseFile.cpp:160
SparseFileManager::totalLoadedBlocks
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
Definition SparseFile.cpp:358
SparseFileManager::SparseFileManager
SparseFileManager()
Private to prevent instantiation.
Definition SparseFile.cpp:282
SparseFileManager::numLoadedBlocks
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
Definition SparseFile.cpp:325
SparseFileManager::totalLoads
long long totalLoads()
Returns the total number of block loads in the cache.
Definition SparseFile.cpp:292
SparseFileManager::m_maxMemUseInBytes
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition SparseFile.h:520
SparseFile::CacheBlock
Definition SparseFile.h:322
SparseFile::CacheBlock::refIdx
int refIdx
Definition SparseFile.h:325
SparseFile::CacheBlock::blockType
DataTypeEnum blockType
Definition SparseFile.h:324
SparseFile::CacheBlock::blockIdx
int blockIdx
Definition SparseFile.h:326
SparseFile::FileReferences::memSize
long long int memSize() const
Returns the memory use for the refs.
Definition SparseFile.cpp:452
SparseFile::FileReferences::m_dRefs
std::deque< Reference< double >::Ptr > m_dRefs
Definition SparseFile.h:309
SparseFile::FileReferences::m_fRefs
std::deque< Reference< float >::Ptr > m_fRefs
Definition SparseFile.h:307
SparseFile::FileReferences::m_mutex
Mutex m_mutex
Definition SparseFile.h:316
SparseFile::FileReferences::m_vdRefs
std::deque< Reference< V3d >::Ptr > m_vdRefs
Definition SparseFile.h:310
SparseFile::FileReferences::m_vfRefs
std::deque< Reference< V3f >::Ptr > m_vfRefs
Definition SparseFile.h:308
SparseFile::FileReferences::m_hRefs
std::deque< Reference< half >::Ptr > m_hRefs
Definition SparseFile.h:305
SparseFile::FileReferences::m_vhRefs
std::deque< Reference< V3h >::Ptr > m_vhRefs
Definition SparseFile.h:306
SparseFile::Reference
Definition SparseFile.h:106
SparseFile::Reference::openFile
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
Definition SparseFile.cpp:537
SparseFile::Reference::layerPath
std::string layerPath
Definition SparseFile.h:128
SparseFile::Reference::blocks
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please.
Definition SparseFile.h:141
SparseFile::Reference::fileBlockIndices
std::vector< int > fileBlockIndices
Index in file for each block.
Definition SparseFile.h:135
SparseFile::Reference::valuesPerBlock
int valuesPerBlock
Definition SparseFile.h:129
SparseFile::Reference::filename
std::string filename
Definition SparseFile.h:127
SparseFile::Reference::m_ogRoot
OgIGroupPtr m_ogRoot
Ogawa archive root.
Definition SparseFile.h:249
SparseFile::Reference::m_mutex
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Definition SparseFile.h:254
SparseFile::Reference::m_layerGroup
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
Definition SparseFile.h:235
SparseFile::Reference::m_ogArchive
IArchivePtr m_ogArchive
Ogawa archive.
Definition SparseFile.h:247
SparseFile::Reference::m_ogLayerGroup
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
Definition SparseFile.h:251
SparseFile::Reference::blockLoaded
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
Definition SparseFile.h:138
SparseFile::Reference::m_ogReader
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
Definition SparseFile.h:245
SparseFile::Reference::m_ogReaderPtr
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
Definition SparseFile.h:242
SparseFile::Reference::m_reader
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition SparseFile.h:239
SparseFile::Reference::numVoxels
int numVoxels
Definition SparseFile.h:130
SparseFile::Reference::loadBlock
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Definition SparseFile.cpp:498
SparseFile::Reference::m_fileHandle
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
Definition SparseFile.h:231
SparseFile::Reference::occupiedBlocks
int occupiedBlocks
Definition SparseFile.h:132
SparseFile::Reference::m_numActiveBlocks
size_t m_numActiveBlocks
Number of currently active blocks.
Definition SparseFile.h:257
SparseFile::Reference::Ptr
Reference * Ptr
Definition SparseFile.h:116
SparseFile::Reference::fileIsOpen
bool fileIsOpen()
Checks if the file used by this reference is open already.
Definition SparseFile.h:669
Exc
Namespace for Exception objects.
Definition Exception.h:57
FIELD3D_NAMESPACE_OPEN
Definition FieldMapping.cpp:74
Hdf5Util
Contains utility functions and classes for Hdf5 files.
Definition Hdf5Util.h:86
Msg::SevWarning
@ SevWarning
Definition Log.h:68
Msg::print
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Definition Log.cpp:70
SparseFile
Definition SparseFile.h:91
FIELD3D_NAMESPACE_HEADER_CLOSE
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition ns.h:58