SparseFile.h
Go to the documentation of this file.
//----------------------------------------------------------------------------//
/*
* Copyright (c) 2009 Sony Pictures Imageworks Inc
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the
* distribution. Neither the name of Sony Pictures Imageworks nor the
* names of its contributors may be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
*/
//----------------------------------------------------------------------------//
//----------------------------------------------------------------------------//
#ifndef _INCLUDED_Field3D_SparseFile_H_
#define _INCLUDED_Field3D_SparseFile_H_
//----------------------------------------------------------------------------//
#include <boost/scoped_ptr.hpp>
#include <deque>
#include <list>
#include <vector>
#include <hdf5.h>
#include "Exception.h"
#include "Hdf5Util.h"
#include "OgawaFwd.h"
#include "SparseDataReader.h"
#include "Traits.h"
//----------------------------------------------------------------------------//
#define F3D_SHORT_MUTEX_ARRAY 1
#define F3D_MUTEX_ARRAY_SIZE 1000
#define F3D_NO_BLOCKS_ARRAY 1
#include "ns.h"
FIELD3D_NAMESPACE_OPEN
//----------------------------------------------------------------------------//
// Forward declarations
//----------------------------------------------------------------------------//
namespace Sparse {
template <typename Data_T>
struct SparseBlock;
}
template <typename Data_T>
class SparseField;
template <typename Data_T>
class OgSparseDataReader;
//----------------------------------------------------------------------------//
namespace SparseFile {
//----------------------------------------------------------------------------//
// Reference
//----------------------------------------------------------------------------//
//----------------------------------------------------------------------------//
template <class Data_T>
class Reference
{
public:
// Typedefs ------------------------------------------------------------------
#define USE_SHPTR 0
#if USE_SHPTR
typedef boost::shared_ptr<Reference> Ptr;
#else
typedef Reference* Ptr;
#endif
#if F3D_NO_BLOCKS_ARRAY
typedef Sparse::SparseBlock<Data_T>* BlockPtrs;
#else
typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
#endif
// Public data members -------------------------------------------------------
std::string filename;
std::string layerPath;
int valuesPerBlock;
int numVoxels;
int numBlocks;
int occupiedBlocks;
std::vector<int> fileBlockIndices;
std::vector<int> blockLoaded;
BlockPtrs blocks;
std::vector<bool> blockUsed;
std::vector<int> loadCounts;
std::vector<int> refCounts;
boost::mutex *blockMutex;
#if F3D_SHORT_MUTEX_ARRAY
int blockMutexSize;
#endif
// Ctors, dtor ---------------------------------------------------------------
~Reference();
// Main methods --------------------------------------------------------------
static Ptr create(const std::string a_filename,
const std::string a_layerPath);
bool fileIsOpen();
void setNumBlocks(int numBlocks);
void openFile();
void closeFile();
void loadBlock(int blockIdx);
void unloadBlock(int blockIdx);
void incBlockRef(int blockIdx);
void decBlockRef(int blockIdx);
int blockSize(int blockIdx) const;
int totalLoads() const;
int numLoadedBlocks() const;
int totalLoadedBlocks() const;
float averageLoads() const;
void resetCacheStatistics();
long long int memSize() const;
private:
typedef boost::mutex Mutex;
// Private constructors ---
Reference(const std::string filename, const std::string layerPath);
Reference(const Reference &o);
Reference & operator=(const Reference &o);
// Data members ---
hid_t m_fileHandle;
Hdf5Util::H5ScopedGopen m_layerGroup;
SparseDataReader<Data_T> *m_reader;
boost::shared_ptr<OgSparseDataReader<Data_T> > m_ogReaderPtr;
OgSparseDataReader<Data_T> *m_ogReader;
IArchivePtr m_ogArchive;
OgIGroupPtr m_ogRoot;
OgIGroupPtr m_ogLayerGroup;
mutable Mutex m_mutex;
size_t m_numActiveBlocks;
};
//----------------------------------------------------------------------------//
// FileReferences
//----------------------------------------------------------------------------//
class FileReferences
{
public:
/* A note on thread safety
FileReferences uses a std::deque to store the References, which means
that already-added references 'stay put' in memory, even as more elements
are added. For threading purposes, we only need to lock during append()
and numRefs(), but ref() is safe, since we never remove references.
*/
// Ctors, dtor ---------------------------------------------------------------
~FileReferences();
// Main methods --------------------------------------------------------------
template <class Data_T>
Reference<Data_T>* ref(size_t idx);
template <class Data_T>
size_t append(typename Reference<Data_T>::Ptr ref);
template <class Data_T>
size_t numRefs() const;
long long int memSize() const;
private:
// Data members --------------------------------------------------------------
std::deque<Reference<half>::Ptr> m_hRefs;
std::deque<Reference<V3h>::Ptr> m_vhRefs;
std::deque<Reference<float>::Ptr> m_fRefs;
std::deque<Reference<V3f>::Ptr> m_vfRefs;
std::deque<Reference<double>::Ptr> m_dRefs;
std::deque<Reference<V3d>::Ptr> m_vdRefs;
// Mutexes -------------------------------------------------------------------
typedef boost::mutex Mutex;
mutable Mutex m_mutex;
};
//----------------------------------------------------------------------------//
class CacheBlock {
public:
DataTypeEnum blockType;
int refIdx;
int blockIdx;
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn) :
blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
{ }
};
//----------------------------------------------------------------------------//
} // namespace SparseFile
//----------------------------------------------------------------------------//
// SparseFileManager
//----------------------------------------------------------------------------//
//----------------------------------------------------------------------------//
class FIELD3D_API SparseFileManager
{
public:
template <class Data_T>
friend class SparseField;
// typedefs ------------------------------------------------------------------
typedef std::list<SparseFile::CacheBlock> CacheList;
// Main methods --------------------------------------------------------------
static SparseFileManager &singleton();
void setLimitMemUse(bool enabled);
bool doLimitMemUse() const;
void setMaxMemUse(float maxMemUse);
void flushCache();
long long totalLoads();
long long numLoadedBlocks();
long long totalLoadedBlocks();
float cacheFractionLoaded();
float cacheLoadsPerBlock();
float cacheEfficiency();
void resetCacheStatistics();
long long int memSize() const;
//--------------------------------------------------------------------------//
// Utility functions
template <class Data_T>
void incBlockRef(int fileId, int blockIdx);
template <class Data_T>
void decBlockRef(int fileId, int blockIdx);
template <class Data_T>
void activateBlock(int fileId, int blockIdx);
protected:
template <class Data_T>
SparseFile::Reference<Data_T>* reference(int index);
template <class Data_T>
int getNextId(const std::string filename, const std::string layerPath);
template <class Data_T>
void removeFieldFromCache(int refIdx);
private:
SparseFileManager();
static boost::scoped_ptr<SparseFileManager> ms_singleton;
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
void deallocateBlocks(int64_t bytesNeeded);
template <class Data_T>
int64_t deallocateBlock(const SparseFile::CacheBlock &cb);
template <class Data_T>
void deallocateBlock(CacheList::iterator &it);
float m_maxMemUse;
int64_t m_maxMemUseInBytes;
int64_t m_memUse;
bool m_limitMemUse;
SparseFile::FileReferences m_fileData;
CacheList m_blockCacheList;
CacheList::iterator m_nextBlock;
mutable boost::mutex m_mutex;
};
//----------------------------------------------------------------------------//
// Reference implementations
//----------------------------------------------------------------------------//
namespace SparseFile {
//----------------------------------------------------------------------------//
template <class Data_T>
Reference<Data_T>::Reference(const std::string a_filename,
const std::string a_layerPath)
: filename(a_filename), layerPath(a_layerPath),
valuesPerBlock(-1), numVoxels(-1), numBlocks(-1), occupiedBlocks(-1),
blockMutex(NULL), m_fileHandle(-1), m_reader(NULL), m_ogReader(NULL),
m_numActiveBlocks(0)
{
/* Empty */
}
//----------------------------------------------------------------------------//
template <class Data_T>
Reference<Data_T>::~Reference()
{
closeFile();
if (m_reader) {
delete m_reader;
}
if (blockMutex)
delete [] blockMutex;
}
//----------------------------------------------------------------------------//
template <class Data_T>
Reference<Data_T>::Reference(const Reference<Data_T> &o)
{
m_ogReaderPtr.reset();
m_ogReader = NULL;
m_reader = NULL;
blockMutex = NULL;
*this = o;
}
//----------------------------------------------------------------------------//
template <class Data_T>
Reference<Data_T> &
Reference<Data_T>::operator=(const Reference<Data_T> &o)
{
if (this == &o) {
return *this;
}
// Copy public member variables (where appropriate)
filename = o.filename;
layerPath = o.layerPath;
valuesPerBlock = o.valuesPerBlock;
numVoxels = o.numVoxels;
occupiedBlocks = o.occupiedBlocks;
fileBlockIndices = o.fileBlockIndices;
blockLoaded = o.blockLoaded;
blocks = o.blocks;
blockUsed = o.blockUsed;
loadCounts = o.loadCounts;
refCounts = o.refCounts;
if (blockMutex)
delete[] blockMutex;
#if F3D_SHORT_MUTEX_ARRAY
# if F3D_NO_BLOCKS_ARRAY
blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
# else
blockMutexSize = std::min(static_cast<int>(blocks.size()),
F3D_MUTEX_ARRAY_SIZE);
# endif
blockMutex = new boost::mutex[blockMutexSize];
#else
blockMutex = new boost::mutex[blocks.size()];
#endif
#if 0
// MW: Should this be copying the file handle? Never seems to happen,
// but it also seems bad. Ifdef'ing out for now.
// Copy private member variables (where appropriate)
m_fileHandle = o.m_fileHandle;
// Don't copy id, let hdf5 generate a new one.
if (m_fileHandle >= 0) {
m_layerGroup.open(m_fileHandle, layerPath.c_str());
}
#else
m_fileHandle = -1;
#endif
// Re-allocate reader
if (m_reader)
delete m_reader;
m_reader = NULL;
m_ogReaderPtr.reset();
m_ogReader = NULL;
return *this;
}
//----------------------------------------------------------------------------//
template <class Data_T>
typename Reference<Data_T>::Ptr
Reference<Data_T>::create(const std::string a_filename,
const std::string a_layerPath)
{
return Ptr(new Reference(a_filename, a_layerPath));
}
//----------------------------------------------------------------------------//
template <class Data_T>
bool Reference<Data_T>::fileIsOpen()
{
return m_fileHandle >= 0;
}
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::setNumBlocks(int a_numBlocks)
{
boost::mutex::scoped_lock lock(m_mutex);
// Store number of blocks in member variable
numBlocks = a_numBlocks;
fileBlockIndices.resize(numBlocks);
blockLoaded.resize(numBlocks, 0);
#if !F3D_NO_BLOCKS_ARRAY
blocks.resize(numBlocks, 0);
#endif
blockUsed.resize(numBlocks, false);
loadCounts.resize(numBlocks, 0);
refCounts.resize(numBlocks, 0);
if (blockMutex)
delete[] blockMutex;
#if F3D_SHORT_MUTEX_ARRAY
# if F3D_NO_BLOCKS_ARRAY
blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
# else
blockMutexSize = std::min(blocks.size(),
static_cast<size_t>(F3D_MUTEX_ARRAY_SIZE));
# endif
blockMutex = new boost::mutex[blockMutexSize];
#else
blockMutex = new boost::mutex[numBlocks];
#endif
}
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::closeFile()
{
if (m_fileHandle >= 0) {
if (H5Fclose(m_fileHandle) < 0) {
Msg::print("In ~Reference(): Error closing file " + filename);
}
}
}
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::unloadBlock(int blockIdx)
{
// Deallocate the block
#if F3D_NO_BLOCKS_ARRAY
blocks[blockIdx].clear();
#else
blocks[blockIdx]->clear();
#endif
// Mark block as unloaded
blockLoaded[blockIdx] = 0;
// Track count
m_numActiveBlocks--;
#if 0
// If no active blocks, close the file. De-activate for now.
if (m_numActiveBlocks == 0) {
closeFile();
}
#endif
}
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::incBlockRef(int blockIdx)
{
#if F3D_SHORT_MUTEX_ARRAY
boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
#else
boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
#endif
++refCounts[blockIdx];
}
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::decBlockRef(int blockIdx)
{
#if F3D_SHORT_MUTEX_ARRAY
boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
#else
boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
#endif
--refCounts[blockIdx];
}
//----------------------------------------------------------------------------//
template <class Data_T>
int Reference<Data_T>::blockSize(int /* blockIdx */) const
{
return numVoxels * sizeof(Data_T);
}
//----------------------------------------------------------------------------//
template <class Data_T>
int Reference<Data_T>::totalLoads() const
{
std::vector<int>::const_iterator i = loadCounts.begin();
std::vector<int>::const_iterator end = loadCounts.end();
int numLoads = 0;
for (; i != end; ++i)
numLoads += *i;
return numLoads;
}
//----------------------------------------------------------------------------//
template <class Data_T>
int Reference<Data_T>::numLoadedBlocks() const
{
std::vector<int>::const_iterator i = blockLoaded.begin();
std::vector<int>::const_iterator end = blockLoaded.end();
int numBlockCounter = 0;
for (; i != end; ++i)
if (*i)
numBlockCounter++;
return numBlockCounter;
}
//----------------------------------------------------------------------------//
template <class Data_T>
int Reference<Data_T>::totalLoadedBlocks() const
{
std::vector<int>::const_iterator i = loadCounts.begin();
std::vector<int>::const_iterator li = blockLoaded.begin();
std::vector<int>::const_iterator end = loadCounts.end();
int numBlockCounter = 0;
if (blockLoaded.size() == 0) {
for (; i != end; ++i)
if (*i)
numBlockCounter++;
} else {
assert(loadCounts.size() == blockLoaded.size());
for (; i != end; ++i, ++li)
if (*i || *li)
numBlockCounter++;
}
return numBlockCounter;
}
//----------------------------------------------------------------------------//
template <class Data_T>
float Reference<Data_T>::averageLoads() const
{
std::vector<int>::const_iterator i = loadCounts.begin();
std::vector<int>::const_iterator end = loadCounts.end();
int numLoads = 0, numBlockCounter = 0;
for (; i != end; ++i) {
if (*i) {
numLoads += *i;
numBlockCounter++;
}
}
return (float)numLoads / std::max(1, numBlockCounter);
}
//----------------------------------------------------------------------------//
template <class Data_T>
void Reference<Data_T>::resetCacheStatistics()
{
std::vector<int>::iterator li = loadCounts.begin();
std::vector<int>::iterator lend = loadCounts.end();
for (; li != lend; ++li)
*li = 0;
}
//----------------------------------------------------------------------------//
template <class Data_T>
long long int
Reference<Data_T>::memSize() const
{
boost::mutex::scoped_lock lock(m_mutex);
return sizeof(*this) +
fileBlockIndices.capacity() * sizeof(int) +
blockLoaded.capacity() * sizeof(int) +
#if !F3D_NO_BLOCKS_ARRAY
blocks.capacity() * sizeof(Sparse::SparseBlock<Data_T>*) +
#endif
blockUsed.capacity() * sizeof(bool) +
loadCounts.capacity() * sizeof(int) +
refCounts.capacity() * sizeof(int) +
#if F3D_SHORT_MUTEX_ARRAY
blockMutexSize * sizeof(boost::mutex) +
#else
numBlocks * sizeof(boost::mutex) +
#endif
sizeof(SparseDataReader<Data_T>);
}
//----------------------------------------------------------------------------//
} // namespace SparseFile
//----------------------------------------------------------------------------//
// Specializations for FileReferences
//----------------------------------------------------------------------------//
namespace SparseFile {
//----------------------------------------------------------------------------//
inline FileReferences::~FileReferences()
{
#if !USE_SHPTR
for (size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
delete m_hRefs[i];
}
for (size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
delete m_fRefs[i];
}
for (size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
delete m_dRefs[i];
}
for (size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
delete m_vhRefs[i];
}
for (size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
delete m_vfRefs[i];
}
for (size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
delete m_vdRefs[i];
}
#endif
}
//----------------------------------------------------------------------------//
template <>
inline Reference<half>*
FileReferences::ref(size_t idx)
{
#if USE_SHPTR
return m_hRefs[idx].get();
#else
return m_hRefs[idx];
#endif
}
//----------------------------------------------------------------------------//
template <>
inline Reference<V3h>*
FileReferences::ref(size_t idx)
{
#if USE_SHPTR
return m_vhRefs[idx].get();
#else
return m_vhRefs[idx];
#endif
}
//----------------------------------------------------------------------------//
template <>
inline Reference<float>*
FileReferences::ref(size_t idx)
{
#if USE_SHPTR
return m_fRefs[idx].get();
#else
return m_fRefs[idx];
#endif
}
//----------------------------------------------------------------------------//
template <>
inline Reference<V3f>*
FileReferences::ref(size_t idx)
{
#if USE_SHPTR
return m_vfRefs[idx].get();
#else
return m_vfRefs[idx];
#endif
}
//----------------------------------------------------------------------------//
template <>
inline Reference<double>*
FileReferences::ref(size_t idx)
{
#if USE_SHPTR
return m_dRefs[idx].get();
#else
return m_dRefs[idx];
#endif
}
//----------------------------------------------------------------------------//
template <>
inline Reference<V3d>*
FileReferences::ref(size_t idx)
{
#if USE_SHPTR
return m_vdRefs[idx].get();
#else
return m_vdRefs[idx];
#endif
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::append<half>(Reference<half>::Ptr ref)
{
Mutex::scoped_lock lock(m_mutex);
m_hRefs.push_back(ref);
return m_hRefs.size() - 1;
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::append<V3h>(Reference<V3h>::Ptr ref)
{
Mutex::scoped_lock lock(m_mutex);
m_vhRefs.push_back(ref);
return m_vhRefs.size() - 1;
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::append<float>(Reference<float>::Ptr ref)
{
Mutex::scoped_lock lock(m_mutex);
m_fRefs.push_back(ref);
return m_fRefs.size() - 1;
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::append<V3f>(Reference<V3f>::Ptr ref)
{
Mutex::scoped_lock lock(m_mutex);
m_vfRefs.push_back(ref);
return m_vfRefs.size() - 1;
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::append<double>(Reference<double>::Ptr ref)
{
Mutex::scoped_lock lock(m_mutex);
m_dRefs.push_back(ref);
return m_dRefs.size() - 1;
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::append<V3d>(Reference<V3d>::Ptr ref)
{
Mutex::scoped_lock lock(m_mutex);
m_vdRefs.push_back(ref);
return m_vdRefs.size() - 1;
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::numRefs<half>() const
{
Mutex::scoped_lock lock(m_mutex);
return m_hRefs.size();
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::numRefs<V3h>() const
{
Mutex::scoped_lock lock(m_mutex);
return m_vhRefs.size();
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::numRefs<float>() const
{
Mutex::scoped_lock lock(m_mutex);
return m_fRefs.size();
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::numRefs<V3f>() const
{
Mutex::scoped_lock lock(m_mutex);
return m_vfRefs.size();
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::numRefs<double>() const
{
Mutex::scoped_lock lock(m_mutex);
return m_dRefs.size();
}
//----------------------------------------------------------------------------//
template <>
inline size_t FileReferences::numRefs<V3d>() const
{
Mutex::scoped_lock lock(m_mutex);
return m_vdRefs.size();
}
//----------------------------------------------------------------------------//
// Implementations for FileReferences
//----------------------------------------------------------------------------//
#if 0
template <class Data_T>
Reference<Data_T>& FileReferences::ref(size_t)
{
assert(false && "Do not use memory limiting on sparse fields that aren't "
"simple scalars or vectors!");
Msg::print(Msg::SevWarning,
"FileReferences::ref(): Do not use memory limiting on sparse "
"fields that aren't simple scalars or vectors!");
static Reference<Data_T> dummy("", "");
return dummy;
}
//----------------------------------------------------------------------------//
template <class Data_T>
size_t FileReferences::append(const Reference<Data_T>&)
{
assert(false && "Do not use memory limiting on sparse fields that aren't "
"simple scalars or vectors!");
Msg::print(Msg::SevWarning,
"FileReferences::append(): Do not use memory limiting on sparse "
"fields that aren't simple scalars or vectors!");
return -1;
}
//----------------------------------------------------------------------------//
template <class Data_T>
size_t FileReferences::numRefs() const
{
assert(false && "Do not use memory limiting on sparse fields that aren't "
"simple scalars or vectors!");
Msg::print(Msg::SevWarning,
"FileReferences::numRefs(): "
"Do not use memory limiting on sparse "
"fields that aren't "
"simple scalars or vectors!");
return -1;
}
#endif
//----------------------------------------------------------------------------//
} // namespace SparseFile
//----------------------------------------------------------------------------//
// SparseFileManager implementations
//----------------------------------------------------------------------------//
template <class Data_T>
int
SparseFileManager::getNextId(const std::string filename,
const std::string layerPath)
{
using namespace SparseFile;
// Must hold a mutex while appending to m_fileData
boost::mutex::scoped_lock lock(m_mutex);
int id = m_fileData.append<Data_T>(Reference<Data_T>::create(filename,
layerPath));
return id;
}
//----------------------------------------------------------------------------//
template <class Data_T>
void
SparseFileManager::removeFieldFromCache(int refIdx)
{
boost::mutex::scoped_lock lock(m_mutex);
DataTypeEnum blockType = DataTypeTraits<Data_T>::typeEnum();
SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(refIdx);
CacheList::iterator it = m_blockCacheList.begin();
CacheList::iterator end = m_blockCacheList.end();
CacheList::iterator next;
int64_t bytesFreed = 0;
while (it != end) {
if (it->blockType == blockType && it->refIdx == refIdx) {
if (it == m_nextBlock) {
++m_nextBlock;
}
next = it;
++next;
bytesFreed += reference->blockSize(it->blockIdx);
m_blockCacheList.erase(it);
it = next;
} else {
++it;
}
}
m_memUse -= bytesFreed;
std::vector<int>().swap(reference->fileBlockIndices);
#if F3D_NO_BLOCKS_ARRAY
reference->fileBlockIndices.resize(reference->numBlocks, -1);
#else
reference->fileBlockIndices.resize(reference->blocks.size(), -1);
typedef typename SparseFile::Reference<Data_T>::BlockPtrs BlockPtrs;
BlockPtrs().swap(reference->blocks);
#endif
std::vector<int>().swap(reference->blockLoaded);
std::vector<bool>().swap(reference->blockUsed);
std::vector<int>().swap(reference->loadCounts);
std::vector<int>().swap(reference->refCounts);
delete[] reference->blockMutex;
reference->blockMutex = NULL;
}
//----------------------------------------------------------------------------//
template <class Data_T>
SparseFile::Reference<Data_T>*
SparseFileManager::reference(int index)
{
return m_fileData.ref<Data_T>(index);
}
//----------------------------------------------------------------------------//
template <class Data_T>
void
SparseFileManager::activateBlock(int fileId, int blockIdx)
{
SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(fileId);
if (reference->fileBlockIndices[blockIdx] >= 0) {
if (!reference->blockLoaded[blockIdx]) {
int blockSize = reference->blockSize(blockIdx);
if (m_limitMemUse) {
// if we already have enough free memory, deallocateBlocks()
// will just return
deallocateBlocks(blockSize);
}
if (!reference->fileIsOpen()) {
reference->openFile();
}
boost::mutex::scoped_lock lock_A(m_mutex);
#if F3D_SHORT_MUTEX_ARRAY
boost::mutex::scoped_lock
lock_B(reference->blockMutex[blockIdx % reference->blockMutexSize]);
#else
boost::mutex::scoped_lock lock_B(reference->blockMutex[blockIdx]);
#endif
// check to see if it was loaded between when the function
// started and we got the lock on the block
if (!reference->blockLoaded[blockIdx]) {
reference->loadBlock(blockIdx);
reference->loadCounts[blockIdx]++;
addBlockToCache(DataTypeTraits<Data_T>::typeEnum(), fileId, blockIdx);
m_memUse += blockSize;
}
}
}
reference->blockUsed[blockIdx] = true;
}
//----------------------------------------------------------------------------//
template <class Data_T>
void
SparseFileManager::incBlockRef(int fileId, int blockIdx)
{
SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(fileId);
if (reference->fileBlockIndices[blockIdx] >= 0) {
reference->incBlockRef(blockIdx);
}
}
//----------------------------------------------------------------------------//
template <class Data_T>
void
SparseFileManager::decBlockRef(int fileId, int blockIdx)
{
SparseFile::Reference<Data_T> *reference = m_fileData.ref<Data_T>(fileId);
if (reference->fileBlockIndices[blockIdx] >= 0) {
reference->decBlockRef(blockIdx);
}
}
//----------------------------------------------------------------------------//
FIELD3D_NAMESPACE_HEADER_CLOSE
//----------------------------------------------------------------------------//
#endif
Exception.h
Contains Exception base class.
Hdf5Util.h
Contains various utility functions for Hdf5.
OgawaFwd.h
Contains forward declarations for Ogawa classes.
IArchivePtr
boost::shared_ptr< Alembic::Ogawa::IArchive > IArchivePtr
Definition OgawaFwd.h:87
OgIGroupPtr
boost::shared_ptr< OgIGroup > OgIGroupPtr
Definition OgawaFwd.h:71
SparseDataReader.h
Contains functions controlling the loading of sparse fields.
F3D_SHORT_MUTEX_ARRAY
#define F3D_SHORT_MUTEX_ARRAY
Definition SparseFile.h:64
F3D_NO_BLOCKS_ARRAY
#define F3D_NO_BLOCKS_ARRAY
Definition SparseFile.h:66
F3D_MUTEX_ARRAY_SIZE
#define F3D_MUTEX_ARRAY_SIZE
Definition SparseFile.h:65
Traits.h
DataTypeEnum
DataTypeEnum
Definition Traits.h:108
Ptr
Hdf5Util::H5ScopedGopen
Scoped object - opens a group on creation and closes it on destruction.
Definition Hdf5Util.h:195
OgSparseDataReader
Definition SparseFile.h:87
SparseDataReader
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
Definition SparseDataReader.h:71
SparseField
This Field subclass stores voxel data in block-allocated arrays.
Definition SparseField.h:352
SparseFileManager::setMaxMemUse
void setMaxMemUse(float maxMemUse)
Sets the maximum memory usage, in MB, by dynamically loaded sparse fields.
Definition SparseFile.cpp:90
SparseFileManager::cacheFractionLoaded
float cacheFractionLoaded()
Computes the ratio of blocks in the cache to the total number of blocks that have been loaded (includ...
Definition SparseFile.cpp:391
SparseFileManager::SparseField
friend class SparseField
Definition SparseFile.h:403
SparseFileManager::resetCacheStatistics
void resetCacheStatistics()
Resets block load.
Definition SparseFile.cpp:412
SparseFileManager::m_limitMemUse
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition SparseFile.h:527
SparseFileManager::m_blockCacheList
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
Definition SparseFile.h:538
SparseFileManager::deallocateBlock
int64_t deallocateBlock(const SparseFile::CacheBlock &cb)
Utility function to attempt to deallocate a single block and advance the "hand".
Definition SparseFile.cpp:99
SparseFileManager::cacheLoadsPerBlock
float cacheLoadsPerBlock()
Computes the overall loaded-blocks-to-load ratio for cached files.
Definition SparseFile.cpp:398
SparseFileManager::getNextId
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
Definition SparseFile.h:1182
SparseFileManager::m_memUse
int64_t m_memUse
Current amount of memory in use in bytes.
Definition SparseFile.h:523
SparseFileManager::activateBlock
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user,...
Definition SparseFile.h:1257
SparseFileManager::m_nextBlock
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition SparseFile.h:542
SparseFileManager::setLimitMemUse
void setLimitMemUse(bool enabled)
Sets whether to limit memory usage and do dynamic loading for sparse fields.
Definition SparseFile.cpp:76
SparseFileManager::removeFieldFromCache
void removeFieldFromCache(int refIdx)
Definition SparseFile.h:1199
SparseFileManager::decBlockRef
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
Definition SparseFile.h:1311
SparseFileManager::doLimitMemUse
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Definition SparseFile.cpp:83
SparseFileManager::CacheList
std::list< SparseFile::CacheBlock > CacheList
Definition SparseFile.h:407
SparseFileManager::m_mutex
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition SparseFile.h:546
SparseFileManager::m_fileData
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition SparseFile.h:531
SparseFileManager::addBlockToCache
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
Definition SparseFile.cpp:260
SparseFileManager::cacheEfficiency
float cacheEfficiency()
Computes the efficiency, the ratio of the number of blocks ever loaded to the number of loads....
Definition SparseFile.cpp:405
SparseFileManager::singleton
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Definition SparseFile.cpp:66
SparseFileManager::reference
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
Definition SparseFile.h:1248
SparseFileManager::ms_singleton
static boost::scoped_ptr< SparseFileManager > ms_singleton
Pointer to singleton.
Definition SparseFile.h:498
SparseFileManager::memSize
long long int memSize() const
Returns the number of bytes used by the SparseFileManager itself.
Definition SparseFile.cpp:442
SparseFileManager::flushCache
void flushCache()
Flushes the entire block cache for all files, should probably only be used for debugging.
Definition SparseFile.cpp:223
SparseFileManager::m_maxMemUse
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition SparseFile.h:517
SparseFileManager::deallocateBlocks
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
Definition SparseFile.cpp:160
SparseFileManager::totalLoadedBlocks
long long totalLoadedBlocks()
Returns the total number of blocks loaded (max 1 per block) into cache.
Definition SparseFile.cpp:358
SparseFileManager::SparseFileManager
SparseFileManager()
Private to prevent instantiation.
Definition SparseFile.cpp:282
SparseFileManager::incBlockRef
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
Definition SparseFile.h:1298
SparseFileManager::numLoadedBlocks
long long numLoadedBlocks()
Returns the total number of blocks currently loaded into cache.
Definition SparseFile.cpp:325
SparseFileManager::totalLoads
long long totalLoads()
Returns the total number of block loads in the cache.
Definition SparseFile.cpp:292
SparseFileManager::m_maxMemUseInBytes
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition SparseFile.h:520
SparseFile::CacheBlock
Definition SparseFile.h:322
SparseFile::CacheBlock::refIdx
int refIdx
Definition SparseFile.h:325
SparseFile::CacheBlock::CacheBlock
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
Definition SparseFile.h:327
SparseFile::CacheBlock::blockType
DataTypeEnum blockType
Definition SparseFile.h:324
SparseFile::CacheBlock::blockIdx
int blockIdx
Definition SparseFile.h:326
SparseFile::FileReferences
Definition SparseFile.h:266
SparseFile::FileReferences::~FileReferences
~FileReferences()
Definition SparseFile.h:896
SparseFile::FileReferences::ref
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed.
SparseFile::FileReferences::memSize
long long int memSize() const
Returns the memory use for the refs.
Definition SparseFile.cpp:452
SparseFile::FileReferences::m_dRefs
std::deque< Reference< double >::Ptr > m_dRefs
Definition SparseFile.h:309
SparseFile::FileReferences::m_fRefs
std::deque< Reference< float >::Ptr > m_fRefs
Definition SparseFile.h:307
SparseFile::FileReferences::m_mutex
Mutex m_mutex
Definition SparseFile.h:316
SparseFile::FileReferences::m_vdRefs
std::deque< Reference< V3d >::Ptr > m_vdRefs
Definition SparseFile.h:310
SparseFile::FileReferences::numRefs
size_t numRefs() const
Returns the number of file references of the corresponding collection.
SparseFile::FileReferences::m_vfRefs
std::deque< Reference< V3f >::Ptr > m_vfRefs
Definition SparseFile.h:308
SparseFile::FileReferences::append
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
SparseFile::FileReferences::m_hRefs
std::deque< Reference< half >::Ptr > m_hRefs
Definition SparseFile.h:305
SparseFile::FileReferences::m_vhRefs
std::deque< Reference< V3h >::Ptr > m_vhRefs
Definition SparseFile.h:306
SparseFile::FileReferences::Mutex
boost::mutex Mutex
Definition SparseFile.h:314
SparseFile::Reference
Definition SparseFile.h:106
SparseFile::Reference::openFile
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
Definition SparseFile.cpp:537
SparseFile::Reference::layerPath
std::string layerPath
Definition SparseFile.h:128
SparseFile::Reference::blockUsed
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
Definition SparseFile.h:145
SparseFile::Reference::blocks
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please.
Definition SparseFile.h:141
SparseFile::Reference::fileBlockIndices
std::vector< int > fileBlockIndices
Index in file for each block.
Definition SparseFile.h:135
SparseFile::Reference::create
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
Definition SparseFile.h:660
SparseFile::Reference::valuesPerBlock
int valuesPerBlock
Definition SparseFile.h:129
SparseFile::Reference::filename
std::string filename
Definition SparseFile.h:127
SparseFile::Reference::incBlockRef
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
Definition SparseFile.h:745
SparseFile::Reference::m_ogRoot
OgIGroupPtr m_ogRoot
Ogawa archive root.
Definition SparseFile.h:249
SparseFile::Reference::BlockPtrs
Sparse::SparseBlock< Data_T > * BlockPtrs
Definition SparseFile.h:120
SparseFile::Reference::memSize
long long int memSize() const
Memory use for the Reference.
Definition SparseFile.h:863
SparseFile::Reference::Reference
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
Definition SparseFile.h:559
SparseFile::Reference::m_mutex
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Definition SparseFile.h:254
SparseFile::Reference::m_layerGroup
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
Definition SparseFile.h:235
SparseFile::Reference::m_ogArchive
IArchivePtr m_ogArchive
Ogawa archive.
Definition SparseFile.h:247
SparseFile::Reference::closeFile
void closeFile()
Closes the file. Will be re-opened as needed.
Definition SparseFile.h:710
SparseFile::Reference::m_ogLayerGroup
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
Definition SparseFile.h:251
SparseFile::Reference::Mutex
boost::mutex Mutex
Definition SparseFile.h:215
SparseFile::Reference::blockMutexSize
int blockMutexSize
Size of the mutex array. Used as modulus base.
Definition SparseFile.h:159
SparseFile::Reference::blockLoaded
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
Definition SparseFile.h:138
SparseFile::Reference::totalLoadedBlocks
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks),...
Definition SparseFile.h:808
SparseFile::Reference::numBlocks
int numBlocks
Definition SparseFile.h:131
SparseFile::Reference::averageLoads
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
Definition SparseFile.h:833
SparseFile::Reference::~Reference
~Reference()
Destructor.
Definition SparseFile.h:572
SparseFile::Reference::m_ogReader
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
Definition SparseFile.h:245
SparseFile::Reference::m_ogReaderPtr
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
Definition SparseFile.h:242
SparseFile::Reference::m_reader
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition SparseFile.h:239
SparseFile::Reference::numVoxels
int numVoxels
Definition SparseFile.h:130
SparseFile::Reference::blockSize
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
Definition SparseFile.h:771
SparseFile::Reference::refCounts
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Definition SparseFile.h:152
SparseFile::Reference::loadBlock
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Definition SparseFile.cpp:498
SparseFile::Reference::m_fileHandle
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
Definition SparseFile.h:231
SparseFile::Reference::totalLoads
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
Definition SparseFile.h:779
SparseFile::Reference::occupiedBlocks
int occupiedBlocks
Definition SparseFile.h:132
SparseFile::Reference::blockMutex
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
Definition SparseFile.h:156
SparseFile::Reference::decBlockRef
void decBlockRef(int blockIdx)
Decrement reference count on a block.
Definition SparseFile.h:758
SparseFile::Reference::m_numActiveBlocks
size_t m_numActiveBlocks
Number of currently active blocks.
Definition SparseFile.h:257
SparseFile::Reference::numLoadedBlocks
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
Definition SparseFile.h:793
SparseFile::Reference::operator=
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
Definition SparseFile.h:600
SparseFile::Reference::unloadBlock
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
Definition SparseFile.h:722
SparseFile::Reference::setNumBlocks
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
Definition SparseFile.h:677
SparseFile::Reference::loadCounts
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
Definition SparseFile.h:148
SparseFile::Reference::resetCacheStatistics
void resetCacheStatistics()
Resets counts of total block loads.
Definition SparseFile.h:851
SparseFile::Reference::Ptr
Reference * Ptr
Definition SparseFile.h:116
SparseFile::Reference::fileIsOpen
bool fileIsOpen()
Checks if the file used by this reference is open already.
Definition SparseFile.h:669
FIELD3D_NAMESPACE_OPEN
Definition FieldMapping.cpp:74
Msg::SevWarning
@ SevWarning
Definition Log.h:68
Msg::print
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Definition Log.cpp:70
SparseFile
Definition SparseFile.h:91
Sparse
Namespace for sparse field specifics.
Definition SparseField.h:221
ns.h
FIELD3D_API
#define FIELD3D_API
Definition ns.h:77
FIELD3D_NAMESPACE_HEADER_CLOSE
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition ns.h:58
DataTypeTraits::typeEnum
static DataTypeEnum typeEnum()
Sparse::SparseBlock
Storage for one individual block of a SparseField.
Definition SparseField.h:228