Field3D
|
#include <SparseFile.h>
Public Types | |
typedef std::vector < Sparse::SparseBlock< Data_T > * > | BlockPtrs |
Public Member Functions | |
float | averageLoads () const |
Returns the average number of loads per accessed block in this file, for cache statistics. | |
int | blockSize (int blockIdx) const |
Returns the number of bytes used by the data in the block. | |
void | decBlockRef (int blockIdx) |
Decrement reference count on a block. | |
bool | fileIsOpen () |
Checks if the file used by this reference is open already. | |
void | incBlockRef (int blockIdx) |
Increment reference count on a block, indicates the block is currently in use, so prevents it from being unloaded. | |
void | loadBlock (int blockIdx) |
Loads the block with the given index into memory. We don't pass in a reference to where the data should go since this is already know in the blocks data member. | |
int | numLoadedBlocks () const |
Returns the total number of blocks that are currently loaded, for statistics. | |
void | openFile () |
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the original file open has closed the file and doesn't cause any Hdf5 hiccups. | |
Reference & | operator= (const Reference &o) |
Assignment operator. Clears ref counts and rebuilds mutex array. | |
Reference (const std::string filename, const std::string layerPath) | |
Constructor. Requires the filename and layer path of the field to be known. | |
Reference (const Reference &o) | |
Copy constructor. Clears ref counts and rebuilds mutex array. | |
void | resetCacheStatistics () |
Resets counts of total block loads. | |
void | setNumBlocks (int numBlocks) |
Sets the number of blocks used by the SparseField we're supporting. | |
int | totalLoadedBlocks () const |
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks), for statistics. | |
int | totalLoads () const |
Returns the total number of loads of the blocks of this file, for cache statistics. | |
void | unloadBlock (int blockIdx) |
Unloads the block with the given index from memory. | |
~Reference () | |
Public Attributes | |
std::vector< int > | blockLoaded |
Whether each block is loaded. We don't use bools since vector<bool> is weird. | |
boost::mutex * | blockMutex |
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-safe updates of the ref counts. | |
BlockPtrs | blocks |
Pointers to each block. This is so we can go in and manipulate them as we please. | |
std::vector< bool > | blockUsed |
Flags of whether the blocks have been accessed since they were last considered for deallocation by the Second-chance/Clock caching system. | |
std::vector< int > | fileBlockIndices |
Index in file for each block. | |
std::string | filename |
std::string | layerPath |
std::vector< int > | loadCounts |
Per-block counts of the number of times each block has been loaded, for cache statistics. | |
int | occupiedBlocks |
std::vector< int > | refCounts |
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zero, then the block shouldn't be unloaded. | |
int | valuesPerBlock |
Private Attributes | |
hid_t | m_fileHandle |
Holds the Hdf5 handle to the file. | |
Hdf5Util::H5ScopedGopen | m_layerGroup |
Hold the group containing the data open for the duration of the Reference's existence. | |
boost::mutex | m_mutex |
Mutex to prevent two threads from modifying conflicting data. | |
SparseDataReader< Data_T > * | m_reader |
Pointer to the reader object. NULL at construction time. Created in openFile(). |
Handles information about and block loading for a single SparseField as referenced on disk
Definition at line 95 of file SparseFile.h.
typedef std::vector<Sparse::SparseBlock<Data_T>*> SparseFile::Reference< Data_T >::BlockPtrs |
Definition at line 101 of file SparseFile.h.
SparseFile::Reference< Data_T >::Reference | ( | const std::string | filename, |
const std::string | layerPath | ||
) |
Constructor. Requires the filename and layer path of the field to be known.
Definition at line 475 of file SparseFile.h.
: filename(a_filename), layerPath(a_layerPath), valuesPerBlock(-1), occupiedBlocks(-1), blockMutex(NULL), m_fileHandle(-1), m_reader(NULL) { /* Empty */ }
SparseFile::Reference< Data_T >::~Reference | ( | ) |
Definition at line 486 of file SparseFile.h.
{ if (m_reader) delete m_reader; if (blockMutex) delete [] blockMutex; }
SparseFile::Reference< Data_T >::Reference | ( | const Reference< Data_T > & | o | ) |
Copy constructor. Clears ref counts and rebuilds mutex array.
Definition at line 498 of file SparseFile.h.
{ m_reader = NULL; blockMutex = NULL; *this = o; }
Reference< Data_T > & SparseFile::Reference< Data_T >::operator= | ( | const Reference< Data_T > & | o | ) |
Assignment operator. Clears ref counts and rebuilds mutex array.
Definition at line 509 of file SparseFile.h.
References SparseFile::Reference< Data_T >::blockLoaded, SparseFile::Reference< Data_T >::blocks, SparseFile::Reference< Data_T >::blockUsed, SparseFile::Reference< Data_T >::fileBlockIndices, SparseFile::Reference< Data_T >::filename, SparseFile::Reference< Data_T >::layerPath, SparseFile::Reference< Data_T >::loadCounts, SparseFile::Reference< Data_T >::m_fileHandle, SparseFile::Reference< Data_T >::occupiedBlocks, SparseFile::Reference< Data_T >::refCounts, and SparseFile::Reference< Data_T >::valuesPerBlock.
{ // Copy public member variables (where appropriate) filename = o.filename; layerPath = o.layerPath; valuesPerBlock = o.valuesPerBlock; occupiedBlocks = o.occupiedBlocks; fileBlockIndices = o.fileBlockIndices; blockLoaded = o.blockLoaded; blocks = o.blocks; blockUsed = o.blockUsed; loadCounts = o.loadCounts; refCounts = o.refCounts; if (blockMutex) delete[] blockMutex; blockMutex = new boost::mutex[blocks.size()]; // Copy private member variables (where appropriate) m_fileHandle = o.m_fileHandle; // Don't copy id, let hdf5 generate a new one. if (m_fileHandle >= 0) { m_layerGroup.open(m_fileHandle, layerPath.c_str()); } if (m_reader) delete m_reader; m_reader = NULL; return *this; }
bool SparseFile::Reference< Data_T >::fileIsOpen | ( | ) |
Checks if the file used by this reference is open already.
Definition at line 543 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock().
{ return m_fileHandle >= 0; }
void SparseFile::Reference< Data_T >::setNumBlocks | ( | int | numBlocks | ) |
Sets the number of blocks used by the SparseField we're supporting.
Definition at line 551 of file SparseFile.h.
{ boost::mutex::scoped_lock lock(m_mutex); fileBlockIndices.resize(numBlocks); blockLoaded.resize(numBlocks, 0); blocks.resize(numBlocks, 0); blockUsed.resize(numBlocks, false); loadCounts.resize(numBlocks, 0); refCounts.resize(numBlocks, 0); if (blockMutex) delete[] blockMutex; blockMutex = new boost::mutex[numBlocks]; }
void SparseFile::Reference< Data_T >::openFile | ( | ) |
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the original file open has closed the file and doesn't cause any Hdf5 hiccups.
Definition at line 569 of file SparseFile.h.
References Msg::print(), and Msg::SevWarning.
Referenced by SparseFileManager::activateBlock().
{ using namespace Exc; using namespace Hdf5Util; boost::mutex::scoped_lock lock_A(m_mutex); // check that the file wasn't already opened before obtaining the lock if (fileIsOpen()) { return; } m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); if (m_fileHandle < 0) throw NoSuchFileException(filename); m_layerGroup.open(m_fileHandle, layerPath.c_str()); if (m_layerGroup.id() < 0) { Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: " "Couldn't find layer group " + layerPath + " in .f3d file "); throw FileIntegrityException(filename); } m_reader = new SparseDataReader<Data_T>(m_layerGroup.id(), valuesPerBlock, occupiedBlocks); }
void SparseFile::Reference< Data_T >::loadBlock | ( | int | blockIdx | ) |
Loads the block with the given index into memory. We don't pass in a reference to where the data should go since this is already know in the blocks data member.
Definition at line 600 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock().
{ boost::mutex::scoped_lock lock(m_mutex); // Allocate the block blocks[blockIdx]->resize(valuesPerBlock); assert(blocks[blockIdx]->data.size() > 0); // Read the data assert(m_reader); m_reader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->dataRef()); // Mark block as loaded blockLoaded[blockIdx] = 1; }
void SparseFile::Reference< Data_T >::unloadBlock | ( | int | blockIdx | ) |
Unloads the block with the given index from memory.
Definition at line 617 of file SparseFile.h.
Referenced by SparseFileManager::deallocateBlock().
{ // Deallocate the block blocks[blockIdx]->clear(); // Mark block as unloaded blockLoaded[blockIdx] = 0; }
void SparseFile::Reference< Data_T >::incBlockRef | ( | int | blockIdx | ) |
Increment reference count on a block, indicates the block is currently in use, so prevents it from being unloaded.
Definition at line 629 of file SparseFile.h.
Referenced by SparseFileManager::incBlockRef().
{ boost::mutex::scoped_lock lock(blockMutex[blockIdx]); ++refCounts[blockIdx]; }
void SparseFile::Reference< Data_T >::decBlockRef | ( | int | blockIdx | ) |
Decrement reference count on a block.
Definition at line 638 of file SparseFile.h.
Referenced by SparseFileManager::decBlockRef().
{ boost::mutex::scoped_lock lock(blockMutex[blockIdx]); --refCounts[blockIdx]; }
int SparseFile::Reference< Data_T >::blockSize | ( | int | blockIdx | ) | const |
Returns the number of bytes used by the data in the block.
Definition at line 647 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock(), SparseFileManager::deallocateBlock(), and SparseFileManager::removeFieldFromCache().
{ return valuesPerBlock * sizeof(Data_T); }
int SparseFile::Reference< Data_T >::totalLoads | ( | ) | const |
Returns the total number of loads of the blocks of this file, for cache statistics.
Definition at line 655 of file SparseFile.h.
{ std::vector<int>::const_iterator i = loadCounts.begin(); std::vector<int>::const_iterator end = loadCounts.end(); int numLoads = 0; for (; i != end; ++i) numLoads += *i; return numLoads; }
int SparseFile::Reference< Data_T >::numLoadedBlocks | ( | ) | const |
Returns the total number of blocks that are currently loaded, for statistics.
Definition at line 669 of file SparseFile.h.
{ std::vector<int>::const_iterator i = blockLoaded.begin(); std::vector<int>::const_iterator end = blockLoaded.end(); int numBlocks = 0; for (; i != end; ++i) if (*i) numBlocks++; return numBlocks; }
int SparseFile::Reference< Data_T >::totalLoadedBlocks | ( | ) | const |
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks), for statistics.
Definition at line 684 of file SparseFile.h.
{ std::vector<int>::const_iterator i = loadCounts.begin(); std::vector<int>::const_iterator li = blockLoaded.begin(); std::vector<int>::const_iterator end = loadCounts.end(); int numBlocks = 0; if (blockLoaded.size() == 0) { for (; i != end; ++i) if (*i) numBlocks++; } else { assert(loadCounts.size() == blockLoaded.size()); for (; i != end; ++i, ++li) if (*i || *li) numBlocks++; } return numBlocks; }
float SparseFile::Reference< Data_T >::averageLoads | ( | ) | const |
Returns the average number of loads per accessed block in this file, for cache statistics.
Definition at line 709 of file SparseFile.h.
{ std::vector<int>::const_iterator i = loadCounts.begin(); std::vector<int>::const_iterator end = loadCounts.end(); int numLoads = 0, numBlocks = 0; for (; i != end; ++i) { if (*i) { numLoads += *i; numBlocks++; } } return (float)numLoads / std::max(1, numBlocks); }
void SparseFile::Reference< Data_T >::resetCacheStatistics | ( | ) |
Resets counts of total block loads.
Definition at line 727 of file SparseFile.h.
{ std::vector<int>::iterator li = loadCounts.begin(); std::vector<int>::iterator lend = loadCounts.end(); for (; li != lend; ++li) *li = 0; }
std::string SparseFile::Reference< Data_T >::filename |
Definition at line 105 of file SparseFile.h.
Referenced by SparseField< Data_T >::copySparseField(), and SparseFile::Reference< Data_T >::operator=().
std::string SparseFile::Reference< Data_T >::layerPath |
Definition at line 106 of file SparseFile.h.
Referenced by SparseField< Data_T >::copySparseField(), and SparseFile::Reference< Data_T >::operator=().
int SparseFile::Reference< Data_T >::valuesPerBlock |
Definition at line 107 of file SparseFile.h.
Referenced by SparseField< Data_T >::addReference(), SparseField< Data_T >::copySparseField(), and SparseFile::Reference< Data_T >::operator=().
int SparseFile::Reference< Data_T >::occupiedBlocks |
Definition at line 108 of file SparseFile.h.
Referenced by SparseField< Data_T >::copySparseField(), and SparseFile::Reference< Data_T >::operator=().
std::vector<int> SparseFile::Reference< Data_T >::fileBlockIndices |
Index in file for each block.
Definition at line 111 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock(), SparseFileManager::decBlockRef(), SparseFileManager::incBlockRef(), SparseFile::Reference< Data_T >::operator=(), SparseFileManager::removeFieldFromCache(), and SparseField< Data_T >::setupReferenceBlocks().
std::vector<int> SparseFile::Reference< Data_T >::blockLoaded |
Whether each block is loaded. We don't use bools since vector<bool> is weird.
Definition at line 114 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock(), SparseFile::Reference< Data_T >::operator=(), and SparseFileManager::removeFieldFromCache().
BlockPtrs SparseFile::Reference< Data_T >::blocks |
Pointers to each block. This is so we can go in and manipulate them as we please.
Definition at line 117 of file SparseFile.h.
Referenced by SparseFile::Reference< Data_T >::operator=(), SparseFileManager::removeFieldFromCache(), and SparseField< Data_T >::setupReferenceBlocks().
std::vector<bool> SparseFile::Reference< Data_T >::blockUsed |
Flags of whether the blocks have been accessed since they were last considered for deallocation by the Second-chance/Clock caching system.
Definition at line 121 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock(), SparseFileManager::deallocateBlock(), SparseFile::Reference< Data_T >::operator=(), and SparseFileManager::removeFieldFromCache().
std::vector<int> SparseFile::Reference< Data_T >::loadCounts |
Per-block counts of the number of times each block has been loaded, for cache statistics.
Definition at line 124 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock(), and SparseFile::Reference< Data_T >::operator=().
std::vector<int> SparseFile::Reference< Data_T >::refCounts |
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zero, then the block shouldn't be unloaded.
Definition at line 128 of file SparseFile.h.
Referenced by SparseFileManager::deallocateBlock(), and SparseFile::Reference< Data_T >::operator=().
boost::mutex* SparseFile::Reference< Data_T >::blockMutex |
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-safe updates of the ref counts.
Definition at line 132 of file SparseFile.h.
Referenced by SparseFileManager::activateBlock(), and SparseFileManager::deallocateBlock().
hid_t SparseFile::Reference< Data_T >::m_fileHandle [private] |
Holds the Hdf5 handle to the file.
Definition at line 187 of file SparseFile.h.
Referenced by SparseFile::Reference< Data_T >::operator=().
Hdf5Util::H5ScopedGopen SparseFile::Reference< Data_T >::m_layerGroup [private] |
Hold the group containing the data open for the duration of the Reference's existence.
Definition at line 191 of file SparseFile.h.
SparseDataReader<Data_T>* SparseFile::Reference< Data_T >::m_reader [private] |
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition at line 195 of file SparseFile.h.
boost::mutex SparseFile::Reference< Data_T >::m_mutex [private] |
Mutex to prevent two threads from modifying conflicting data.
Definition at line 198 of file SparseFile.h.