Field3D
|
#include <SparseFieldIO.h>
Public Types | |
typedef SparseFieldIO | class_type |
typedef boost::intrusive_ptr < SparseFieldIO > | Ptr |
Public Member Functions | |
virtual std::string | className () const |
Returns the class name. | |
const char * | classType () const |
virtual FieldBase::Ptr | read (hid_t layerGroup, const std::string &filename, const std::string &layerPath, DataTypeEnum typeEnum) |
Reads the field at the given location and tries to create a SparseField object from it. | |
SparseFieldIO () | |
Ctor. | |
virtual bool | write (hid_t layerGroup, FieldBase::Ptr field) |
Writes the given field to disk. | |
virtual | ~SparseFieldIO () |
Dtor. | |
Static Public Member Functions | |
static FieldIO::Ptr | create () |
Public Attributes | |
DEFINE_FIELD_RTTI_CONCRETE_CLASS | |
Private Types | |
typedef FieldIO | base |
Convenience typedef for referring to base class. | |
Private Member Functions | |
template<class Data_T > | |
bool | readData (hid_t location, int numBlocks, const std::string &filename, const std::string &layerPath, typename SparseField< Data_T >::Ptr result) |
Reads the data that is dependent on the data type on disk. | |
template<class Data_T > | |
bool | writeInternal (hid_t layerGroup, typename SparseField< Data_T >::Ptr field) |
This call writes all the attributes and sets up the data space. | |
Static Private Attributes | |
static const std::string | k_bitsPerComponentStr |
static const std::string | k_blockOrderStr |
static const std::string | k_blockResStr |
static const std::string | k_componentsStr |
static const std::string | k_dataStr |
static const std::string | k_dataWindowStr |
static const std::string | k_extentsStr |
static const std::string | k_numBlocksStr |
static const std::string | k_numOccupiedBlocksStr |
static const std::string | k_versionAttrName |
static const int | k_versionNumber |
Defines the IO for a SparseField object
Definition at line 79 of file SparseFieldIO.h.
typedef boost::intrusive_ptr<SparseFieldIO> SparseFieldIO::Ptr |
Reimplemented from FieldIO.
Definition at line 86 of file SparseFieldIO.h.
Reimplemented from FieldIO.
Definition at line 90 of file SparseFieldIO.h.
typedef FieldIO SparseFieldIO::base [private] |
Convenience typedef for referring to base class.
Reimplemented from FieldIO.
Definition at line 163 of file SparseFieldIO.h.
SparseFieldIO::SparseFieldIO | ( | ) | [inline] |
virtual SparseFieldIO::~SparseFieldIO | ( | ) | [inline, virtual] |
const char* SparseFieldIO::classType | ( | ) | const [inline] |
Definition at line 93 of file SparseFieldIO.h.
{ return "SparseFieldIO"; }
bool Field3DOutputFile::create | ( | ) | [inline, static] |
Definition at line 110 of file SparseFieldIO.h.
Referenced by initIO().
{ return Ptr(new SparseFieldIO); }
FieldBase::Ptr SparseFieldIO::read | ( | hid_t | layerGroup, |
const std::string & | filename, | ||
const std::string & | layerPath, | ||
DataTypeEnum | typeEnum | ||
) | [virtual] |
Reads the field at the given location and tries to create a SparseField object from it.
Implements FieldIO.
Definition at line 84 of file SparseFieldIO.cpp.
References DataTypeDouble, DataTypeFloat, DataTypeHalf, DataTypeVecDouble, DataTypeVecFloat, DataTypeVecHalf, Msg::print(), Hdf5Util::readAttribute(), SparseField< Data_T >::setBlockOrder(), ResizableField< Data_T >::setSize(), and Msg::SevWarning.
{ Box3i extents, dataW; int components; int blockOrder; int numBlocks; V3i blockRes; if (layerGroup == -1) { Msg::print(Msg::SevWarning, "Bad layerGroup."); return FieldBase::Ptr(); } int version; if (!readAttribute(layerGroup, k_versionAttrName, 1, version)) throw MissingAttributeException("Couldn't find attribute: " + k_versionAttrName); if (version != k_versionNumber) throw UnsupportedVersionException("SparseField version not supported: " + lexical_cast<std::string>(version)); if (!readAttribute(layerGroup, k_extentsStr, 6, extents.min.x)) throw MissingAttributeException("Couldn't find attribute: " + k_extentsStr); if (!readAttribute(layerGroup, k_dataWindowStr, 6, dataW.min.x)) throw MissingAttributeException("Couldn't find attribute: " + k_dataWindowStr); if (!readAttribute(layerGroup, k_componentsStr, 1, components)) throw MissingAttributeException("Couldn't find attribute: " + k_componentsStr); // Read block order if (!readAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) throw MissingAttributeException("Couldn't find attribute: " + k_blockOrderStr); // Read number of blocks total if (!readAttribute(layerGroup, k_numBlocksStr, 1, numBlocks)) throw MissingAttributeException("Couldn't find attribute: " + k_numBlocksStr); // Read block resolution in each dimension if (!readAttribute(layerGroup, k_blockResStr, 3, blockRes.x)) throw MissingAttributeException("Couldn't find attribute: " + k_blockResStr); // ... Check that it matches the # reported by summing the active blocks int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z; if (numCalculatedBlocks != numBlocks) throw FileIntegrityException("Incorrect block count in SparseFieldIO::read"); // Call the appropriate read function based on the data type --- FieldBase::Ptr result; int occupiedBlocks; if (!readAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) throw MissingAttributeException("Couldn't find attribute: " + k_numOccupiedBlocksStr); // Check the data type --- int bits; if (!readAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) throw MissingAttributeException("Couldn't find attribute: " + k_bitsPerComponentStr); bool isHalf = false; bool isFloat = false; bool isDouble = false; switch (bits) { case 16: isHalf = true; break; case 64: isDouble = true; break; case 32: default: isFloat = true; } // Finally, read the data --- if (components == 1) { if (isHalf && typeEnum == DataTypeHalf) { SparseField<half>::Ptr field(new SparseField<half>); field->setSize(extents, dataW); field->setBlockOrder(blockOrder); readData<half>(layerGroup, numBlocks, filename, layerPath, field); result = field; } else if (isFloat && typeEnum == DataTypeFloat) { SparseField<float>::Ptr field(new SparseField<float>); field->setSize(extents, dataW); field->setBlockOrder(blockOrder); readData<float>(layerGroup, numBlocks, filename, layerPath, field); result = field; } else if (isDouble && typeEnum == DataTypeDouble) { SparseField<double>::Ptr field(new SparseField<double>); field->setSize(extents, dataW); field->setBlockOrder(blockOrder); readData<double>(layerGroup, numBlocks, filename, layerPath, field); result = field; } } else if (components == 3) { if (isHalf && typeEnum == DataTypeVecHalf) { SparseField<V3h>::Ptr field(new SparseField<V3h>); field->setSize(extents, dataW); field->setBlockOrder(blockOrder); readData<V3h>(layerGroup, numBlocks, filename, layerPath, field); result = field; } else if (isFloat && typeEnum == DataTypeVecFloat) { SparseField<V3f>::Ptr field(new SparseField<V3f>); field->setSize(extents, dataW); field->setBlockOrder(blockOrder); readData<V3f>(layerGroup, numBlocks, filename, layerPath, field); result = field; } else if (isDouble && typeEnum == DataTypeVecDouble) { SparseField<V3d>::Ptr field(new SparseField<V3d>); field->setSize(extents, dataW); field->setBlockOrder(blockOrder); readData<V3d>(layerGroup, numBlocks, filename, layerPath, field); result = field; } } return result; }
bool SparseFieldIO::write | ( | hid_t | layerGroup, |
FieldBase::Ptr | field | ||
) | [virtual] |
Writes the given field to disk.
Implements FieldIO.
Definition at line 223 of file SparseFieldIO.cpp.
References field_dynamic_cast(), Msg::print(), Msg::SevWarning, and Hdf5Util::writeAttribute().
{ if (layerGroup == -1) { Msg::print(Msg::SevWarning, "Bad layerGroup."); return false; } // Add version attribute if (!writeAttribute(layerGroup, k_versionAttrName, 1, k_versionNumber)) { Msg::print(Msg::SevWarning, "Error adding version attribute."); return false; } SparseField<half>::Ptr halfField = field_dynamic_cast<SparseField<half> >(field); SparseField<float>::Ptr floatField = field_dynamic_cast<SparseField<float> >(field); SparseField<double>::Ptr doubleField = field_dynamic_cast<SparseField<double> >(field); SparseField<V3h>::Ptr vecHalfField = field_dynamic_cast<SparseField<V3h> >(field); SparseField<V3f>::Ptr vecFloatField = field_dynamic_cast<SparseField<V3f> >(field); SparseField<V3d>::Ptr vecDoubleField = field_dynamic_cast<SparseField<V3d> >(field); bool success = true; if (halfField) { success = writeInternal<half>(layerGroup, halfField); } else if (floatField) { success = writeInternal<float>(layerGroup, floatField); } else if (doubleField) { success = writeInternal<double>(layerGroup, doubleField); } else if (vecHalfField) { success = writeInternal<V3h>(layerGroup, vecHalfField); } else if (vecFloatField) { success = writeInternal<V3f>(layerGroup, vecFloatField); } else if (vecDoubleField) { success = writeInternal<V3d>(layerGroup, vecDoubleField); } else { throw WriteLayerException("SparseFieldIO::write does not support the given " "SparseField template parameter"); } return success; }
virtual std::string SparseFieldIO::className | ( | ) | const [inline, virtual] |
Returns the class name.
Implements FieldIO.
Definition at line 127 of file SparseFieldIO.h.
{ return "SparseField"; }
bool SparseFieldIO::writeInternal | ( | hid_t | layerGroup, |
typename SparseField< Data_T >::Ptr | field | ||
) | [private] |
This call writes all the attributes and sets up the data space.
Definition at line 172 of file SparseFieldIO.h.
References Hdf5Util::checkHdf5Gzip(), FieldRes::dataWindow(), FieldRes::extents(), Hdf5Util::H5Base::id(), k_bitsPerComponentStr, k_blockOrderStr, k_blockResStr, k_componentsStr, k_dataStr, k_dataWindowStr, k_extentsStr, k_numBlocksStr, k_numOccupiedBlocksStr, SparseField< Data_T >::m_blockOrder, SparseField< Data_T >::m_blockRes, SparseField< Data_T >::m_blocks, Msg::print(), Msg::SevWarning, and Hdf5Util::writeAttribute().
{ using namespace std; using namespace Exc; using namespace Hdf5Util; using namespace Sparse; Box3i ext(field->extents()), dw(field->dataWindow()); int components = FieldTraits<Data_T>::dataDims(); int valuesPerBlock = (1 << (field->m_blockOrder * 3)) * components; int size[3]; size[0] = dw.max.x - dw.min.x + 1; size[1] = dw.max.y - dw.min.y + 1; size[2] = dw.max.z - dw.min.z + 1; hsize_t totalSize[1]; totalSize[0] = size[0] * size[1] * size[2] * components; // Add extents attribute --- int extents[6] = { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z }; if (!writeAttribute(layerGroup, k_extentsStr, 6, extents[0])) { Msg::print(Msg::SevWarning, "Error adding size attribute."); return false; } // Add data window attribute --- int dataWindow[6] = { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z }; if (!writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) { Msg::print(Msg::SevWarning, "Error adding size attribute."); return false; } // Add components attribute --- if (!writeAttribute(layerGroup, k_componentsStr, 1, components)) { Msg::print(Msg::SevWarning, "Error adding components attribute."); return false; } // Add block order attribute --- int blockOrder = field->m_blockOrder; if (!writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) { Msg::print(Msg::SevWarning, "Error adding block order attribute."); return false; } // Add number of blocks attribute --- V3i &blockRes = field->m_blockRes; int numBlocks = blockRes.x * blockRes.y * blockRes.z; if (!writeAttribute(layerGroup, k_numBlocksStr, 1, numBlocks)) { Msg::print(Msg::SevWarning, "Error adding number of blocks attribute."); return false; } // Add block resolution in each dimension --- if (!writeAttribute(layerGroup, k_blockResStr, 3, blockRes.x)) { Msg::print(Msg::SevWarning, "Error adding block res attribute."); return false; } // Add the bits per component attribute --- int bits = DataTypeTraits<Data_T>::h5bits(); if (!writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) { Msg::print(Msg::SevWarning, "Error adding bits per component attribute."); return false; } // Write the block info data sets --- // ... Write the isAllocated array { vector<char> isAllocated(numBlocks); vector<char>::iterator i = isAllocated.begin(); typename vector<SparseBlock<Data_T> >::const_iterator b = field->m_blocks.begin(); for (; i != isAllocated.end(); ++i, ++b) *i = static_cast<char>(b->isAllocated); writeSimpleData<char>(layerGroup, "block_is_allocated_data", isAllocated); } // ... Write the emptyValue array { vector<Data_T> emptyValue(numBlocks); typename vector<Data_T>::iterator i = emptyValue.begin(); typename vector<SparseBlock<Data_T> >::const_iterator b = field->m_blocks.begin(); for (; i != emptyValue.end(); ++i, ++b) *i = static_cast<Data_T>(b->emptyValue); writeSimpleData<Data_T>(layerGroup, "block_empty_value_data", emptyValue); } // Count the number of occupied blocks --- int occupiedBlocks = 0; typename vector<SparseBlock<Data_T> >::iterator b = field->m_blocks.begin(); for (; b != field->m_blocks.end(); ++b) { if (b->isAllocated) occupiedBlocks++; } if (!writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) { throw WriteAttributeException("Couldn't add attribute " + k_numOccupiedBlocksStr); } if (occupiedBlocks > 0) { // Make the memory data space hsize_t memDims[1]; memDims[0] = valuesPerBlock; H5ScopedScreate memDataSpace(H5S_SIMPLE); H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL); // Make the file data space hsize_t fileDims[2]; fileDims[0] = occupiedBlocks; fileDims[1] = valuesPerBlock; H5ScopedScreate fileDataSpace(H5S_SIMPLE); H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL); // Set up gzip property list bool gzipAvailable = checkHdf5Gzip(); hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); hsize_t chunkSize[2]; chunkSize[0] = 1; chunkSize[1] = valuesPerBlock; if (gzipAvailable) { herr_t status = H5Pset_deflate(dcpl, 9); if (status < 0) { return false; } status = H5Pset_chunk(dcpl, 2, chunkSize); if (status < 0) { return false; } } // Add the data set H5ScopedDcreate dataSet(layerGroup, k_dataStr, DataTypeTraits<Data_T>::h5type(), fileDataSpace.id(), H5P_DEFAULT, dcpl, H5P_DEFAULT); if (dataSet.id() < 0) throw CreateDataSetException("Couldn't create data set in " "SparseFieldIO::writeInternal"); // For each allocated block --- int nextBlockIdx = 0; hsize_t offset[2]; hsize_t count[2]; herr_t status; for (b = field->m_blocks.begin(); b != field->m_blocks.end(); ++b) { if (b->isAllocated) { offset[0] = nextBlockIdx; // Index of next block offset[1] = 0; // Index of first data in block. Always 0 count[0] = 1; // Number of columns to read. Always 1 count[1] = valuesPerBlock; // Number of values in one column status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET, offset, NULL, count, NULL); if (status < 0) { throw WriteHyperSlabException( "Couldn't select slab " + boost::lexical_cast<std::string>(nextBlockIdx)); } Data_T *data = &b->data[0]; status = H5Dwrite(dataSet.id(), DataTypeTraits<Data_T>::h5type(), memDataSpace.id(), fileDataSpace.id(), H5P_DEFAULT, data); if (status < 0) { throw WriteHyperSlabException( "Couldn't write slab " + boost::lexical_cast<std::string>(nextBlockIdx)); } // Increment nextBlockIdx nextBlockIdx++; } } } // if occupiedBlocks > 0 return true; }
bool SparseFieldIO::readData | ( | hid_t | location, |
int | numBlocks, | ||
const std::string & | filename, | ||
const std::string & | layerPath, | ||
typename SparseField< Data_T >::Ptr | result | ||
) | [private] |
Reads the data that is dependent on the data type on disk.
Definition at line 377 of file SparseFieldIO.h.
References SparseField< Data_T >::addReference(), SparseFileManager::doLimitMemUse(), k_numOccupiedBlocksStr, SparseField< Data_T >::m_blockOrder, SparseField< Data_T >::m_blocks, Hdf5Util::readAttribute(), SparseDataReader< Data_T >::readBlockList(), SparseField< Data_T >::setupReferenceBlocks(), and SparseFileManager::singleton().
{ using namespace std; using namespace Exc; using namespace Hdf5Util; using namespace Sparse; int occupiedBlocks; bool dynamicLoading = SparseFileManager::singleton().doLimitMemUse(); int components = FieldTraits<Data_T>::dataDims(); int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components; // Read the number of occupied blocks --- if (!readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks)) throw MissingAttributeException("Couldn't find attribute: " + k_numOccupiedBlocksStr); // Set up the dynamic read info --- if (dynamicLoading) { // Set up the field reference result->addReference(filename, layerPath, valuesPerBlock, occupiedBlocks); } // Read the block info data sets --- // ... Read the isAllocated array { vector<char> isAllocated(numBlocks); vector<char>::iterator i = isAllocated.begin(); readSimpleData<char>(location, "block_is_allocated_data", isAllocated); typename vector<SparseBlock<Data_T> >::iterator b = result->m_blocks.begin(); typename vector<SparseBlock<Data_T> >::iterator bend = result->m_blocks.end(); // We're assuming there are as many blocks in isAllocated as in the field. for (; b != bend; ++b, ++i) { b->isAllocated = static_cast<bool>(*i); if (*i && !dynamicLoading) { b->data.resize(valuesPerBlock); } } } // ... Read the emptyValue array --- { vector<Data_T> emptyValue(numBlocks); readSimpleData<Data_T>(location, "block_empty_value_data", emptyValue); typename vector<SparseBlock<Data_T> >::iterator b = result->m_blocks.begin(); typename vector<SparseBlock<Data_T> >::iterator bend = result->m_blocks.end(); typename vector<Data_T>::iterator i = emptyValue.begin(); // We're assuming there are as many blocks in isAllocated as in the field. for (; b != bend; ++b, ++i) { b->emptyValue = *i; } } // Read the data --- if (occupiedBlocks > 0) { if (dynamicLoading) { result->setupReferenceBlocks(); } else { typename vector<SparseBlock<Data_T> >::iterator b = result->m_blocks.begin(); typename vector<SparseBlock<Data_T> >::iterator bend = result->m_blocks.end(); SparseDataReader<Data_T> reader(location, valuesPerBlock, occupiedBlocks); // We'll read at most 50meg at a time static const long maxMemPerPass = 50*1024*1024; for (int nextBlockIdx = 0;;) { long mem = 0; std::vector<Data_T*> memoryList; for (; b != bend && mem < maxMemPerPass; ++b) { if (b->isAllocated) { mem += sizeof(Data_T)*valuesPerBlock; memoryList.push_back(&b->data[0]); } } // all done. if (!memoryList.size()) { break; } reader.readBlockList(nextBlockIdx, memoryList); nextBlockIdx += memoryList.size(); } } } // if occupiedBlocks > 0 return true; }
Definition at line 91 of file SparseFieldIO.h.
const int SparseFieldIO::k_versionNumber [static, private] |
Definition at line 148 of file SparseFieldIO.h.
const std::string SparseFieldIO::k_versionAttrName [static, private] |
Definition at line 149 of file SparseFieldIO.h.
const std::string SparseFieldIO::k_extentsStr [static, private] |
Definition at line 150 of file SparseFieldIO.h.
Referenced by writeInternal().
const std::string SparseFieldIO::k_dataWindowStr [static, private] |
Definition at line 151 of file SparseFieldIO.h.
Referenced by writeInternal().
const std::string SparseFieldIO::k_componentsStr [static, private] |
Definition at line 152 of file SparseFieldIO.h.
Referenced by writeInternal().
const std::string SparseFieldIO::k_blockOrderStr [static, private] |
Definition at line 153 of file SparseFieldIO.h.
Referenced by writeInternal().
const std::string SparseFieldIO::k_numBlocksStr [static, private] |
Definition at line 154 of file SparseFieldIO.h.
Referenced by writeInternal().
const std::string SparseFieldIO::k_blockResStr [static, private] |
Definition at line 155 of file SparseFieldIO.h.
Referenced by writeInternal().
const std::string SparseFieldIO::k_bitsPerComponentStr [static, private] |
Definition at line 156 of file SparseFieldIO.h.
Referenced by writeInternal().
const std::string SparseFieldIO::k_numOccupiedBlocksStr [static, private] |
Definition at line 157 of file SparseFieldIO.h.
Referenced by readData(), and writeInternal().
const std::string SparseFieldIO::k_dataStr [static, private] |
Definition at line 158 of file SparseFieldIO.h.
Referenced by writeInternal().