44 #include <boost/intrusive_ptr.hpp>
46 #include <boost/thread/thread.hpp>
47 #include <boost/thread/mutex.hpp>
50 #include "SparseFieldIO.h"
55 using namespace boost;
77 template <
typename Data_T>
78 struct ReadThreadingState
80 ReadThreadingState(
const OgIGroup &i_location,
82 const size_t i_numVoxels,
83 const size_t i_numBlocks,
84 const size_t i_numOccupiedBlocks,
85 const bool i_isCompressed,
86 const std::vector<size_t> &i_blockIdxToDatasetIdx)
87 : location(i_location),
89 numVoxels(i_numVoxels),
90 numBlocks(i_numBlocks),
91 numOccupiedBlocks(i_numOccupiedBlocks),
92 isCompressed(i_isCompressed),
93 blockIdxToDatasetIdx(i_blockIdxToDatasetIdx),
97 const OgIGroup &location;
99 const size_t numVoxels;
100 const size_t numBlocks;
101 const size_t numOccupiedBlocks;
102 const bool isCompressed;
103 const std::vector<size_t> &blockIdxToDatasetIdx;
104 size_t nextBlockToRead;
106 boost::mutex readMutex;
111 template <
typename Data_T>
115 ReadBlockOp(ReadThreadingState<Data_T> &state,
const size_t threadId)
119 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
120 const uLong cmpLenBound = compressBound(srcLen);
121 m_cache.resize(cmpLenBound);
125 m_state.numOccupiedBlocks,
126 m_state.isCompressed));
127 m_reader = m_readerPtr.get();
129 m_reader->setThreadId(threadId);
136 boost::mutex::scoped_lock lock(m_state.readMutex);
137 blockIdx = m_state.nextBlockToRead;
138 m_state.nextBlockToRead++;
141 while (blockIdx < m_state.numBlocks) {
142 if (m_state.blocks[blockIdx].isAllocated) {
143 const size_t datasetIdx = m_state.blockIdxToDatasetIdx[blockIdx];
144 m_reader->readBlock(datasetIdx, m_state.blocks[blockIdx].data);
148 boost::mutex::scoped_lock lock(m_state.readMutex);
149 blockIdx = m_state.nextBlockToRead;
150 m_state.nextBlockToRead++;
156 ReadThreadingState<Data_T> &m_state;
157 std::vector<uint8_t> m_cache;
158 boost::shared_ptr<OgSparseDataReader<Data_T> > m_readerPtr;
164 template <
typename Data_T>
165 struct ThreadingState
167 ThreadingState(OgOCDataset<Data_T> &i_data,
169 const size_t i_numVoxels,
170 const size_t i_numBlocks,
171 const std::vector<uint8_t> &i_isAllocated)
174 numVoxels(i_numVoxels),
175 numBlocks(i_numBlocks),
176 isAllocated(i_isAllocated),
177 nextBlockToCompress(0),
181 for (
size_t i = 0; i < numBlocks; ++i) {
182 if (blocks[i].isAllocated) {
183 nextBlockToCompress = i;
184 nextBlockToWrite = i;
189 nextBlockToCompress = numBlocks;
190 nextBlockToWrite = numBlocks;
193 OgOCDataset<Data_T> &data;
195 const size_t numVoxels;
196 const size_t numBlocks;
197 const std::vector<uint8_t> isAllocated;
198 size_t nextBlockToCompress;
199 size_t nextBlockToWrite;
201 boost::mutex compressMutex;
206 template <
typename Data_T>
210 WriteBlockOp(ThreadingState<Data_T> &state,
const size_t threadId)
211 : m_state(state), m_threadId(threadId)
213 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
214 const uLong cmpLenBound = compressBound(srcLen);
215 m_cache.resize(cmpLenBound);
223 boost::mutex::scoped_lock lock(m_state.compressMutex);
224 blockIdx = m_state.nextBlockToCompress++;
226 while (m_state.nextBlockToCompress < m_state.numBlocks) {
227 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
230 m_state.nextBlockToCompress++;
234 while (blockIdx < m_state.numBlocks) {
235 if (m_state.blocks[blockIdx].isAllocated) {
237 const uint8_t *srcData =
238 reinterpret_cast<const uint8_t *
>(m_state.blocks[blockIdx].data);
240 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
241 const uLong cmpLenBound = compressBound(srcLen);
242 uLong cmpLen = cmpLenBound;
244 const int status = compress2(&m_cache[0], &cmpLen,
245 srcData, srcLen, level);
247 if (status != Z_OK) {
248 std::cout <<
"ERROR: Couldn't compress in SparseFieldIO." << std::endl
249 <<
" Level: " << level << std::endl
250 <<
" Status: " << status << std::endl
251 <<
" srcLen: " << srcLen << std::endl
252 <<
" cmpLenBound: " << cmpLenBound << std::endl
253 <<
" cmpLen: " << cmpLen << std::endl;
257 while (m_state.nextBlockToWrite != blockIdx) {
259 boost::this_thread::sleep(boost::posix_time::microseconds(1));
262 m_state.data.addData(cmpLen, &m_cache[0]);
264 m_state.nextBlockToWrite++;
265 while (m_state.nextBlockToWrite < m_state.numBlocks){
267 if (m_state.blocks[m_state.nextBlockToWrite].isAllocated) {
270 m_state.nextBlockToWrite++;
275 boost::mutex::scoped_lock lock(m_state.compressMutex);
276 blockIdx = m_state.nextBlockToCompress++;
278 while (m_state.nextBlockToCompress < m_state.numBlocks) {
279 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
282 m_state.nextBlockToCompress++;
289 ThreadingState<Data_T> &m_state;
290 std::vector<uint8_t> m_cache;
291 const size_t m_threadId;
302 const int SparseFieldIO::k_versionNumber(1);
303 const std::string SparseFieldIO::k_versionAttrName(
"version");
304 const std::string SparseFieldIO::k_extentsStr(
"extents");
305 const std::string SparseFieldIO::k_extentsMinStr(
"extents_min");
306 const std::string SparseFieldIO::k_extentsMaxStr(
"extents_max");
307 const std::string SparseFieldIO::k_dataWindowStr(
"data_window");
308 const std::string SparseFieldIO::k_dataWindowMinStr(
"data_window_min");
309 const std::string SparseFieldIO::k_dataWindowMaxStr(
"data_window_max");
310 const std::string SparseFieldIO::k_componentsStr(
"components");
311 const std::string SparseFieldIO::k_dataStr(
"data");
312 const std::string SparseFieldIO::k_blockOrderStr(
"block_order");
313 const std::string SparseFieldIO::k_numBlocksStr(
"num_blocks");
314 const std::string SparseFieldIO::k_blockResStr(
"block_res");
315 const std::string SparseFieldIO::k_bitsPerComponentStr(
"bits_per_component");
316 const std::string SparseFieldIO::k_numOccupiedBlocksStr(
"num_occupied_blocks");
317 const std::string SparseFieldIO::k_isCompressed(
"data_is_compressed");
322 SparseFieldIO::read(hid_t layerGroup,
const std::string &filename,
323 const std::string &layerPath,
326 Box3i extents, dataW;
332 if (layerGroup == -1) {
338 if (!
readAttribute(layerGroup, k_versionAttrName, 1, version))
339 throw MissingAttributeException(
"Couldn't find attribute: " +
342 if (version != k_versionNumber)
343 throw UnsupportedVersionException(
"SparseField version not supported: " +
344 lexical_cast<std::string>(version));
346 if (!
readAttribute(layerGroup, k_extentsStr, 6, extents.min.x))
347 throw MissingAttributeException(
"Couldn't find attribute: " +
350 if (!
readAttribute(layerGroup, k_dataWindowStr, 6, dataW.min.x))
351 throw MissingAttributeException(
"Couldn't find attribute: " +
354 if (!
readAttribute(layerGroup, k_componentsStr, 1, components))
355 throw MissingAttributeException(
"Couldn't find attribute: " +
359 if (!
readAttribute(layerGroup, k_blockOrderStr, 1, blockOrder))
360 throw MissingAttributeException(
"Couldn't find attribute: " +
364 if (!
readAttribute(layerGroup, k_numBlocksStr, 1, numBlocks))
365 throw MissingAttributeException(
"Couldn't find attribute: " +
369 if (!
readAttribute(layerGroup, k_blockResStr, 3, blockRes.x))
370 throw MissingAttributeException(
"Couldn't find attribute: " +
375 int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z;
376 if (numCalculatedBlocks != numBlocks)
377 throw FileIntegrityException(
"Incorrect block count in SparseFieldIO::read");
384 if (!
readAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks))
385 throw MissingAttributeException(
"Couldn't find attribute: " +
386 k_numOccupiedBlocksStr);
391 if (!
readAttribute(layerGroup, k_bitsPerComponentStr, 1, bits))
392 throw MissingAttributeException(
"Couldn't find attribute: " +
393 k_bitsPerComponentStr);
396 bool isFloat =
false;
397 bool isDouble =
false;
413 if (components == 1) {
416 field->setSize(extents, dataW);
417 field->setBlockOrder(blockOrder);
418 readData<half>(layerGroup, numBlocks, filename, layerPath, field);
422 field->setSize(extents, dataW);
423 field->setBlockOrder(blockOrder);
424 readData<float>(layerGroup, numBlocks, filename, layerPath, field);
428 field->setSize(extents, dataW);
429 field->setBlockOrder(blockOrder);
430 readData<double>(layerGroup, numBlocks, filename, layerPath, field);
433 }
else if (components == 3) {
436 field->setSize(extents, dataW);
437 field->setBlockOrder(blockOrder);
438 readData<V3h>(layerGroup, numBlocks, filename, layerPath, field);
442 field->setSize(extents, dataW);
443 field->setBlockOrder(blockOrder);
444 readData<V3f>(layerGroup, numBlocks, filename, layerPath, field);
448 field->setSize(extents, dataW);
449 field->setBlockOrder(blockOrder);
450 readData<V3d>(layerGroup, numBlocks, filename, layerPath, field);
461 SparseFieldIO::read(
const OgIGroup &layerGroup,
const std::string &filename,
462 const std::string &layerPath,
OgDataType typeEnum)
464 Box3i extents, dataW;
469 if (!layerGroup.isValid()) {
470 throw MissingGroupException(
"Invalid group in SparseFieldIO::read()");
476 layerGroup.findAttribute<
int>(k_versionAttrName);
477 if (!versionAttr.isValid()) {
478 throw MissingAttributeException(
"Couldn't find attribute: " +
481 const int version = versionAttr.value();
483 if (version != k_versionNumber) {
484 throw UnsupportedVersionException(
"SparseField version not supported: " +
485 lexical_cast<std::string>(version));
491 layerGroup.findAttribute<
veci32_t>(k_extentsMinStr);
493 layerGroup.findAttribute<
veci32_t>(k_extentsMaxStr);
494 if (!extMinAttr.isValid()) {
495 throw MissingAttributeException(
"Couldn't find attribute " +
498 if (!extMaxAttr.isValid()) {
499 throw MissingAttributeException(
"Couldn't find attribute " +
503 extents.min = extMinAttr.value();
504 extents.max = extMaxAttr.value();
509 layerGroup.findAttribute<
veci32_t>(k_dataWindowMinStr);
511 layerGroup.findAttribute<
veci32_t>(k_dataWindowMaxStr);
512 if (!dwMinAttr.isValid()) {
513 throw MissingAttributeException(
"Couldn't find attribute " +
516 if (!dwMaxAttr.isValid()) {
517 throw MissingAttributeException(
"Couldn't find attribute " +
521 dataW.min = dwMinAttr.value();
522 dataW.max = dwMaxAttr.value();
527 layerGroup.findAttribute<uint8_t>(k_componentsStr);
528 if (!numComponentsAttr.isValid()) {
529 throw MissingAttributeException(
"Couldn't find attribute " +
536 layerGroup.findAttribute<uint8_t>(k_blockOrderStr);
537 if (!blockOrderAttr.isValid()) {
538 throw MissingAttributeException(
"Couldn't find attribute: " +
541 blockOrder = blockOrderAttr.value();
546 layerGroup.findAttribute<uint32_t>(k_numBlocksStr);
547 if (!numBlocksAttr.isValid()) {
548 throw MissingAttributeException(
"Couldn't find attribute: " +
551 numBlocks = numBlocksAttr.value();
556 layerGroup.findAttribute<
veci32_t>(k_blockResStr);
557 if (!blockResAttr.isValid()) {
558 throw MissingAttributeException(
"Couldn't find attribute: " +
561 blockRes = blockResAttr.value();
565 int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z;
566 if (numCalculatedBlocks != numBlocks) {
567 throw FileIntegrityException(
"Incorrect block count in "
568 "SparseFieldIO::read()");
574 layerGroup.findAttribute<uint32_t>(k_numOccupiedBlocksStr);
575 if (!occupiedBlocksAttr.isValid()) {
576 throw MissingAttributeException(
"Couldn't find attribute: " +
577 k_numOccupiedBlocksStr);
583 layerGroup.findAttribute<uint8_t>(k_isCompressed);
584 if (!isCompressedAttr.isValid()) {
585 throw MissingAttributeException(
"Couldn't find attribute: " +
595 if (isCompressedAttr.value() == 0) {
596 typeOnDisk = layerGroup.datasetType(k_dataStr);
598 typeOnDisk = layerGroup.compressedDatasetType(k_dataStr);
601 if (typeEnum == typeOnDisk) {
603 result = readData<float16_t>(layerGroup, extents, dataW, blockOrder,
604 numBlocks, filename, layerPath);
606 result = readData<float32_t>(layerGroup, extents, dataW, blockOrder,
607 numBlocks, filename, layerPath);
609 result = readData<float64_t>(layerGroup, extents, dataW, blockOrder,
610 numBlocks, filename, layerPath);
612 result = readData<vec16_t>(layerGroup, extents, dataW, blockOrder,
613 numBlocks, filename, layerPath);
615 result = readData<vec32_t>(layerGroup, extents, dataW, blockOrder,
616 numBlocks, filename, layerPath);
618 result = readData<vec64_t>(layerGroup, extents, dataW, blockOrder,
619 numBlocks, filename, layerPath);
631 if (layerGroup == -1) {
638 1, k_versionNumber)) {
644 field_dynamic_cast<SparseField<half> >(field);
646 field_dynamic_cast<SparseField<float> >(field);
648 field_dynamic_cast<SparseField<double> >(field);
650 field_dynamic_cast<SparseField<V3h> >(field);
652 field_dynamic_cast<SparseField<V3f> >(field);
654 field_dynamic_cast<SparseField<V3d> >(field);
658 success = writeInternal<half>(layerGroup, halfField);
659 }
else if (floatField) {
660 success = writeInternal<float>(layerGroup, floatField);
661 }
else if (doubleField) {
662 success = writeInternal<double>(layerGroup, doubleField);
663 }
else if (vecHalfField) {
664 success = writeInternal<V3h>(layerGroup, vecHalfField);
665 }
else if (vecFloatField) {
666 success = writeInternal<V3f>(layerGroup, vecFloatField);
667 }
else if (vecDoubleField) {
668 success = writeInternal<V3d>(layerGroup, vecDoubleField);
670 throw WriteLayerException(
"SparseFieldIO::write does not support the given "
671 "SparseField template parameter");
688 field_dynamic_cast<SparseField<half> >(field);
690 field_dynamic_cast<SparseField<float> >(field);
692 field_dynamic_cast<SparseField<double> >(field);
694 field_dynamic_cast<SparseField<V3h> >(field);
696 field_dynamic_cast<SparseField<V3f> >(field);
698 field_dynamic_cast<SparseField<V3d> >(field);
703 success = writeInternal<float>(layerGroup, floatField);
705 else if (halfField) {
706 success = writeInternal<half>(layerGroup, halfField);
708 else if (doubleField) {
709 success = writeInternal<double>(layerGroup, doubleField);
711 else if (vecFloatField) {
712 success = writeInternal<V3f>(layerGroup, vecFloatField);
714 else if (vecHalfField) {
715 success = writeInternal<V3h>(layerGroup, vecHalfField);
717 else if (vecDoubleField) {
718 success = writeInternal<V3d>(layerGroup, vecDoubleField);
721 throw WriteLayerException(
"SparseFieldIO does not support the given "
722 "SparseField template parameter");
730 template <
class Data_T>
732 SparseFieldIO::readData(
const OgIGroup &location,
const Box3i &extents,
733 const Box3i &dataW,
const size_t blockOrder,
734 const size_t numBlocks,
const std::string &filename,
735 const std::string &layerPath)
742 result->setSize(extents, dataW);
743 result->setBlockOrder(blockOrder);
747 const size_t numVoxels = (1 << (result->m_blockOrder * 3));
748 const int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
753 location.findAttribute<uint32_t>(k_numOccupiedBlocksStr);
754 if (!occupiedBlocksAttr.isValid()) {
755 throw MissingAttributeException(
"Couldn't find attribute: " +
756 k_numOccupiedBlocksStr);
758 const size_t occupiedBlocks = occupiedBlocksAttr.value();
762 if (dynamicLoading) {
765 result->addReference(filename, layerPath, valuesPerBlock, numVoxels,
774 std::vector<size_t> blockIdxToDatasetIdx(numBlocks);
778 vector<uint8_t> isAllocated(numBlocks);
780 location.findDataset<uint8_t>(
"block_is_allocated_data");
781 if (!isAllocatedData.isValid()) {
782 throw MissingGroupException(
"Couldn't find block_is_allocated_data: ");
784 isAllocatedData.getData(0, &isAllocated[0], OGAWA_THREAD);
786 for (
size_t i = 0, nextBlockOnDisk = 0; i < numBlocks; ++i) {
788 if (!dynamicLoading && isAllocated[i]) {
789 blocks[i].
resize(numVoxels);
791 blockIdxToDatasetIdx[i] = nextBlockOnDisk;
801 vector<Data_T> emptyValue(numBlocks);
803 location.findDataset<Data_T>(
"block_empty_value_data");
804 if (!emptyValueData.isValid()) {
805 throw MissingGroupException(
"Couldn't find block_empty_value_data: ");
807 emptyValueData.getData(0, &emptyValue[0], OGAWA_THREAD);
809 for (
size_t i = 0; i < numBlocks; ++i) {
818 location.findAttribute<uint8_t>(k_isCompressed);
819 const bool isCompressed = isCompressedAttr.value() != 0;
821 if (occupiedBlocks > 0) {
822 if (dynamicLoading) {
824 result->setupReferenceBlocks();
827 ReadThreadingState<Data_T> state(location, blocks, numVoxels, numBlocks,
828 occupiedBlocks, isCompressed,
829 blockIdxToDatasetIdx);
833 boost::thread_group threads;
834 for (
size_t i = 0; i < numThreads; ++i) {
835 threads.create_thread(ReadBlockOp<Data_T>(state, i));
849 template <
class Data_T>
850 bool SparseFieldIO::writeInternal(hid_t layerGroup,
862 int valuesPerBlock = (1 << (field->
m_blockOrder * 3)) * components;
867 { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
877 { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
879 if (!
writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) {
886 if (!
writeAttribute(layerGroup, k_componentsStr, 1, components)) {
895 if (!
writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) {
903 int numBlocks = blockRes.x * blockRes.y * blockRes.z;
920 if (!
writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) {
931 vector<char> isAllocated(numBlocks);
932 for (
int i = 0; i < numBlocks; ++i) {
933 isAllocated[i] =
static_cast<char>(blocks[i].
isAllocated);
935 writeSimpleData<char>(layerGroup,
"block_is_allocated_data", isAllocated);
940 vector<Data_T> emptyValue(numBlocks);
941 for (
int i = 0; i < numBlocks; ++i) {
942 emptyValue[i] =
static_cast<Data_T
>(blocks[i].
emptyValue);
944 writeSimpleData<Data_T>(layerGroup,
"block_empty_value_data", emptyValue);
948 int occupiedBlocks = 0;
949 for (
int i = 0; i < numBlocks; ++i) {
950 if (blocks[i].isAllocated) {
955 if (!
writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) {
956 throw WriteAttributeException(
"Couldn't add attribute " +
957 k_numOccupiedBlocksStr);
960 if (occupiedBlocks > 0) {
964 memDims[0] = valuesPerBlock;
966 H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
970 fileDims[0] = occupiedBlocks;
971 fileDims[1] = valuesPerBlock;
973 H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL);
977 hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
978 hsize_t chunkSize[2];
980 chunkSize[1] = valuesPerBlock;
982 herr_t status = H5Pset_deflate(dcpl, 9);
986 status = H5Pset_chunk(dcpl, 2, chunkSize);
996 H5P_DEFAULT, dcpl, H5P_DEFAULT);
997 if (dataSet.id() < 0)
998 throw CreateDataSetException(
"Couldn't create data set in "
999 "SparseFieldIO::writeInternal");
1003 int nextBlockIdx = 0;
1008 for (
int i = 0; i < numBlocks; ++i) {
1009 if (blocks[i].isAllocated) {
1010 offset[0] = nextBlockIdx;
1013 count[1] = valuesPerBlock;
1014 status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
1015 offset, NULL, count, NULL);
1017 throw WriteHyperSlabException(
1018 "Couldn't select slab " +
1019 boost::lexical_cast<std::string>(nextBlockIdx));
1024 fileDataSpace.id(), H5P_DEFAULT, data);
1026 throw WriteHyperSlabException(
1027 "Couldn't write slab " +
1028 boost::lexical_cast<std::string>(nextBlockIdx));
1043 template <
class Data_T>
1044 bool SparseFieldIO::writeInternal(OgOGroup &layerGroup,
1047 using namespace Exc;
1055 const size_t numBlocks = blockRes.x * blockRes.y * blockRes.z;
1056 const size_t numVoxels = (1 << (field->
m_blockOrder * 3));
1082 std::vector<uint8_t> isAllocated(numBlocks);
1083 for (
size_t i = 0; i < numBlocks; ++i) {
1084 isAllocated[i] =
static_cast<uint8_t
>(blocks[i].
isAllocated);
1087 isAllocatedData.addData(numBlocks, &isAllocated[0]);
1090 std::vector<Data_T> emptyValue(numBlocks);
1091 for (
size_t i = 0; i < numBlocks; ++i) {
1092 emptyValue[i] =
static_cast<Data_T
>(blocks[i].
emptyValue);
1095 emptyValueData.addData(numBlocks, &emptyValue[0]);
1098 int occupiedBlocks = 0;
1099 for (
size_t i = 0; i < numBlocks; ++i) {
1100 if (blocks[i].isAllocated) {
1105 k_numOccupiedBlocksStr,
1112 OgOCDataset<Data_T> data(layerGroup, k_dataStr);
1114 if (occupiedBlocks > 0) {
1116 ThreadingState<Data_T> state(data, blocks, numVoxels, numBlocks,
1121 boost::thread_group threads;
1122 for (
size_t i = 0; i < numThreads; ++i) {
1123 threads.create_thread(WriteBlockOp<Data_T>(state, i));
1133 template <
class Data_T>
1134 bool SparseFieldIO::readData(hid_t location,
1136 const std::string &filename,
1137 const std::string &layerPath,
1140 using namespace std;
1141 using namespace Exc;
1151 int valuesPerBlock = numVoxels * components;
1155 if (!
readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks))
1156 throw MissingAttributeException(
"Couldn't find attribute: " +
1157 k_numOccupiedBlocksStr);
1161 if (dynamicLoading) {
1164 valuesPerBlock, numVoxels,
1175 vector<char> isAllocated(numBlocks);
1176 readSimpleData<char>(location,
"block_is_allocated_data", isAllocated);
1177 for (
int i = 0; i < numBlocks; ++i) {
1179 if (!dynamicLoading && isAllocated[i]) {
1180 blocks[i].
resize(numVoxels);
1188 vector<Data_T> emptyValue(numBlocks);
1189 readSimpleData<Data_T>(location,
"block_empty_value_data", emptyValue);
1190 for (
int i = 0; i < numBlocks; ++i) {
1197 if (occupiedBlocks > 0) {
1199 if (dynamicLoading) {
1205 size_t b = 0, bend = b + numBlocks;
1210 static const long maxMemPerPass = 50*1024*1024;
1212 for (
int nextBlockIdx = 0;;) {
1215 std::vector<Data_T*> memoryList;
1217 for (; b != bend && mem < maxMemPerPass; ++b) {
1218 if (blocks[b].isAllocated) {
1219 mem +=
sizeof(Data_T)*numVoxels;
1220 memoryList.push_back(blocks[b].data);
1225 if (!memoryList.size()) {
1229 reader.readBlockList(nextBlockIdx, memoryList);
1230 nextBlockIdx += memoryList.size();