Field3D
SparseDataReader.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 #ifndef _INCLUDED_Field3D_SparseDataReader_H_
45 #define _INCLUDED_Field3D_SparseDataReader_H_
46 
47 //----------------------------------------------------------------------------//
48 
49 #include <hdf5.h>
50 #include <string.h> // for memcpy
51 
52 #include "Hdf5Util.h"
53 #include "Log.h"
54 
55 //----------------------------------------------------------------------------//
56 
57 #include "ns.h"
58 
60 
61 //----------------------------------------------------------------------------//
62 // SparseDataReader
63 //----------------------------------------------------------------------------//
64 
69 template <class Data_T>
71 {
72 public:
73 
74  // Constructors --------------------------------------------------------------
75 
78  SparseDataReader(hid_t location, int valuesPerBlock, int occupiedBlocks);
79 
80  // Main methods --------------------------------------------------------------
81 
84  void readBlock(int idx, Data_T &result);
85 
88  void readBlockList(int idx, const std::vector<Data_T*>& memoryList);
89 
90 private:
91 
92  // Data members --------------------------------------------------------------
93 
94  hid_t m_location;
95 
98 
99  const std::string k_dataStr;
100 };
101 
102 //----------------------------------------------------------------------------//
103 // SparseDataReader implementations
104 //----------------------------------------------------------------------------//
105 
106 template <class Data_T>
107 SparseDataReader<Data_T>::SparseDataReader(hid_t location, int valuesPerBlock,
108  int occupiedBlocks)
109  : m_location(location),
110  m_valuesPerBlock(valuesPerBlock),
111  m_occupiedBlocks(occupiedBlocks),
112  k_dataStr("data")
113 {
114 
115 }
116 
117 //----------------------------------------------------------------------------//
118 
119 template <class Data_T>
120 void SparseDataReader<Data_T>::readBlock(int idx, Data_T &result)
121 {
122  using namespace Hdf5Util;
123  using namespace Exc;
124 
125  GlobalLock lock(g_hdf5Mutex);
126 
127  Hdf5Util::H5ScopedDopen dataSet;
128  Hdf5Util::H5ScopedDget_space fileDataSpace;
130  Hdf5Util::H5ScopedScreate memDataSpace;
131 
132  hsize_t dims[2];
133  hsize_t memDims[1];
134 
135  // Open the data set
136  dataSet.open(m_location, k_dataStr, H5P_DEFAULT);
137  if (dataSet.id() < 0)
138  throw OpenDataSetException("Couldn't open data set: " + k_dataStr);
139 
140  // Get the space and type
141  fileDataSpace.open(dataSet.id());
142  dataType.open(dataSet.id());
143  if (fileDataSpace.id() < 0)
144  throw GetDataSpaceException("Couldn't get data space");
145  if (dataType.id() < 0)
146  throw GetDataTypeException("Couldn't get data type");
147 
148  // Make the memory data space
149  memDims[0] = m_valuesPerBlock;
150  memDataSpace.create(H5S_SIMPLE);
151  H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
152 
153  // Get the dimensions and check they match
154  H5Sget_simple_extent_dims(fileDataSpace.id(), dims, NULL);
155  if (dims[1] != static_cast<hsize_t>(m_valuesPerBlock)) {
156  throw FileIntegrityException("Block length mismatch in "
157  "SparseDataReader");
158  }
159  if (dims[0] != static_cast<hsize_t>(m_occupiedBlocks))
160  throw FileIntegrityException("Block count mismatch in "
161  "SparseDataReader");
162 
163  hsize_t offset[2];
164  hsize_t count[2];
165  herr_t status;
166 
167  offset[0] = idx; // Index of block
168  offset[1] = 0; // Index of first data in block. Always 0
169  count[0] = 1; // Number of columns to read. Always 1
170  count[1] = m_valuesPerBlock; // Number of values in one column
171 
172  status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
173  offset, NULL, count, NULL);
174 
175  if (status < 0) {
176  throw ReadHyperSlabException("Couldn't select slab in readBlock(): " +
177  boost::lexical_cast<std::string>(idx));
178  }
179 
180  status = H5Dread(dataSet.id(), DataTypeTraits<Data_T>::h5type(),
181  memDataSpace.id(), fileDataSpace.id(),
182  H5P_DEFAULT, &result);
183 }
184 
185 //----------------------------------------------------------------------------//
186 
187 template <class Data_T>
189 (int idxLo, const std::vector<Data_T*>& memoryList)
190 {
191  using namespace Hdf5Util;
192  using namespace Exc;
193 
194  GlobalLock lock(g_hdf5Mutex);
195 
196  Hdf5Util::H5ScopedDopen dataSet;
197  Hdf5Util::H5ScopedDget_space fileDataSpace;
199  Hdf5Util::H5ScopedScreate memDataSpace;
200 
201  hsize_t dims[2];
202  hsize_t memDims[1];
203 
204  // Open the data set
205  dataSet.open(m_location, k_dataStr, H5P_DEFAULT);
206  if (dataSet.id() < 0)
207  throw OpenDataSetException("Couldn't open data set: " + k_dataStr);
208 
209  // Get the space and type
210  fileDataSpace.open(dataSet.id());
211  dataType.open(dataSet.id());
212  if (fileDataSpace.id() < 0)
213  throw GetDataSpaceException("Couldn't get data space");
214  if (dataType.id() < 0)
215  throw GetDataTypeException("Couldn't get data type");
216 
217  // Make the memory data space
218  memDims[0] = m_valuesPerBlock;
219  memDataSpace.create(H5S_SIMPLE);
220  H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
221 
222  // Get the dimensions and check they match
223  H5Sget_simple_extent_dims(fileDataSpace.id(), dims, NULL);
224  if (dims[1] != static_cast<hsize_t>(m_valuesPerBlock)) {
225  throw FileIntegrityException("Block length mismatch in "
226  "SparseDataReader");
227  }
228  if (dims[0] != static_cast<hsize_t>(m_occupiedBlocks))
229  throw FileIntegrityException("Block count mismatch in "
230  "SparseDataReader");
231 
232  hsize_t offset[2];
233  hsize_t count[2];
234  herr_t status;
235 
236  offset[0] = idxLo; // Index of block
237  offset[1] = 0; // Index of first data in block. Always 0
238  count[0] = memoryList.size(); // Number of columns to read.
239  count[1] = m_valuesPerBlock; // Number of values in one column
240 
241  status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
242  offset, NULL, count, NULL);
243  if (status < 0) {
244  throw ReadHyperSlabException("Couldn't select slab in readBlockList():" +
245  boost::lexical_cast<std::string>(idxLo));
246  }
247 
248  // Make the memory data space ---
249 
250  Hdf5Util::H5ScopedScreate localMemDataSpace;
251  hsize_t fileDims[2];
252  fileDims[0] = memoryList.size();
253  fileDims[1] = m_valuesPerBlock;
254  localMemDataSpace.create(H5S_SIMPLE);
255  H5Sset_extent_simple(localMemDataSpace.id(), 2, fileDims, NULL);
256 
257  // Setup the temporary memory region ---
258 
259  int bytesPerValue = 0;
260  {
261  hid_t t = DataTypeTraits<Data_T>::h5type();
262  if (t == H5T_NATIVE_CHAR)
263  bytesPerValue = 1;
264  else if (t == H5T_NATIVE_SHORT)
265  bytesPerValue = 2;
266  else if (t == H5T_NATIVE_FLOAT)
267  bytesPerValue = 4;
268  else if (t == H5T_NATIVE_DOUBLE)
269  bytesPerValue = 8;
270  }
271 
272  int dim = sizeof(Data_T) / bytesPerValue;
273  std::vector<Data_T> bigblock(memoryList.size() * m_valuesPerBlock/dim);
274 
275  status = H5Dread(dataSet.id(),
277  localMemDataSpace.id(),
278  fileDataSpace.id(),
279  H5P_DEFAULT, &bigblock[0]);
280 
281  if (status < 0) {
282  throw Hdf5DataReadException("Couldn't read slab " +
283  boost::lexical_cast<std::string>(idxLo));
284  }
285 
286  // Distribute block data into memory slots ---
287  for (size_t i = 0; i < memoryList.size(); ++i) {
288  memcpy(memoryList[i],
289  &bigblock[i * m_valuesPerBlock / dim],
290  bytesPerValue * m_valuesPerBlock);
291  }
292 }
293 
294 //----------------------------------------------------------------------------//
295 
297 
298 //----------------------------------------------------------------------------//
299 
300 #endif
SparseDataReader::m_occupiedBlocks
int m_occupiedBlocks
Definition: SparseDataReader.h:97
FIELD3D_NAMESPACE_HEADER_CLOSE
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
SparseDataReader::k_dataStr
const std::string k_dataStr
Definition: SparseDataReader.h:99
DataTypeTraits::h5type
static hid_t h5type()
Hdf5Util::H5ScopedScreate::create
void create(H5S_class_t type)
Definition: Hdf5Util.h:246
Hdf5Util::H5Base::id
hid_t id() const
Query the hid_t value.
Definition: Hdf5Util.h:100
SparseDataReader::readBlockList
void readBlockList(int idx, const std::vector< Data_T * > &memoryList)
Reads a series of blocks, storing each block of data in memoryList, which is assumed to contain enoug...
Definition: SparseDataReader.h:189
SparseDataReader::SparseDataReader
SparseDataReader(hid_t location, int valuesPerBlock, int occupiedBlocks)
Constructor. Requires knowledge of the Hdf5 location where data is stored.
Definition: SparseDataReader.h:107
Log.h
Contains the Log class which can be used to redirect output to an arbitrary destination.
SparseDataReader::m_location
hid_t m_location
Definition: SparseDataReader.h:94
ns.h
Hdf5Util.h
Contains various utility functions for Hdf5.
Hdf5Util::H5ScopedDopen
Scoped object - opens a dataset on creation and closes it on destruction.
Definition: Hdf5Util.h:357
SparseDataReader
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
Definition: SparseDataReader.h:71
Hdf5Util::H5ScopedScreate
Scoped object - creates a dataspace on creation and closes it on destruction.
Definition: Hdf5Util.h:235
g_hdf5Mutex
FIELD3D_NAMESPACE_OPEN FIELD3D_API boost::recursive_mutex g_hdf5Mutex
Definition: Hdf5Util.cpp:67
Hdf5Util::H5ScopedDget_type::open
void open(hid_t dataset_id)
Definition: Hdf5Util.h:428
Exc
Namespace for Exception objects.
Definition: Exception.h:57
Hdf5Util::H5ScopedDget_space::open
void open(hid_t dataset_id)
Definition: Hdf5Util.h:398
GlobalLock
boost::recursive_mutex::scoped_lock GlobalLock
Definition: Hdf5Util.h:78
SparseDataReader::m_valuesPerBlock
int m_valuesPerBlock
Definition: SparseDataReader.h:96
Hdf5Util
Contains utility functions and classes for Hdf5 files.
Definition: Hdf5Util.h:86
DataTypeTraits
Definition: Traits.h:266
FIELD3D_NAMESPACE_OPEN
Definition: FieldMapping.cpp:74
Hdf5Util::H5ScopedDopen::open
void open(hid_t parentLocation, const std::string &name, hid_t dapl_id)
Definition: Hdf5Util.h:368
SparseDataReader::readBlock
void readBlock(int idx, Data_T &result)
Reads a block, storing the data in result, which is assumed to contain enough room for m_valuesPerBlo...
Definition: SparseDataReader.h:120
Hdf5Util::H5ScopedDget_type
Scoped object - opens a dataset on creation and closes it on destruction.
Definition: Hdf5Util.h:417
Hdf5Util::H5ScopedDget_space
Scoped object - opens a dataset on creation and closes it on destruction.
Definition: Hdf5Util.h:387