Field3D
SparseFile.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
42 //----------------------------------------------------------------------------//
43 
44 #ifndef _INCLUDED_Field3D_SparseFile_H_
45 #define _INCLUDED_Field3D_SparseFile_H_
46 
47 //----------------------------------------------------------------------------//
48 
49 #include <boost/scoped_ptr.hpp>
50 #include <deque>
51 #include <list>
52 #include <vector>
53 
54 #include <hdf5.h>
55 
56 #include "Exception.h"
57 #include "Hdf5Util.h"
58 #include "OgawaFwd.h"
59 #include "SparseDataReader.h"
60 #include "Traits.h"
61 
62 //----------------------------------------------------------------------------//
63 
64 #define F3D_SHORT_MUTEX_ARRAY 1
65 #define F3D_MUTEX_ARRAY_SIZE 1000
66 #define F3D_NO_BLOCKS_ARRAY 1
67 
68 #include "ns.h"
69 
71 
72 //----------------------------------------------------------------------------//
73 // Forward declarations
74 //----------------------------------------------------------------------------//
75 
76 namespace Sparse {
77 
78  template <typename Data_T>
79  struct SparseBlock;
80 
81 }
82 
83 template <typename Data_T>
84 class SparseField;
85 
86 template <typename Data_T>
88 
89 //----------------------------------------------------------------------------//
90 
91 namespace SparseFile {
92 
93 //----------------------------------------------------------------------------//
94 // Reference
95 //----------------------------------------------------------------------------//
96 
102 //----------------------------------------------------------------------------//
103 
104 template <class Data_T>
105 class Reference
106 {
107 public:
108 
109  // Typedefs ------------------------------------------------------------------
110 
111 #define USE_SHPTR 0
112 
113 #if USE_SHPTR
114  typedef boost::shared_ptr<Reference> Ptr;
115 #else
116  typedef Reference* Ptr;
117 #endif
118 
119 #if F3D_NO_BLOCKS_ARRAY
121 #else
122  typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
123 #endif
124 
125  // Public data members -------------------------------------------------------
126 
127  std::string filename;
128  std::string layerPath;
133 
135  std::vector<int> fileBlockIndices;
138  std::vector<int> blockLoaded;
145  std::vector<bool> blockUsed;
148  std::vector<int> loadCounts;
152  std::vector<int> refCounts;
156  boost::mutex *blockMutex;
157 #if F3D_SHORT_MUTEX_ARRAY
158  int blockMutexSize;
160 #endif
161 
162  // Ctors, dtor ---------------------------------------------------------------
163 
165  ~Reference();
166 
167  // Main methods --------------------------------------------------------------
168 
171  static Ptr create(const std::string a_filename,
172  const std::string a_layerPath);
174  bool fileIsOpen();
176  void setNumBlocks(int numBlocks);
180  void openFile();
182  void closeFile();
186  void loadBlock(int blockIdx);
188  void unloadBlock(int blockIdx);
191  void incBlockRef(int blockIdx);
193  void decBlockRef(int blockIdx);
195  int blockSize(int blockIdx) const;
198  int totalLoads() const;
201  int numLoadedBlocks() const;
204  int totalLoadedBlocks() const;
207  float averageLoads() const;
209  void resetCacheStatistics();
211  long long int memSize() const;
212 
213 private:
214 
215  typedef boost::mutex Mutex;
216 
217  // Private constructors ---
218 
220  Reference(const std::string filename, const std::string layerPath);
221 
223  Reference(const Reference &o);
224 
226  Reference & operator=(const Reference &o);
227 
228  // Data members ---
229 
232 
236 
240 
242  boost::shared_ptr<OgSparseDataReader<Data_T> > m_ogReaderPtr;
252 
254  mutable Mutex m_mutex;
255 
258 
259 };
260 
261 //----------------------------------------------------------------------------//
262 // FileReferences
263 //----------------------------------------------------------------------------//
264 
266 {
267 public:
268 
269  /* A note on thread safety
270 
271  FileReferences uses a std::deque to store the References, which means
272  that already-added references 'stay put' in memory, even as more elements
273  are added. For threading purposes, we only need to lock during append()
274  and numRefs(), but ref() is safe, since we never remove references.
275 
276  */
277 
278  // Ctors, dtor ---------------------------------------------------------------
279 
280  ~FileReferences();
281 
282  // Main methods --------------------------------------------------------------
283 
286  template <class Data_T>
287  Reference<Data_T>* ref(size_t idx);
288 
291  template <class Data_T>
293 
295  template <class Data_T>
296  size_t numRefs() const;
297 
299  long long int memSize() const;
300 
301 private:
302 
303  // Data members --------------------------------------------------------------
304 
305  std::deque<Reference<half>::Ptr> m_hRefs;
306  std::deque<Reference<V3h>::Ptr> m_vhRefs;
307  std::deque<Reference<float>::Ptr> m_fRefs;
308  std::deque<Reference<V3f>::Ptr> m_vfRefs;
309  std::deque<Reference<double>::Ptr> m_dRefs;
310  std::deque<Reference<V3d>::Ptr> m_vdRefs;
311 
312  // Mutexes -------------------------------------------------------------------
313 
314  typedef boost::mutex Mutex;
315 
316  mutable Mutex m_mutex;
317 
318 };
319 
320 //----------------------------------------------------------------------------//
321 
322 class CacheBlock {
323 public:
325  int refIdx;
326  int blockIdx;
327  CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn) :
328  blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn)
329  { }
330 };
331 
332 //----------------------------------------------------------------------------//
333 
334 } // namespace SparseFile
335 
336 //----------------------------------------------------------------------------//
337 // SparseFileManager
338 //----------------------------------------------------------------------------//
339 
395 //----------------------------------------------------------------------------//
396 
398 {
399 
400 public:
401 
402  template <class Data_T>
403  friend class SparseField;
404 
405  // typedefs ------------------------------------------------------------------
406 
407  typedef std::list<SparseFile::CacheBlock> CacheList;
408 
409  // Main methods --------------------------------------------------------------
410 
412  static SparseFileManager &singleton();
413 
416  void setLimitMemUse(bool enabled);
417 
420  bool doLimitMemUse() const;
421 
423  void setMaxMemUse(float maxMemUse);
424 
427  void flushCache();
428 
430  long long totalLoads();
431 
433  long long numLoadedBlocks();
434 
436  long long totalLoadedBlocks();
437 
440  float cacheFractionLoaded();
441 
443  float cacheLoadsPerBlock();
444 
447  float cacheEfficiency();
448 
450  void resetCacheStatistics();
451 
453  long long int memSize() const;
454 
455  //--------------------------------------------------------------------------//
456  // Utility functions
457 
462  template <class Data_T>
463  void incBlockRef(int fileId, int blockIdx);
464 
469  template <class Data_T>
470  void decBlockRef(int fileId, int blockIdx);
471 
475  template <class Data_T>
476  void activateBlock(int fileId, int blockIdx);
477 
478 protected:
479 
481  template <class Data_T>
482  SparseFile::Reference<Data_T>* reference(int index);
483 
486  template <class Data_T>
487  int getNextId(const std::string filename, const std::string layerPath);
488 
489  template <class Data_T>
490  void removeFieldFromCache(int refIdx);
491 
492 private:
493 
496 
498  static boost::scoped_ptr<SparseFileManager> ms_singleton;
499 
501  void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
502 
505  void deallocateBlocks(int64_t bytesNeeded);
506 
509  template <class Data_T>
510  int64_t deallocateBlock(const SparseFile::CacheBlock &cb);
511 
513  template <class Data_T>
514  void deallocateBlock(CacheList::iterator &it);
515 
517  float m_maxMemUse;
518 
521 
523  int64_t m_memUse;
524 
528 
532 
539 
542  CacheList::iterator m_nextBlock;
543 
546  mutable boost::mutex m_mutex;
547 
548 };
549 
550 //----------------------------------------------------------------------------//
551 // Reference implementations
552 //----------------------------------------------------------------------------//
553 
554 namespace SparseFile {
555 
556 //----------------------------------------------------------------------------//
557 
558 template <class Data_T>
559 Reference<Data_T>::Reference(const std::string a_filename,
560  const std::string a_layerPath)
561  : filename(a_filename), layerPath(a_layerPath),
562  valuesPerBlock(-1), numVoxels(-1), numBlocks(-1), occupiedBlocks(-1),
563  blockMutex(NULL), m_fileHandle(-1), m_reader(NULL), m_ogReader(NULL),
564  m_numActiveBlocks(0)
565 {
566  /* Empty */
567 }
568 
569 //----------------------------------------------------------------------------//
570 
571 template <class Data_T>
573 {
574  closeFile();
575 
576  if (m_reader) {
577  delete m_reader;
578  }
579 
580  if (blockMutex)
581  delete [] blockMutex;
582 }
583 
584 //----------------------------------------------------------------------------//
585 
586 template <class Data_T>
588 {
589  m_ogReaderPtr.reset();
590  m_ogReader = NULL;
591  m_reader = NULL;
592  blockMutex = NULL;
593  *this = o;
594 }
595 
596 //----------------------------------------------------------------------------//
597 
598 template <class Data_T>
601 {
602  if (this == &o) {
603  return *this;
604  }
605 
606  // Copy public member variables (where appropriate)
607  filename = o.filename;
608  layerPath = o.layerPath;
609  valuesPerBlock = o.valuesPerBlock;
610  numVoxels = o.numVoxels;
611  occupiedBlocks = o.occupiedBlocks;
612  fileBlockIndices = o.fileBlockIndices;
613  blockLoaded = o.blockLoaded;
614  blocks = o.blocks;
615  blockUsed = o.blockUsed;
616  loadCounts = o.loadCounts;
617  refCounts = o.refCounts;
618  if (blockMutex)
619  delete[] blockMutex;
620 #if F3D_SHORT_MUTEX_ARRAY
621 # if F3D_NO_BLOCKS_ARRAY
622  blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
623 # else
624  blockMutexSize = std::min(static_cast<int>(blocks.size()),
626 # endif
627  blockMutex = new boost::mutex[blockMutexSize];
628 #else
629  blockMutex = new boost::mutex[blocks.size()];
630 #endif
631 
632 #if 0
633  // MW: Should this be copying the file handle? Never seems to happen,
634  // but it also seems bad. Ifdef'ing out for now.
635  // Copy private member variables (where appropriate)
636  m_fileHandle = o.m_fileHandle;
637  // Don't copy id, let hdf5 generate a new one.
638  if (m_fileHandle >= 0) {
639  m_layerGroup.open(m_fileHandle, layerPath.c_str());
640  }
641 #else
642  m_fileHandle = -1;
643 #endif
644 
645  // Re-allocate reader
646  if (m_reader)
647  delete m_reader;
648  m_reader = NULL;
649 
650  m_ogReaderPtr.reset();
651  m_ogReader = NULL;
652 
653  return *this;
654 }
655 
656 //----------------------------------------------------------------------------//
657 
658 template <class Data_T>
659 typename Reference<Data_T>::Ptr
660 Reference<Data_T>::create(const std::string a_filename,
661  const std::string a_layerPath)
662 {
663  return Ptr(new Reference(a_filename, a_layerPath));
664 }
665 
666 //----------------------------------------------------------------------------//
667 
668 template <class Data_T>
670 {
671  return m_fileHandle >= 0;
672 }
673 
674 //----------------------------------------------------------------------------//
675 
676 template <class Data_T>
677 void Reference<Data_T>::setNumBlocks(int a_numBlocks)
678 {
679  boost::mutex::scoped_lock lock(m_mutex);
680 
681  // Store number of blocks in member variable
682  numBlocks = a_numBlocks;
683 
684  fileBlockIndices.resize(numBlocks);
685  blockLoaded.resize(numBlocks, 0);
686 #if !F3D_NO_BLOCKS_ARRAY
687  blocks.resize(numBlocks, 0);
688 #endif
689  blockUsed.resize(numBlocks, false);
690  loadCounts.resize(numBlocks, 0);
691  refCounts.resize(numBlocks, 0);
692  if (blockMutex)
693  delete[] blockMutex;
694 #if F3D_SHORT_MUTEX_ARRAY
695 # if F3D_NO_BLOCKS_ARRAY
696  blockMutexSize = std::min(numBlocks, F3D_MUTEX_ARRAY_SIZE);
697 # else
698  blockMutexSize = std::min(blocks.size(),
699  static_cast<size_t>(F3D_MUTEX_ARRAY_SIZE));
700 # endif
701  blockMutex = new boost::mutex[blockMutexSize];
702 #else
703  blockMutex = new boost::mutex[numBlocks];
704 #endif
705 }
706 
707 //----------------------------------------------------------------------------//
708 
709 template <class Data_T>
711 {
712  if (m_fileHandle >= 0) {
713  if (H5Fclose(m_fileHandle) < 0) {
714  Msg::print("In ~Reference(): Error closing file " + filename);
715  }
716  }
717 }
718 
719 //----------------------------------------------------------------------------//
720 
721 template <class Data_T>
723 {
724  // Deallocate the block
725 #if F3D_NO_BLOCKS_ARRAY
726  blocks[blockIdx].clear();
727 #else
728  blocks[blockIdx]->clear();
729 #endif
730  // Mark block as unloaded
731  blockLoaded[blockIdx] = 0;
732  // Track count
733  m_numActiveBlocks--;
734 #if 0
735  // If no active blocks, close the file. De-activate for now.
736  if (m_numActiveBlocks == 0) {
737  closeFile();
738  }
739 #endif
740 }
741 
742 //----------------------------------------------------------------------------//
743 
744 template <class Data_T>
746 {
747 #if F3D_SHORT_MUTEX_ARRAY
748  boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
749 #else
750  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
751 #endif
752  ++refCounts[blockIdx];
753 }
754 
755 //----------------------------------------------------------------------------//
756 
757 template <class Data_T>
759 {
760 #if F3D_SHORT_MUTEX_ARRAY
761  boost::mutex::scoped_lock lock(blockMutex[blockIdx % blockMutexSize]);
762 #else
763  boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
764 #endif
765  --refCounts[blockIdx];
766 }
767 
768 //----------------------------------------------------------------------------//
769 
770 template <class Data_T>
771 int Reference<Data_T>::blockSize(int /* blockIdx */) const
772 {
773  return numVoxels * sizeof(Data_T);
774 }
775 
776 //----------------------------------------------------------------------------//
777 
778 template <class Data_T>
780 {
781  std::vector<int>::const_iterator i = loadCounts.begin();
782  std::vector<int>::const_iterator end = loadCounts.end();
783  int numLoads = 0;
784  for (; i != end; ++i)
785  numLoads += *i;
786 
787  return numLoads;
788 }
789 
790 //----------------------------------------------------------------------------//
791 
792 template <class Data_T>
794 {
795  std::vector<int>::const_iterator i = blockLoaded.begin();
796  std::vector<int>::const_iterator end = blockLoaded.end();
797  int numBlockCounter = 0;
798  for (; i != end; ++i)
799  if (*i)
800  numBlockCounter++;
801 
802  return numBlockCounter;
803 }
804 
805 //----------------------------------------------------------------------------//
806 
807 template <class Data_T>
809 {
810  std::vector<int>::const_iterator i = loadCounts.begin();
811  std::vector<int>::const_iterator li = blockLoaded.begin();
812  std::vector<int>::const_iterator end = loadCounts.end();
813  int numBlockCounter = 0;
814 
815  if (blockLoaded.size() == 0) {
816  for (; i != end; ++i)
817  if (*i)
818  numBlockCounter++;
819  } else {
820  assert(loadCounts.size() == blockLoaded.size());
821 
822  for (; i != end; ++i, ++li)
823  if (*i || *li)
824  numBlockCounter++;
825  }
826 
827  return numBlockCounter;
828 }
829 
830 //----------------------------------------------------------------------------//
831 
832 template <class Data_T>
834 {
835  std::vector<int>::const_iterator i = loadCounts.begin();
836  std::vector<int>::const_iterator end = loadCounts.end();
837  int numLoads = 0, numBlockCounter = 0;
838  for (; i != end; ++i) {
839  if (*i) {
840  numLoads += *i;
841  numBlockCounter++;
842  }
843  }
844 
845  return (float)numLoads / std::max(1, numBlockCounter);
846 }
847 
848 //----------------------------------------------------------------------------//
849 
850 template <class Data_T>
852 {
853  std::vector<int>::iterator li = loadCounts.begin();
854  std::vector<int>::iterator lend = loadCounts.end();
855  for (; li != lend; ++li)
856  *li = 0;
857 }
858 
859 //----------------------------------------------------------------------------//
860 
861 template <class Data_T>
862 long long int
864 {
865  boost::mutex::scoped_lock lock(m_mutex);
866 
867  return sizeof(*this) +
868  fileBlockIndices.capacity() * sizeof(int) +
869  blockLoaded.capacity() * sizeof(int) +
871  blocks.capacity() * sizeof(Sparse::SparseBlock<Data_T>*) +
872 #endif
873  blockUsed.capacity() * sizeof(bool) +
874  loadCounts.capacity() * sizeof(int) +
875  refCounts.capacity() * sizeof(int) +
877  blockMutexSize * sizeof(boost::mutex) +
878 #else
879  numBlocks * sizeof(boost::mutex) +
880 #endif
881  sizeof(SparseDataReader<Data_T>);
882 }
883 
884 //----------------------------------------------------------------------------//
885 
886 } // namespace SparseFile
887 
888 //----------------------------------------------------------------------------//
889 // Specializations for FileReferences
890 //----------------------------------------------------------------------------//
891 
892 namespace SparseFile {
893 
894 //----------------------------------------------------------------------------//
895 
897 {
898 #if !USE_SHPTR
899  for (size_t i = 0, end = m_hRefs.size(); i < end; ++i) {
900  delete m_hRefs[i];
901  }
902  for (size_t i = 0, end = m_fRefs.size(); i < end; ++i) {
903  delete m_fRefs[i];
904  }
905  for (size_t i = 0, end = m_dRefs.size(); i < end; ++i) {
906  delete m_dRefs[i];
907  }
908  for (size_t i = 0, end = m_vhRefs.size(); i < end; ++i) {
909  delete m_vhRefs[i];
910  }
911  for (size_t i = 0, end = m_vfRefs.size(); i < end; ++i) {
912  delete m_vfRefs[i];
913  }
914  for (size_t i = 0, end = m_vdRefs.size(); i < end; ++i) {
915  delete m_vdRefs[i];
916  }
917 #endif
918 }
919 
920 //----------------------------------------------------------------------------//
921 
922 template <>
923 inline Reference<half>*
925 {
926 #if USE_SHPTR
927  return m_hRefs[idx].get();
928 #else
929  return m_hRefs[idx];
930 #endif
931 }
932 
933 //----------------------------------------------------------------------------//
934 
935 template <>
936 inline Reference<V3h>*
937 FileReferences::ref(size_t idx)
938 {
939 #if USE_SHPTR
940  return m_vhRefs[idx].get();
941 #else
942  return m_vhRefs[idx];
943 #endif
944 }
945 
946 //----------------------------------------------------------------------------//
947 
948 template <>
949 inline Reference<float>*
950 FileReferences::ref(size_t idx)
951 {
952 #if USE_SHPTR
953  return m_fRefs[idx].get();
954 #else
955  return m_fRefs[idx];
956 #endif
957 }
958 
959 //----------------------------------------------------------------------------//
960 
961 template <>
962 inline Reference<V3f>*
963 FileReferences::ref(size_t idx)
964 {
965 #if USE_SHPTR
966  return m_vfRefs[idx].get();
967 #else
968  return m_vfRefs[idx];
969 #endif
970 }
971 
972 //----------------------------------------------------------------------------//
973 
974 template <>
975 inline Reference<double>*
976 FileReferences::ref(size_t idx)
977 {
978 #if USE_SHPTR
979  return m_dRefs[idx].get();
980 #else
981  return m_dRefs[idx];
982 #endif
983 }
984 
985 //----------------------------------------------------------------------------//
986 
987 template <>
988 inline Reference<V3d>*
989 FileReferences::ref(size_t idx)
990 {
991 #if USE_SHPTR
992  return m_vdRefs[idx].get();
993 #else
994  return m_vdRefs[idx];
995 #endif
996 }
997 
998 //----------------------------------------------------------------------------//
999 
1000 template <>
1001 inline size_t FileReferences::append<half>(Reference<half>::Ptr ref)
1002 {
1003  Mutex::scoped_lock lock(m_mutex);
1004 
1005  m_hRefs.push_back(ref);
1006  return m_hRefs.size() - 1;
1007 }
1008 
1009 //----------------------------------------------------------------------------//
1010 
1011 template <>
1012 inline size_t FileReferences::append<V3h>(Reference<V3h>::Ptr ref)
1013 {
1014  Mutex::scoped_lock lock(m_mutex);
1015 
1016  m_vhRefs.push_back(ref);
1017  return m_vhRefs.size() - 1;
1018 }
1019 
1020 //----------------------------------------------------------------------------//
1021 
1022 template <>
1023 inline size_t FileReferences::append<float>(Reference<float>::Ptr ref)
1024 {
1025  Mutex::scoped_lock lock(m_mutex);
1026 
1027  m_fRefs.push_back(ref);
1028  return m_fRefs.size() - 1;
1029 }
1030 
1031 //----------------------------------------------------------------------------//
1032 
1033 template <>
1034 inline size_t FileReferences::append<V3f>(Reference<V3f>::Ptr ref)
1035 {
1036  Mutex::scoped_lock lock(m_mutex);
1037 
1038  m_vfRefs.push_back(ref);
1039  return m_vfRefs.size() - 1;
1040 }
1041 
1042 //----------------------------------------------------------------------------//
1043 
1044 template <>
1045 inline size_t FileReferences::append<double>(Reference<double>::Ptr ref)
1046 {
1047  Mutex::scoped_lock lock(m_mutex);
1048 
1049  m_dRefs.push_back(ref);
1050  return m_dRefs.size() - 1;
1051 }
1052 
1053 //----------------------------------------------------------------------------//
1054 
1055 template <>
1056 inline size_t FileReferences::append<V3d>(Reference<V3d>::Ptr ref)
1057 {
1058  Mutex::scoped_lock lock(m_mutex);
1059 
1060  m_vdRefs.push_back(ref);
1061  return m_vdRefs.size() - 1;
1062 }
1063 
1064 //----------------------------------------------------------------------------//
1065 
1066 template <>
1067 inline size_t FileReferences::numRefs<half>() const
1068 {
1069  Mutex::scoped_lock lock(m_mutex);
1070 
1071  return m_hRefs.size();
1072 }
1073 
1074 //----------------------------------------------------------------------------//
1075 
1076 template <>
1077 inline size_t FileReferences::numRefs<V3h>() const
1078 {
1079  Mutex::scoped_lock lock(m_mutex);
1080 
1081  return m_vhRefs.size();
1082 }
1083 
1084 //----------------------------------------------------------------------------//
1085 
1086 template <>
1087 inline size_t FileReferences::numRefs<float>() const
1088 {
1089  Mutex::scoped_lock lock(m_mutex);
1090 
1091  return m_fRefs.size();
1092 }
1093 
1094 //----------------------------------------------------------------------------//
1095 
1096 template <>
1097 inline size_t FileReferences::numRefs<V3f>() const
1098 {
1099  Mutex::scoped_lock lock(m_mutex);
1100 
1101  return m_vfRefs.size();
1102 }
1103 
1104 //----------------------------------------------------------------------------//
1105 
1106 template <>
1107 inline size_t FileReferences::numRefs<double>() const
1108 {
1109  Mutex::scoped_lock lock(m_mutex);
1110 
1111  return m_dRefs.size();
1112 }
1113 
1114 //----------------------------------------------------------------------------//
1115 
1116 template <>
1117 inline size_t FileReferences::numRefs<V3d>() const
1118 {
1119  Mutex::scoped_lock lock(m_mutex);
1120 
1121  return m_vdRefs.size();
1122 }
1123 
1124 //----------------------------------------------------------------------------//
1125 // Implementations for FileReferences
1126 //----------------------------------------------------------------------------//
1127 
1128 #if 0
1129 
1130 template <class Data_T>
1132 {
1133  assert(false && "Do not use memory limiting on sparse fields that aren't "
1134  "simple scalars or vectors!");
1136  "FileReferences::ref(): Do not use memory limiting on sparse "
1137  "fields that aren't simple scalars or vectors!");
1138  static Reference<Data_T> dummy("", "");
1139  return dummy;
1140 }
1141 
1142 //----------------------------------------------------------------------------//
1143 
1144 template <class Data_T>
1145 size_t FileReferences::append(const Reference<Data_T>&)
1146 {
1147  assert(false && "Do not use memory limiting on sparse fields that aren't "
1148  "simple scalars or vectors!");
1150  "FileReferences::append(): Do not use memory limiting on sparse "
1151  "fields that aren't simple scalars or vectors!");
1152  return -1;
1153 }
1154 
1155 //----------------------------------------------------------------------------//
1156 
1157 template <class Data_T>
1158 size_t FileReferences::numRefs() const
1159 {
1160  assert(false && "Do not use memory limiting on sparse fields that aren't "
1161  "simple scalars or vectors!");
1163  "FileReferences::numRefs(): "
1164  "Do not use memory limiting on sparse "
1165  "fields that aren't "
1166  "simple scalars or vectors!");
1167  return -1;
1168 }
1169 
1170 #endif
1171 
1172 //----------------------------------------------------------------------------//
1173 
1174 } // namespace SparseFile
1175 
1176 //----------------------------------------------------------------------------//
1177 // SparseFileManager implementations
1178 //----------------------------------------------------------------------------//
1179 
1180 template <class Data_T>
1181 int
1182 SparseFileManager::getNextId(const std::string filename,
1183  const std::string layerPath)
1184 {
1185  using namespace SparseFile;
1186 
1187  // Must hold a mutex while appending to m_fileData
1188  boost::mutex::scoped_lock lock(m_mutex);
1189 
1190  int id = m_fileData.append<Data_T>(Reference<Data_T>::create(filename,
1191  layerPath));
1192  return id;
1193 }
1194 
1195 //----------------------------------------------------------------------------//
1196 
1197 template <class Data_T>
1198 void
1200 {
1201  boost::mutex::scoped_lock lock(m_mutex);
1202 
1205 
1206  CacheList::iterator it = m_blockCacheList.begin();
1207  CacheList::iterator end = m_blockCacheList.end();
1208  CacheList::iterator next;
1209 
1210  int64_t bytesFreed = 0;
1211 
1212  while (it != end) {
1213  if (it->blockType == blockType && it->refIdx == refIdx) {
1214  if (it == m_nextBlock) {
1215  ++m_nextBlock;
1216  }
1217  next = it;
1218  ++next;
1219  bytesFreed += reference->blockSize(it->blockIdx);
1220  m_blockCacheList.erase(it);
1221  it = next;
1222  } else {
1223  ++it;
1224  }
1225  }
1226  m_memUse -= bytesFreed;
1227 
1228  std::vector<int>().swap(reference->fileBlockIndices);
1229 #if F3D_NO_BLOCKS_ARRAY
1230  reference->fileBlockIndices.resize(reference->numBlocks, -1);
1231 #else
1232  reference->fileBlockIndices.resize(reference->blocks.size(), -1);
1233  typedef typename SparseFile::Reference<Data_T>::BlockPtrs BlockPtrs;
1234  BlockPtrs().swap(reference->blocks);
1235 #endif
1236  std::vector<int>().swap(reference->blockLoaded);
1237  std::vector<bool>().swap(reference->blockUsed);
1238  std::vector<int>().swap(reference->loadCounts);
1239  std::vector<int>().swap(reference->refCounts);
1240  delete[] reference->blockMutex;
1241  reference->blockMutex = NULL;
1242 }
1243 
1244 //----------------------------------------------------------------------------//
1245 
1246 template <class Data_T>
1249 {
1250  return m_fileData.ref<Data_T>(index);
1251 }
1252 
1253 //----------------------------------------------------------------------------//
1254 
1255 template <class Data_T>
1256 void
1257 SparseFileManager::activateBlock(int fileId, int blockIdx)
1258 {
1260 
1261  if (reference->fileBlockIndices[blockIdx] >= 0) {
1262  if (!reference->blockLoaded[blockIdx]) {
1263  int blockSize = reference->blockSize(blockIdx);
1264  if (m_limitMemUse) {
1265  // if we already have enough free memory, deallocateBlocks()
1266  // will just return
1267  deallocateBlocks(blockSize);
1268  }
1269 
1270  if (!reference->fileIsOpen()) {
1271  reference->openFile();
1272  }
1273 
1274  boost::mutex::scoped_lock lock_A(m_mutex);
1275 #if F3D_SHORT_MUTEX_ARRAY
1276  boost::mutex::scoped_lock
1277  lock_B(reference->blockMutex[blockIdx % reference->blockMutexSize]);
1278 #else
1279  boost::mutex::scoped_lock lock_B(reference->blockMutex[blockIdx]);
1280 #endif
1281  // check to see if it was loaded between when the function
1282  // started and we got the lock on the block
1283  if (!reference->blockLoaded[blockIdx]) {
1284  reference->loadBlock(blockIdx);
1285  reference->loadCounts[blockIdx]++;
1287  m_memUse += blockSize;
1288  }
1289  }
1290  }
1291  reference->blockUsed[blockIdx] = true;
1292 }
1293 
1294 //----------------------------------------------------------------------------//
1295 
1296 template <class Data_T>
1297 void
1298 SparseFileManager::incBlockRef(int fileId, int blockIdx)
1299 {
1301 
1302  if (reference->fileBlockIndices[blockIdx] >= 0) {
1303  reference->incBlockRef(blockIdx);
1304  }
1305 }
1306 
1307 //----------------------------------------------------------------------------//
1308 
1309 template <class Data_T>
1310 void
1311 SparseFileManager::decBlockRef(int fileId, int blockIdx)
1312 {
1314 
1315  if (reference->fileBlockIndices[blockIdx] >= 0) {
1316  reference->decBlockRef(blockIdx);
1317  }
1318 }
1319 
1320 //----------------------------------------------------------------------------//
1321 
1323 
1324 //----------------------------------------------------------------------------//
1325 
1326 #endif
FIELD3D_NAMESPACE_HEADER_CLOSE
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
SparseFileManager::reference
SparseFile::Reference< Data_T > * reference(int index)
Returns a reference to the Reference object with the given index.
Definition: SparseFile.h:1248
SparseFile::Reference::blockUsed
std::vector< bool > blockUsed
Flags of whether the blocks have been accessed since they were last considered for deallocation by th...
Definition: SparseFile.h:145
SparseFile::Reference::layerPath
std::string layerPath
Definition: SparseFile.h:128
SparseFile::CacheBlock::refIdx
int refIdx
Definition: SparseFile.h:325
SparseFile::Reference::averageLoads
float averageLoads() const
Returns the average number of loads per accessed block in this file, for cache statistics.
Definition: SparseFile.h:833
OgSparseDataReader
Definition: SparseFile.h:87
SparseFileManager::removeFieldFromCache
void removeFieldFromCache(int refIdx)
Definition: SparseFile.h:1199
SparseFile::Reference::m_mutex
Mutex m_mutex
Mutex to prevent two threads from modifying conflicting data.
Definition: SparseFile.h:254
SparseFile::Reference::closeFile
void closeFile()
Closes the file. Will be re-opened as needed.
Definition: SparseFile.h:710
SparseFile::Reference::openFile
void openFile()
Opens the file. This is done just before the first request to loadBlock. This is delayed so that the ...
Definition: SparseFile.cpp:537
Msg::SevWarning
@ SevWarning
Definition: Log.h:68
SparseFile::Reference::fileIsOpen
bool fileIsOpen()
Checks if the file used by this reference is open already.
Definition: SparseFile.h:669
SparseFile::Reference::loadCounts
std::vector< int > loadCounts
Per-block counts of the number of times each block has been loaded, for cache statistics.
Definition: SparseFile.h:148
SparseFile::CacheBlock::CacheBlock
CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn, int blockIdxIn)
Definition: SparseFile.h:327
SparseFile::FileReferences::numRefs
size_t numRefs() const
Returns the number of file references of the corresponding collection.
SparseFile::Reference::memSize
long long int memSize() const
Memory use for the Reference.
Definition: SparseFile.h:863
Sparse::SparseBlock
Definition: SparseField.h:228
SparseFileManager::m_blockCacheList
CacheList m_blockCacheList
List of dynamically loaded blocks to be considered for unloading when the cache is full....
Definition: SparseFile.h:538
SparseFileManager::activateBlock
void activateBlock(int fileId, int blockIdx)
Called by SparseField when it's about to read from a block. This should not be called by the user,...
Definition: SparseFile.h:1257
SparseDataReader.h
Contains functions controlling the loading of sparse fields.
SparseFileManager::incBlockRef
void incBlockRef(int fileId, int blockIdx)
Increments the usage reference count on the specified block, to prevent it from getting unloaded whil...
Definition: SparseFile.h:1298
SparseField::memSize
virtual long long int memSize() const
Returns the memory usage (in bytes)
Definition: SparseField.h:1708
F3D_SHORT_MUTEX_ARRAY
#define F3D_SHORT_MUTEX_ARRAY
Definition: SparseFile.h:64
SparseFile::Reference::Ptr
Reference * Ptr
Definition: SparseFile.h:116
SparseFile::Reference::m_numActiveBlocks
size_t m_numActiveBlocks
Number of currently active blocks.
Definition: SparseFile.h:257
SparseFile::Reference::m_fileHandle
hid_t m_fileHandle
Holds the Hdf5 handle to the file.
Definition: SparseFile.h:231
SparseFile::Reference::blockMutexSize
int blockMutexSize
Size of the mutex array. Used as modulus base.
Definition: SparseFile.h:159
DataTypeEnum
DataTypeEnum
Definition: Traits.h:108
SparseFile::FileReferences::m_vhRefs
std::deque< Reference< V3h >::Ptr > m_vhRefs
Definition: SparseFile.h:306
SparseFile::Reference::unloadBlock
void unloadBlock(int blockIdx)
Unloads the block with the given index from memory.
Definition: SparseFile.h:722
SparseFile::Reference::m_ogRoot
OgIGroupPtr m_ogRoot
Ogawa archive root.
Definition: SparseFile.h:249
SparseFile::Reference::m_ogReaderPtr
boost::shared_ptr< OgSparseDataReader< Data_T > > m_ogReaderPtr
Shared pointer to the ogawa reader.
Definition: SparseFile.h:242
SparseFile
Definition: SparseFile.h:91
SparseFileManager::m_maxMemUseInBytes
int64_t m_maxMemUseInBytes
Max amount om memory to use in bytes.
Definition: SparseFile.h:520
SparseFile::FileReferences
Definition: SparseFile.h:266
SparseFile::Reference::m_ogArchive
IArchivePtr m_ogArchive
Ogawa archive.
Definition: SparseFile.h:247
Hdf5Util::H5ScopedGopen
Scoped object - opens a group on creation and closes it on destruction.
Definition: Hdf5Util.h:195
SparseFile::FileReferences::ref
Reference< Data_T > * ref(size_t idx)
Returns a reference to the index. This is specialized so that the correct data member is accessed.
SparseFile::Reference::blocks
BlockPtrs blocks
Pointers to each block. This is so we can go in and manipulate them as we please.
Definition: SparseFile.h:141
SparseFile::Reference::create
static Ptr create(const std::string a_filename, const std::string a_layerPath)
Returns a shared pointer to a reference. Preferred way of creating References.
Definition: SparseFile.h:660
SparseFile::CacheBlock::blockIdx
int blockIdx
Definition: SparseFile.h:326
SparseField
This Field subclass stores voxel data in block-allocated arrays.
Definition: SparseField.h:352
SparseFileManager::m_limitMemUse
bool m_limitMemUse
Whether to limit memory use of sparse fields from disk. Enables the cache and dynamic loading when tr...
Definition: SparseFile.h:527
SparseFile::FileReferences::m_hRefs
std::deque< Reference< half >::Ptr > m_hRefs
Definition: SparseFile.h:305
OgawaFwd.h
Contains forward declarations for Ogawa classes.
SparseFile::FileReferences::append
size_t append(typename Reference< Data_T >::Ptr ref)
Appends a reference to the collection. This is specialized so that the correct data member is accesse...
SparseFile::Reference::incBlockRef
void incBlockRef(int blockIdx)
Increment reference count on a block, indicates the block is currently in use, so prevents it from be...
Definition: SparseFile.h:745
ns.h
Exception.h
Contains Exception base class.
SparseFile::Reference::Mutex
boost::mutex Mutex
Definition: SparseFile.h:215
SparseFile::Reference::setNumBlocks
void setNumBlocks(int numBlocks)
Sets the number of blocks used by the SparseField we're supporting.
Definition: SparseFile.h:677
SparseFile::FileReferences::m_dRefs
std::deque< Reference< double >::Ptr > m_dRefs
Definition: SparseFile.h:309
SparseFile::CacheBlock::blockType
DataTypeEnum blockType
Definition: SparseFile.h:324
SparseFileManager::m_fileData
SparseFile::FileReferences m_fileData
Vector containing information for each of the managed fields. The order matches the index stored in e...
Definition: SparseFile.h:531
SparseFile::Reference::operator=
Reference & operator=(const Reference &o)
Assignment operator. Clears ref counts and rebuilds mutex array.
Definition: SparseFile.h:600
Hdf5Util.h
Contains various utility functions for Hdf5.
SparseFileManager::m_memUse
int64_t m_memUse
Current amount of memory in use in bytes.
Definition: SparseFile.h:523
IArchivePtr
boost::shared_ptr< Alembic::Ogawa::IArchive > IArchivePtr
Definition: OgawaFwd.h:87
detail::max
T max(const T a, const T2 b)
Max operation on mixed types.
Definition: FieldSampler.h:32
SparseFileManager::m_maxMemUse
float m_maxMemUse
Max amount om memory to use in megabytes.
Definition: SparseFile.h:517
SparseFile::Reference::m_ogReader
OgSparseDataReader< Data_T > * m_ogReader
Pointer to the ogawa reader. NULL at construction time. Created in openFile().
Definition: SparseFile.h:245
SparseFile::Reference::m_reader
SparseDataReader< Data_T > * m_reader
Pointer to the reader object. NULL at construction time. Created in openFile().
Definition: SparseFile.h:239
SparseFileManager::m_mutex
boost::mutex m_mutex
Mutex to prevent multiple threads from deallocating blocks at the same time.
Definition: SparseFile.h:546
SparseFile::Reference::fileBlockIndices
std::vector< int > fileBlockIndices
Index in file for each block.
Definition: SparseFile.h:135
SparseFileManager::getNextId
int getNextId(const std::string filename, const std::string layerPath)
Returns the id of the next cache item. This is stored in the SparseField in order to reference its fi...
Definition: SparseFile.h:1182
SparseFile::Reference::blockSize
int blockSize(int blockIdx) const
Returns the number of bytes used by the data in the block.
Definition: SparseFile.h:771
DataTypeTraits::typeEnum
static DataTypeEnum typeEnum()
SparseFileManager::addBlockToCache
void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx)
Adds the newly loaded block to the cache, managed by the paging algorithm.
Definition: SparseFile.cpp:260
SparseFile::Reference::m_ogLayerGroup
OgIGroupPtr m_ogLayerGroup
Ogawa layer group.
Definition: SparseFile.h:251
SparseFile::Reference::decBlockRef
void decBlockRef(int blockIdx)
Decrement reference count on a block.
Definition: SparseFile.h:758
SparseFile::Reference::refCounts
std::vector< int > refCounts
Per-block counts of the number of current references to the blocks. If a block's ref count is non-zer...
Definition: SparseFile.h:152
Sparse
Namespace for sparse field specifics.
Definition: SparseField.h:221
SparseField::activateBlock
void activateBlock(const int blockId) const
Activates a given block.
Definition: SparseField.h:1927
SparseFileManager
Definition: SparseFile.h:398
SparseFileManager::CacheList
std::list< SparseFile::CacheBlock > CacheList
Definition: SparseFile.h:407
SparseDataReader
This class gets used by SparseFieldIO and SparseFileManager to read the block data....
Definition: SparseDataReader.h:71
SparseFile::Reference::Reference
Reference(const std::string filename, const std::string layerPath)
Constructor. Requires the filename and layer path of the field to be known.
Definition: SparseFile.h:559
OgIGroupPtr
boost::shared_ptr< OgIGroup > OgIGroupPtr
Definition: OgawaFwd.h:69
SparseField::decBlockRef
void decBlockRef(const int blockId) const
Decrements the block ref count for the given block.
Definition: SparseField.h:1935
SparseFileManager::m_nextBlock
CacheList::iterator m_nextBlock
Pointer to the next block to test for unloading in the cache, the "hand" of the clock.
Definition: SparseFile.h:542
SparseFile::Reference
Definition: SparseFile.h:106
SparseFile::Reference::occupiedBlocks
int occupiedBlocks
Definition: SparseFile.h:132
SparseFile::FileReferences::m_vfRefs
std::deque< Reference< V3f >::Ptr > m_vfRefs
Definition: SparseFile.h:308
SparseFile::Reference::numBlocks
int numBlocks
Definition: SparseFile.h:131
SparseFile::FileReferences::m_mutex
Mutex m_mutex
Definition: SparseFile.h:316
Traits.h
SparseFile::FileReferences::memSize
long long int memSize() const
Returns the memory use for the refs.
Definition: SparseFile.cpp:452
SparseFile::Reference::numLoadedBlocks
int numLoadedBlocks() const
Returns the total number of blocks that are currently loaded, for statistics.
Definition: SparseFile.h:793
SparseFile::Reference::~Reference
~Reference()
Destructor.
Definition: SparseFile.h:572
SparseFileManager::decBlockRef
void decBlockRef(int fileId, int blockIdx)
Decrements the usage reference count on the specified block, after its value is no longer being used ...
Definition: SparseFile.h:1311
SparseField::incBlockRef
void incBlockRef(const int blockId) const
Increments the block ref count for the given block.
Definition: SparseField.h:1919
SparseFile::FileReferences::~FileReferences
~FileReferences()
Definition: SparseFile.h:896
SparseFile::Reference::blockMutex
boost::mutex * blockMutex
Allocated array of mutexes, one per block, to lock each block individually, for guaranteeing thread-s...
Definition: SparseFile.h:156
SparseFile::Reference::resetCacheStatistics
void resetCacheStatistics()
Resets counts of total block loads.
Definition: SparseFile.h:851
SparseFile::Reference::m_layerGroup
Hdf5Util::H5ScopedGopen m_layerGroup
Hold the group containing the data open for the duration of the Reference's existence.
Definition: SparseFile.h:235
DataTypeTraits
Definition: Traits.h:266
SparseFile::Reference::totalLoads
int totalLoads() const
Returns the total number of loads of the blocks of this file, for cache statistics.
Definition: SparseFile.h:779
SparseFile::FileReferences::m_vdRefs
std::deque< Reference< V3d >::Ptr > m_vdRefs
Definition: SparseFile.h:310
FIELD3D_API
#define FIELD3D_API
Definition: ns.h:77
FIELD3D_NAMESPACE_OPEN
Definition: FieldMapping.cpp:74
SparseFileManager::deallocateBlocks
void deallocateBlocks(int64_t bytesNeeded)
Utility function to reclaim the specified number of bytes by deallocating unneeded blocks.
Definition: SparseFile.cpp:160
SparseFileManager::ms_singleton
static boost::scoped_ptr< SparseFileManager > ms_singleton
Pointer to singleton.
Definition: SparseFile.h:498
SparseFile::Reference::blockLoaded
std::vector< int > blockLoaded
Whether each block is loaded. We don't use bools since vector<bool> is weird.
Definition: SparseFile.h:138
Msg::print
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity.
Definition: Log.cpp:70
SparseFile::Reference::BlockPtrs
Sparse::SparseBlock< Data_T > * BlockPtrs
Definition: SparseFile.h:120
detail::min
T min(const T a, const T2 b)
Min operation on mixed types.
Definition: FieldSampler.h:25
SparseFile::FileReferences::m_fRefs
std::deque< Reference< float >::Ptr > m_fRefs
Definition: SparseFile.h:307
F3D_MUTEX_ARRAY_SIZE
#define F3D_MUTEX_ARRAY_SIZE
Definition: SparseFile.h:65
SparseFile::Reference::numVoxels
int numVoxels
Definition: SparseFile.h:130
SparseFile::FileReferences::Mutex
boost::mutex Mutex
Definition: SparseFile.h:314
SparseFile::Reference::loadBlock
void loadBlock(int blockIdx)
Loads the block with the given index into memory. We don't pass in a reference to where the data shou...
Definition: SparseFile.cpp:498
SparseFile::Reference::totalLoadedBlocks
int totalLoadedBlocks() const
Returns the total number of blocks that were ever loaded (max 1 per block, not the number of blocks),...
Definition: SparseFile.h:808
SparseFile::Reference::filename
std::string filename
Definition: SparseFile.h:127
SparseFile::CacheBlock
Definition: SparseFile.h:322
SparseFile::Reference::valuesPerBlock
int valuesPerBlock
Definition: SparseFile.h:129
F3D_NO_BLOCKS_ARRAY
#define F3D_NO_BLOCKS_ARRAY
Definition: SparseFile.h:66