Field3D
SparseFile.cpp
Go to the documentation of this file.
00001 //----------------------------------------------------------------------------//
00002 
00003 /*
00004  * Copyright (c) 2009 Sony Pictures Imageworks Inc
00005  *
00006  * All rights reserved.
00007  *
00008  * Redistribution and use in source and binary forms, with or without
00009  * modification, are permitted provided that the following conditions
00010  * are met:
00011  *
00012  * Redistributions of source code must retain the above copyright
00013  * notice, this list of conditions and the following disclaimer.
00014  * Redistributions in binary form must reproduce the above copyright
00015  * notice, this list of conditions and the following disclaimer in the
00016  * documentation and/or other materials provided with the
00017  * distribution.  Neither the name of Sony Pictures Imageworks nor the
00018  * names of its contributors may be used to endorse or promote
00019  * products derived from this software without specific prior written
00020  * permission.
00021  *
00022  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
00023  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
00024  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
00025  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
00026  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
00027  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
00028  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
00029  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
00030  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
00031  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
00032  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
00033  * OF THE POSSIBILITY OF SUCH DAMAGE.
00034  */
00035 
00036 //----------------------------------------------------------------------------//
00037 
00042 //----------------------------------------------------------------------------//
00043 
00044 // SparseField.h includes SparseFile.h, but we need the definition of
00045 // SparseBlock from SparseField.h, so just include that to get both
00046 // files
00047 #include "SparseField.h"
00048 
00049 //----------------------------------------------------------------------------//
00050 
00051 FIELD3D_NAMESPACE_OPEN
00052 
00053 //----------------------------------------------------------------------------//
00054 // Static instances
00055 //----------------------------------------------------------------------------//
00056 
00057 SparseFileManager *SparseFileManager::ms_singleton = 0;
00058 
00059 //----------------------------------------------------------------------------//
00060 // SparseFileManager
00061 //----------------------------------------------------------------------------//
00062 
00063 SparseFileManager & SparseFileManager::singleton()
00064 { 
00065   if (!ms_singleton) {
00066     ms_singleton = new SparseFileManager;
00067   }
00068   return *ms_singleton;
00069 }
00070 
00071 //----------------------------------------------------------------------------//
00072 
00073 void SparseFileManager::setLimitMemUse(bool enabled) 
00074 {
00075   m_limitMemUse = enabled;
00076 }
00077 
00078 //----------------------------------------------------------------------------//
00079 
00080 bool SparseFileManager::doLimitMemUse() const
00081 { 
00082   return m_limitMemUse; 
00083 }
00084 
00085 //----------------------------------------------------------------------------//
00086 
00087 void SparseFileManager::setMaxMemUse(float maxMemUse) 
00088 {
00089   m_maxMemUse = maxMemUse;
00090   m_maxMemUseInBytes = static_cast<int>(m_maxMemUse * 1024*1024);
00091 }
00092 
00093 //----------------------------------------------------------------------------//
00094 
00095 template <class Data_T>
00096 int SparseFileManager::deallocateBlock(const SparseFile::CacheBlock &cb)
00097 {
00098   int bytesFreed = 0;
00099   SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(cb.refIdx);
00100 
00101   // Note: we don't need to lock the block's mutex because
00102   // deallocateBlock() is only called while the SparseFileManager's
00103   // mutex is also locked (in flushCache() or deallocateBlocks()).
00104   // Don't lock the block, to make sure we don't have a deadlock by
00105   // holding two locks at the same time.  (Because addBlockToCache()
00106   // locks the manager but is also in a block-specific lock.)
00107 
00108   // lock the current block to make sure its blockUsed flag and ref
00109   // counts don't change
00110   // Note: this lock order is made consistent w/ allocate to prevent
00111   // deadlocks and crashes.
00112 
00113   boost::mutex::scoped_lock lock_B(reference.blockMutex[cb.blockIdx]);
00114   
00115   // check whether the block is still in use
00116   if (reference.refCounts[cb.blockIdx] > 0)
00117     return bytesFreed;
00118 
00119   if (reference.blockUsed[cb.blockIdx]) {
00120     // the block was recently used according to Second-chance paging
00121     // algorithm, so skip it
00122     reference.blockUsed[cb.blockIdx] = false;
00123   }
00124   else {
00125 
00126     // the block wasn't in use, so free it
00127     reference.unloadBlock(cb.blockIdx);
00128     bytesFreed = reference.blockSize(cb.blockIdx);
00129     m_memUse -= bytesFreed;
00130     CacheList::iterator toRemove = m_nextBlock;
00131     ++m_nextBlock;
00132     m_blockCacheList.erase(toRemove);
00133   }
00134   return bytesFreed;
00135 }
00136 
00137 //----------------------------------------------------------------------------//
00138 
00139 template <class Data_T>
00140 void SparseFileManager::deallocateBlock(CacheList::iterator &it)
00141 {
00142   SparseFile::CacheBlock &cb = *it;
00143   SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(cb.refIdx);
00144   int bytesFreed = reference.blockSize(cb.blockIdx);
00145   m_memUse -= bytesFreed;
00146   reference.unloadBlock(cb.blockIdx);
00147   it = m_blockCacheList.erase(it);
00148 }
00149 
00150 //----------------------------------------------------------------------------//
00151 
00152 void SparseFileManager::deallocateBlocks(int bytesNeeded)
00153 {
00154   boost::mutex::scoped_lock lock_A(m_mutex);
00155 
00156   while (m_blockCacheList.begin() != m_blockCacheList.end() &&
00157          m_maxMemUseInBytes-m_memUse < bytesNeeded) {
00158 
00159     if (m_nextBlock == m_blockCacheList.end())
00160       m_nextBlock = m_blockCacheList.begin();
00161 
00162     SparseFile::CacheBlock &cb = *m_nextBlock;
00163 
00164     // if bytesFreed is set to >0, then we've already freed a block
00165     // and advanced the "clock hand" iterator
00166     int bytesFreed = 0;
00167 
00168     switch(cb.blockType) {
00169     case DataTypeHalf:
00170       bytesFreed = deallocateBlock<half>(cb);
00171       if (bytesFreed > 0) {
00172         continue;
00173       }
00174       break;
00175     case DataTypeFloat:
00176       bytesFreed = deallocateBlock<float>(cb);
00177       if (bytesFreed > 0) {
00178         continue;
00179       }
00180       break;
00181     case DataTypeDouble:
00182       bytesFreed = deallocateBlock<double>(cb);
00183       if (bytesFreed > 0) {
00184         continue;
00185       }
00186       break;
00187     case DataTypeVecHalf:
00188       bytesFreed = deallocateBlock<V3h>(cb);
00189       if (bytesFreed > 0) {
00190         continue;
00191       }
00192       break;
00193     case DataTypeVecFloat:
00194       bytesFreed = deallocateBlock<V3f>(cb);
00195       if (bytesFreed > 0) {
00196         continue;
00197       }
00198       break;
00199     case DataTypeVecDouble:
00200       bytesFreed = deallocateBlock<V3d>(cb);
00201       if (bytesFreed > 0) {
00202         continue;
00203       }
00204       break;
00205     case DataTypeUnknown:
00206     default:
00207       break;
00208     }
00209     ++m_nextBlock;
00210   }
00211 }
00212 
00213 //----------------------------------------------------------------------------//
00214 
00215 void SparseFileManager::flushCache()
00216 {
00217   boost::mutex::scoped_lock lock(m_mutex);
00218 
00219   CacheList::iterator it = m_blockCacheList.begin();
00220   while (it != m_blockCacheList.end()) {
00221     SparseFile::CacheBlock &cb = *it;
00222 
00223     switch(cb.blockType) {
00224     case DataTypeHalf:
00225       deallocateBlock<half>(it);
00226       break;
00227     case DataTypeFloat:
00228       deallocateBlock<float>(it);
00229       break;
00230     case DataTypeDouble:
00231       deallocateBlock<double>(it);
00232       break;
00233     case DataTypeVecHalf:
00234       deallocateBlock<V3h>(it);
00235       break;
00236     case DataTypeVecFloat:
00237       deallocateBlock<V3f>(it);
00238       break;
00239     case DataTypeVecDouble:
00240       deallocateBlock<V3d>(it);
00241       break;
00242     case DataTypeUnknown:
00243     default:
00244       break;
00245     }
00246   }
00247   m_nextBlock = m_blockCacheList.begin();
00248 }
00249 
00250 //----------------------------------------------------------------------------//
00251 
00252 void SparseFileManager::addBlockToCache(DataTypeEnum blockType,
00253                                         int fileId, int blockIdx)
00254 {
00255   // Note: this lock is obtained while we also have a lock on the
00256   // specific block (in activateBlock()), so we should make sure we
00257   // never lock the SparseFileManager and *then* a block, to ensure we
00258   // don't have a deadlock.
00259   //
00260   //  Note: this was changed so the order was consistent w/ dealloc
00261   //  again, see activateBlock()
00262   //  boost::mutex::scoped_lock lock(m_mutex);
00263 
00264   SparseFile::CacheBlock block(blockType, fileId, blockIdx);
00265   if (m_nextBlock == m_blockCacheList.end()) {
00266     m_blockCacheList.push_back(block);
00267   } else {
00268     m_blockCacheList.insert(m_nextBlock, block);
00269   }
00270 }
00271 
00272 //----------------------------------------------------------------------------//
00273 
00274 SparseFileManager::SparseFileManager()
00275   : m_memUse(0),
00276     m_limitMemUse(false)
00277 {
00278   setMaxMemUse(1000.0);
00279   m_nextBlock = m_blockCacheList.begin();
00280 }
00281 
00282 //----------------------------------------------------------------------------//
00283 
00284 long long SparseFileManager::totalLoads()
00285 {
00286 
00287   long long int numLoads = 0;
00288 
00289   for (int i=0; i<m_fileData.numRefs<half>(); i++) {
00290     numLoads += m_fileData.ref<half>(i).totalLoads();
00291   }
00292 
00293   for (int i=0; i<m_fileData.numRefs<V3h>(); i++) {
00294     numLoads += m_fileData.ref<V3h>(i).totalLoads();
00295   }
00296 
00297   for (int i=0; i<m_fileData.numRefs<float>(); i++) {
00298     numLoads += m_fileData.ref<float>(i).totalLoads();
00299   }
00300 
00301   for (int i=0; i<m_fileData.numRefs<V3f>(); i++) {
00302     numLoads += m_fileData.ref<V3f>(i).totalLoads();
00303   }
00304 
00305   for (int i=0; i<m_fileData.numRefs<double>(); i++) {
00306     numLoads += m_fileData.ref<double>(i).totalLoads();
00307   }
00308 
00309   for (int i=0; i<m_fileData.numRefs<V3d>(); i++) {
00310     numLoads += m_fileData.ref<V3d>(i).totalLoads();
00311   }
00312   return numLoads;
00313 }
00314 
00315 //----------------------------------------------------------------------------//
00316 
00317 long long SparseFileManager::numLoadedBlocks()
00318 {
00319 
00320   long long int numBlocks = 0;
00321 
00322   for (int i=0; i<m_fileData.numRefs<half>(); i++) {
00323     numBlocks += m_fileData.ref<half>(i).numLoadedBlocks();
00324   }
00325 
00326   for (int i=0; i<m_fileData.numRefs<V3h>(); i++) {
00327     numBlocks += m_fileData.ref<V3h>(i).numLoadedBlocks();
00328   }
00329 
00330   for (int i=0; i<m_fileData.numRefs<float>(); i++) {
00331     numBlocks += m_fileData.ref<float>(i).numLoadedBlocks();
00332   }
00333 
00334   for (int i=0; i<m_fileData.numRefs<V3f>(); i++) {
00335     numBlocks += m_fileData.ref<V3f>(i).numLoadedBlocks();
00336   }
00337 
00338   for (int i=0; i<m_fileData.numRefs<double>(); i++) {
00339     numBlocks += m_fileData.ref<double>(i).numLoadedBlocks();
00340   }
00341 
00342   for (int i=0; i<m_fileData.numRefs<V3d>(); i++) {
00343     numBlocks += m_fileData.ref<V3d>(i).numLoadedBlocks();
00344   }
00345   return numBlocks;
00346 }
00347 
00348 //----------------------------------------------------------------------------//
00349 
00350 long long SparseFileManager::totalLoadedBlocks()
00351 {
00352 
00353   long long int numBlocks = 0;
00354 
00355   for (int i=0; i<m_fileData.numRefs<half>(); i++) {
00356     numBlocks += m_fileData.ref<half>(i).totalLoadedBlocks();
00357   }
00358 
00359   for (int i=0; i<m_fileData.numRefs<V3h>(); i++) {
00360     numBlocks += m_fileData.ref<V3h>(i).totalLoadedBlocks();
00361   }
00362 
00363   for (int i=0; i<m_fileData.numRefs<float>(); i++) {
00364     numBlocks += m_fileData.ref<float>(i).totalLoadedBlocks();
00365   }
00366 
00367   for (int i=0; i<m_fileData.numRefs<V3f>(); i++) {
00368     numBlocks += m_fileData.ref<V3f>(i).totalLoadedBlocks();
00369   }
00370 
00371   for (int i=0; i<m_fileData.numRefs<double>(); i++) {
00372     numBlocks += m_fileData.ref<double>(i).totalLoadedBlocks();
00373   }
00374 
00375   for (int i=0; i<m_fileData.numRefs<V3d>(); i++) {
00376     numBlocks += m_fileData.ref<V3d>(i).totalLoadedBlocks();
00377   }
00378   return numBlocks;
00379 }
00380 
00381 //----------------------------------------------------------------------------//
00382 
00383 float SparseFileManager::cacheFractionLoaded()
00384 {
00385   return ((double)numLoadedBlocks())/std::max(1.0, ((double)totalLoadedBlocks()));
00386 }
00387 
00388 //----------------------------------------------------------------------------//
00389 
00390 float SparseFileManager::cacheLoadsPerBlock()
00391 {
00392   return ((double)totalLoads())/std::max(1.0, ((double)totalLoadedBlocks()));
00393 }
00394 
00395 //----------------------------------------------------------------------------//
00396 
00397 float SparseFileManager::cacheEfficiency()
00398 {
00399   return ((double)totalLoadedBlocks())/std::max(1.0, ((double)totalLoads()));
00400 }
00401 
00402 //----------------------------------------------------------------------------//
00403 
00404 void SparseFileManager::resetCacheStatistics()
00405 {
00406 
00407   for (int i=0; i<m_fileData.numRefs<half>(); i++) {
00408     m_fileData.ref<half>(i).resetCacheStatistics();
00409   }
00410 
00411   for (int i=0; i<m_fileData.numRefs<V3h>(); i++) {
00412     m_fileData.ref<V3h>(i).resetCacheStatistics();
00413   }
00414 
00415   for (int i=0; i<m_fileData.numRefs<float>(); i++) {
00416     m_fileData.ref<float>(i).resetCacheStatistics();
00417   }
00418 
00419   for (int i=0; i<m_fileData.numRefs<V3f>(); i++) {
00420     m_fileData.ref<V3f>(i).resetCacheStatistics();
00421   }
00422 
00423   for (int i=0; i<m_fileData.numRefs<double>(); i++) {
00424     m_fileData.ref<double>(i).resetCacheStatistics();
00425   }
00426 
00427   for (int i=0; i<m_fileData.numRefs<V3d>(); i++) {
00428     m_fileData.ref<V3d>(i).resetCacheStatistics();
00429   }
00430 }
00431 
00432 //----------------------------------------------------------------------------//
00433 
00434 FIELD3D_NAMESPACE_HEADER_CLOSE
00435 
00436 //----------------------------------------------------------------------------//
00437