The Tor Browser / file revision

 /* This Source Code Form is subject to the terms of the Mozilla Public

  * License, v. 2.0. If a copy of the MPL was not distributed with this

  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

 #ifndef CacheIndex__h__

 #define CacheIndex__h__

 #include "CacheLog.h"

 #include "CacheFileIOManager.h"

 #include "nsIRunnable.h"

 #include "CacheHashUtils.h"

 #include "nsICacheStorageService.h"

 #include "nsICacheEntry.h"

 #include "nsILoadContextInfo.h"

 #include "nsTHashtable.h"

 #include "nsThreadUtils.h"

 #include "nsWeakReference.h"

 #include "mozilla/SHA1.h"

 #include "mozilla/Mutex.h"

 #include "mozilla/Endian.h"

 #include "mozilla/TimeStamp.h"

 class nsIFile;

 class nsIDirectoryEnumerator;

 class nsITimer;

 #ifdef DEBUG

 #define DEBUG_STATS 1

 #endif

 namespace mozilla {

 namespace net {

 class CacheFileMetadata;

 class FileOpenHelper;

 class CacheIndexIterator;

 typedef struct {

   // Version of the index. The index must be ignored and deleted when the file

   // on disk was written with a newer version.

   uint32_t mVersion;

   // Timestamp of time when the last successful write of the index started.

   // During update process we use this timestamp for a quick validation of entry

   // files. If last modified time of the file is lower than this timestamp, we

   // skip parsing of such file since the information in index should be up to

   // date.

   uint32_t mTimeStamp;

   // We set this flag as soon as possible after parsing index during startup

   // and clean it after we write journal to disk during shutdown. We ignore the

   // journal and start update process whenever this flag is set during index

   // parsing.

   uint32_t mIsDirty;

 } CacheIndexHeader;

 struct CacheIndexRecord {

   SHA1Sum::Hash mHash;

   uint32_t      mFrecency;

   uint32_t      mExpirationTime;

   uint32_t      mAppId;

   /*

    *    1000 0000 0000 0000 0000 0000 0000 0000 : initialized

    *    0100 0000 0000 0000 0000 0000 0000 0000 : anonymous

    *    0010 0000 0000 0000 0000 0000 0000 0000 : inBrowser

    *    0001 0000 0000 0000 0000 0000 0000 0000 : removed

    *    0000 1000 0000 0000 0000 0000 0000 0000 : dirty

    *    0000 0100 0000 0000 0000 0000 0000 0000 : fresh

    *    0000 0011 0000 0000 0000 0000 0000 0000 : reserved

    *    0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB)

    */

   uint32_t      mFlags;

   CacheIndexRecord()

     : mFrecency(0)

     , mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME)

     , mAppId(nsILoadContextInfo::NO_APP_ID)

     , mFlags(0)

   {}

 };

 class CacheIndexEntry : public PLDHashEntryHdr

 {

 public:

   typedef const SHA1Sum::Hash& KeyType;

   typedef const SHA1Sum::Hash* KeyTypePointer;

   CacheIndexEntry(KeyTypePointer aKey)

   {

     MOZ_COUNT_CTOR(CacheIndexEntry);

     mRec = new CacheIndexRecord();

     LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec.get()));

     memcpy(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash));

   }

   CacheIndexEntry(const CacheIndexEntry& aOther)

   {

     NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!");

   }

   ~CacheIndexEntry()

   {

     MOZ_COUNT_DTOR(CacheIndexEntry);

     LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]",

          mRec.get()));

   }

   // KeyEquals(): does this entry match this key?

   bool KeyEquals(KeyTypePointer aKey) const

   {

     return memcmp(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0;

   }

   // KeyToPointer(): Convert KeyType to KeyTypePointer

   static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }

   // HashKey(): calculate the hash number

   static PLDHashNumber HashKey(KeyTypePointer aKey)

   {

     return (reinterpret_cast<const uint32_t *>(aKey))[0];

   }

   // ALLOW_MEMMOVE can we move this class with memmove(), or do we have

   // to use the copy constructor?

   enum { ALLOW_MEMMOVE = true };

   bool operator==(const CacheIndexEntry& aOther) const

   {

     return KeyEquals(&aOther.mRec->mHash);

   }

   CacheIndexEntry& operator=(const CacheIndexEntry& aOther)

   {

     MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash,

                sizeof(SHA1Sum::Hash)) == 0);

     mRec->mFrecency = aOther.mRec->mFrecency;

     mRec->mExpirationTime = aOther.mRec->mExpirationTime;

     mRec->mAppId = aOther.mRec->mAppId;

     mRec->mFlags = aOther.mRec->mFlags;

     return *this;

   }

   void InitNew()

   {

     mRec->mFrecency = 0;

     mRec->mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;

     mRec->mAppId = nsILoadContextInfo::NO_APP_ID;

     mRec->mFlags = 0;

   }

   void Init(uint32_t aAppId, bool aAnonymous, bool aInBrowser)

   {

     MOZ_ASSERT(mRec->mFrecency == 0);

     MOZ_ASSERT(mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME);

     MOZ_ASSERT(mRec->mAppId == nsILoadContextInfo::NO_APP_ID);

     // When we init the entry it must be fresh and may be dirty

     MOZ_ASSERT((mRec->mFlags & ~kDirtyMask) == kFreshMask);

     mRec->mAppId = aAppId;

     mRec->mFlags |= kInitializedMask;

     if (aAnonymous) {

       mRec->mFlags |= kAnonymousMask;

     }

     if (aInBrowser) {

       mRec->mFlags |= kInBrowserMask;

     }

   }

   const SHA1Sum::Hash * Hash() { return &mRec->mHash; }

   bool IsInitialized() { return !!(mRec->mFlags & kInitializedMask); }

   uint32_t AppId() { return mRec->mAppId; }

   bool     Anonymous() { return !!(mRec->mFlags & kAnonymousMask); }

   bool     InBrowser() { return !!(mRec->mFlags & kInBrowserMask); }

   bool IsRemoved() { return !!(mRec->mFlags & kRemovedMask); }

   void MarkRemoved() { mRec->mFlags |= kRemovedMask; }

   bool IsDirty() { return !!(mRec->mFlags & kDirtyMask); }

   void MarkDirty() { mRec->mFlags |= kDirtyMask; }

   void ClearDirty() { mRec->mFlags &= ~kDirtyMask; }

   bool IsFresh() { return !!(mRec->mFlags & kFreshMask); }

   void MarkFresh() { mRec->mFlags |= kFreshMask; }

   void     SetFrecency(uint32_t aFrecency) { mRec->mFrecency = aFrecency; }

   uint32_t GetFrecency() { return mRec->mFrecency; }

   void     SetExpirationTime(uint32_t aExpirationTime)

   {

     mRec->mExpirationTime = aExpirationTime;

   }

   uint32_t GetExpirationTime() { return mRec->mExpirationTime; }

   // Sets filesize in kilobytes.

   void     SetFileSize(uint32_t aFileSize)

   {

     if (aFileSize > kFileSizeMask) {

       LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, "

            "truncating to %u", kFileSizeMask));

       aFileSize = kFileSizeMask;

     }

     mRec->mFlags &= ~kFileSizeMask;

     mRec->mFlags |= aFileSize;

   }

   // Returns filesize in kilobytes.

   uint32_t GetFileSize() { return mRec->mFlags & kFileSizeMask; }

   bool     IsFileEmpty() { return GetFileSize() == 0; }

   void WriteToBuf(void *aBuf)

   {

     CacheIndexRecord *dst = reinterpret_cast<CacheIndexRecord *>(aBuf);

     // Copy the whole record to the buffer.

     memcpy(aBuf, mRec, sizeof(CacheIndexRecord));

     // Dirty and fresh flags should never go to disk, since they make sense only

     // during current session.

     dst->mFlags &= ~kDirtyMask;

     dst->mFlags &= ~kFreshMask;

 #if defined(IS_LITTLE_ENDIAN)

     // Data in the buffer are in machine byte order and we want them in network

     // byte order.

     NetworkEndian::writeUint32(&dst->mFrecency, dst->mFrecency);

     NetworkEndian::writeUint32(&dst->mExpirationTime, dst->mExpirationTime);

     NetworkEndian::writeUint32(&dst->mAppId, dst->mAppId);

     NetworkEndian::writeUint32(&dst->mFlags, dst->mFlags);

 #endif

   }

   void ReadFromBuf(void *aBuf)

   {

     CacheIndexRecord *src= reinterpret_cast<CacheIndexRecord *>(aBuf);

     MOZ_ASSERT(memcmp(&mRec->mHash, &src->mHash,

                sizeof(SHA1Sum::Hash)) == 0);

     mRec->mFrecency = NetworkEndian::readUint32(&src->mFrecency);

     mRec->mExpirationTime = NetworkEndian::readUint32(&src->mExpirationTime);

     mRec->mAppId = NetworkEndian::readUint32(&src->mAppId);

     mRec->mFlags = NetworkEndian::readUint32(&src->mFlags);

   }

   void Log() {

     LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u,"

          " initialized=%u, removed=%u, dirty=%u, anonymous=%u, inBrowser=%u, "

          "appId=%u, frecency=%u, expirationTime=%u, size=%u]",

          this, LOGSHA1(mRec->mHash), IsFresh(), IsInitialized(), IsRemoved(),

          IsDirty(), Anonymous(), InBrowser(), AppId(), GetFrecency(),

          GetExpirationTime(), GetFileSize()));

   }

   static bool RecordMatchesLoadContextInfo(CacheIndexRecord *aRec,

                                            nsILoadContextInfo *aInfo)

   {

     if (!aInfo->IsPrivate() &&

         aInfo->AppId() == aRec->mAppId &&

         aInfo->IsAnonymous() == !!(aRec->mFlags & kAnonymousMask) &&

         aInfo->IsInBrowserElement() == !!(aRec->mFlags & kInBrowserMask)) {

       return true;

     }

     return false;

   }

   // Memory reporting

   size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const

   {

     return mallocSizeOf(mRec.get());

   }

   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const

   {

     return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);

   }

 private:

   friend class CacheIndex;

   friend class CacheIndexEntryAutoManage;

   static const uint32_t kInitializedMask = 0x80000000;

   static const uint32_t kAnonymousMask   = 0x40000000;

   static const uint32_t kInBrowserMask   = 0x20000000;

   // This flag is set when the entry was removed. We need to keep this

   // information in memory until we write the index file.

   static const uint32_t kRemovedMask     = 0x10000000;

   // This flag is set when the information in memory is not in sync with the

   // information in index file on disk.

   static const uint32_t kDirtyMask       = 0x08000000;

   // This flag is set when the information about the entry is fresh, i.e.

   // we've created or opened this entry during this session, or we've seen

   // this entry during update or build process.

   static const uint32_t kFreshMask       = 0x04000000;

   static const uint32_t kReservedMask    = 0x03000000;

   // FileSize in kilobytes

   static const uint32_t kFileSizeMask    = 0x00FFFFFF;

   nsAutoPtr<CacheIndexRecord> mRec;

 };

 class CacheIndexStats

 {

 public:

   CacheIndexStats()

     : mCount(0)

     , mNotInitialized(0)

     , mRemoved(0)

     , mDirty(0)

     , mFresh(0)

     , mEmpty(0)

     , mSize(0)

 #ifdef DEBUG

     , mStateLogged(false)

     , mDisableLogging(false)

 #endif

   {

   }

   bool operator==(const CacheIndexStats& aOther) const

   {

     return

 #ifdef DEBUG

            aOther.mStateLogged == mStateLogged &&

 #endif

            aOther.mCount == mCount &&

            aOther.mNotInitialized == mNotInitialized &&

            aOther.mRemoved == mRemoved &&

            aOther.mDirty == mDirty &&

            aOther.mFresh == mFresh &&

            aOther.mEmpty == mEmpty &&

            aOther.mSize == mSize;

   }

 #ifdef DEBUG

   void DisableLogging() {

     mDisableLogging = true;

   }

 #endif

   void Log() {

     LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, "

          "dirty=%u, fresh=%u, empty=%u, size=%u]", mCount, mNotInitialized,

          mRemoved, mDirty, mFresh, mEmpty, mSize));

   }

   void Clear() {

     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!");

     mCount = 0;

     mNotInitialized = 0;

     mRemoved = 0;

     mDirty = 0;

     mFresh = 0;

     mEmpty = 0;

     mSize = 0;

   }

 #ifdef DEBUG

   bool StateLogged() {

     return mStateLogged;

   }

 #endif

   uint32_t Count() {

     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!");

     return mCount;

   }

   uint32_t Dirty() {

     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!");

     return mDirty;

   }

   uint32_t Fresh() {

     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!");

     return mFresh;

   }

   uint32_t ActiveEntriesCount() {

     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::ActiveEntriesCount() - state "

                "logged!");

     return mCount - mRemoved - mNotInitialized - mEmpty;

   }

   uint32_t Size() {

     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!");

     return mSize;

   }

   void BeforeChange(CacheIndexEntry *aEntry) {

 #ifdef DEBUG_STATS

     if (!mDisableLogging) {

       LOG(("CacheIndexStats::BeforeChange()"));

       Log();

     }

 #endif

     MOZ_ASSERT(!mStateLogged, "CacheIndexStats::BeforeChange() - state "

                "logged!");

 #ifdef DEBUG

     mStateLogged = true;

 #endif

     if (aEntry) {

       MOZ_ASSERT(mCount);

       mCount--;

       if (aEntry->IsDirty()) {

         MOZ_ASSERT(mDirty);

         mDirty--;

       }

       if (aEntry->IsFresh()) {

         MOZ_ASSERT(mFresh);

         mFresh--;

       }

       if (aEntry->IsRemoved()) {

         MOZ_ASSERT(mRemoved);

         mRemoved--;

       } else {

         if (!aEntry->IsInitialized()) {

           MOZ_ASSERT(mNotInitialized);

           mNotInitialized--;

         } else {

           if (aEntry->IsFileEmpty()) {

             MOZ_ASSERT(mEmpty);

             mEmpty--;

           } else {

             MOZ_ASSERT(mSize >= aEntry->GetFileSize());

             mSize -= aEntry->GetFileSize();

           }

         }

       }

     }

   }

   void AfterChange(CacheIndexEntry *aEntry) {

     MOZ_ASSERT(mStateLogged, "CacheIndexStats::AfterChange() - state not "

                "logged!");

 #ifdef DEBUG

     mStateLogged = false;

 #endif

     if (aEntry) {

       ++mCount;

       if (aEntry->IsDirty()) {

         mDirty++;

       }

       if (aEntry->IsFresh()) {

         mFresh++;

       }

       if (aEntry->IsRemoved()) {

         mRemoved++;

       } else {

         if (!aEntry->IsInitialized()) {

           mNotInitialized++;

         } else {

           if (aEntry->IsFileEmpty()) {

             mEmpty++;

           } else {

             mSize += aEntry->GetFileSize();

           }

         }

       }

     }

 #ifdef DEBUG_STATS

     if (!mDisableLogging) {

       LOG(("CacheIndexStats::AfterChange()"));

       Log();

     }

 #endif

   }

 private:

   uint32_t mCount;

   uint32_t mNotInitialized;

   uint32_t mRemoved;

   uint32_t mDirty;

   uint32_t mFresh;

   uint32_t mEmpty;

   uint32_t mSize;

 #ifdef DEBUG

   // We completely remove the data about an entry from the stats in

   // BeforeChange() and set this flag to true. The entry is then modified,

   // deleted or created and the data is again put into the stats and this flag

   // set to false. Statistics must not be read during this time since the

   // information is not correct.

   bool     mStateLogged;

   // Disables logging in this instance of CacheIndexStats

   bool     mDisableLogging;

 #endif

 };

 class CacheIndex : public CacheFileIOListener

                  , public nsIRunnable

 {

 public:

   NS_DECL_THREADSAFE_ISUPPORTS

   NS_DECL_NSIRUNNABLE

   CacheIndex();

   static nsresult Init(nsIFile *aCacheDirectory);

   static nsresult PreShutdown();

   static nsresult Shutdown();

   // Following methods can be called only on IO thread.

   // Add entry to the index. The entry shouldn't be present in index. This

   // method is called whenever a new handle for a new entry file is created. The

   // newly created entry is not initialized and it must be either initialized

   // with InitEntry() or removed with RemoveEntry().

   static nsresult AddEntry(const SHA1Sum::Hash *aHash);

   // Inform index about an existing entry that should be present in index. This

   // method is called whenever a new handle for an existing entry file is

   // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry()

   // must be called on the entry, since the entry is not initizlized if the

   // index is outdated.

   static nsresult EnsureEntryExists(const SHA1Sum::Hash *aHash);

   // Initialize the entry. It MUST be present in index. Call to AddEntry() or

   // EnsureEntryExists() must precede the call to this method.

   static nsresult InitEntry(const SHA1Sum::Hash *aHash,

                             uint32_t             aAppId,

                             bool                 aAnonymous,

                             bool                 aInBrowser);

   // Remove entry from index. The entry should be present in index.

   static nsresult RemoveEntry(const SHA1Sum::Hash *aHash);

   // Update some information in entry. The entry MUST be present in index and

   // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to

   // InitEntry() must precede the call to this method.

   // Pass nullptr if the value didn't change.

   static nsresult UpdateEntry(const SHA1Sum::Hash *aHash,

                               const uint32_t      *aFrecency,

                               const uint32_t      *aExpirationTime,

                               const uint32_t      *aSize);

   // Remove all entries from the index. Called when clearing the whole cache.

   static nsresult RemoveAll();

   enum EntryStatus {

     EXISTS         = 0,

     DOES_NOT_EXIST = 1,

     DO_NOT_KNOW    = 2

   };

   // Returns status of the entry in index for the given key. It can be called

   // on any thread.

   static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval);

   // Returns a hash of the least important entry that should be evicted if the

   // cache size is over limit and also returns a total number of all entries in

   // the index.

   static nsresult GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt);

   // Returns cache size in kB.

   static nsresult GetCacheSize(uint32_t *_retval);

   // Asynchronously gets the disk cache size, used for display in the UI.

   static nsresult AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver);

   // Returns an iterator that returns entries matching a given context that were

   // present in the index at the time this method was called. If aAddNew is true

   // then the iterator will also return entries created after this call.

   // NOTE: When some entry is removed from index it is removed also from the

   // iterator regardless what aAddNew was passed.

   static nsresult GetIterator(nsILoadContextInfo *aInfo, bool aAddNew,

                               CacheIndexIterator **_retval);

   // Returns true if we _think_ that the index is up to date. I.e. the state is

   // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false.

   static nsresult IsUpToDate(bool *_retval);

   // Memory reporting

   static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);

   static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);

 private:

   friend class CacheIndexEntryAutoManage;

   friend class CacheIndexAutoLock;

   friend class CacheIndexAutoUnlock;

   friend class FileOpenHelper;

   friend class CacheIndexIterator;

   virtual ~CacheIndex();

   NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult);

   nsresult   OnFileOpenedInternal(FileOpenHelper *aOpener,

                                   CacheFileHandle *aHandle, nsresult aResult);

   NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,

                            nsresult aResult);

   NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult);

   NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);

   NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult);

   NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult);

   void     Lock();

   void     Unlock();

   void     AssertOwnsLock();

   nsresult InitInternal(nsIFile *aCacheDirectory);

   void     PreShutdownInternal();

   // This method returns false when index is not initialized or is shut down.

   bool IsIndexUsable();

   // This method checks whether the entry has the same values of appId,

   // isAnonymous and isInBrowser. We don't expect to find a collision since

   // these values are part of the key that we hash and we use a strong hash

   // function.

   static bool IsCollision(CacheIndexEntry *aEntry,

                           uint32_t         aAppId,

                           bool             aAnonymous,

                           bool             aInBrowser);

   // Checks whether any of the information about the entry has changed.

   static bool HasEntryChanged(CacheIndexEntry *aEntry,

                               const uint32_t  *aFrecency,

                               const uint32_t  *aExpirationTime,

                               const uint32_t  *aSize);

   // Merge all pending operations from mPendingUpdates into mIndex.

   void ProcessPendingOperations();

   static PLDHashOperator UpdateEntryInIndex(CacheIndexEntry *aEntry,

                                             void* aClosure);

   // Following methods perform writing of the index file.

   //

   // The index is written periodically, but not earlier than once in

   // kMinDumpInterval and there must be at least kMinUnwrittenChanges

   // differences between index on disk and in memory. Index is always first

   // written to a temporary file and the old index file is replaced when the

   // writing process succeeds.

   //

   // Starts writing of index when both limits (minimal delay between writes and

   // minimum number of changes in index) were exceeded.

   bool WriteIndexToDiskIfNeeded();

   // Starts writing of index file.

   void WriteIndexToDisk();

   // Serializes part of mIndex hashtable to the write buffer a writes the buffer

   // to the file.

   void WriteRecords();

   // Finalizes writing process.

   void FinishWrite(bool aSucceeded);

   static PLDHashOperator CopyRecordsToRWBuf(CacheIndexEntry *aEntry,

                                             void* aClosure);

   static PLDHashOperator ApplyIndexChanges(CacheIndexEntry *aEntry,

                                            void* aClosure);

   // Following methods perform writing of the journal during shutdown. All these

   // methods must be called only during shutdown since they write/delete files

   // directly on the main thread instead of using CacheFileIOManager that does

   // it asynchronously on IO thread. Journal contains only entries that are

   // dirty, i.e. changes that are not present in the index file on the disk.

   // When the log is written successfully, the dirty flag in index file is

   // cleared.

   nsresult GetFile(const nsACString &aName, nsIFile **_retval);

   nsresult RemoveFile(const nsACString &aName);

   void     RemoveIndexFromDisk();

   // Writes journal to the disk and clears dirty flag in index header.

   nsresult WriteLogToDisk();

   static PLDHashOperator WriteEntryToLog(CacheIndexEntry *aEntry,

                                          void* aClosure);

   // Following methods perform reading of the index from the disk.

   //

   // Index is read at startup just after initializing the CacheIndex. There are

   // 3 files used when manipulating with index: index file, journal file and

   // a temporary file. All files contain the hash of the data, so we can check

   // whether the content is valid and complete. Index file contains also a dirty

   // flag in the index header which is unset on a clean shutdown. During opening

   // and reading of the files we determine the status of the whole index from

   // the states of the separate files. Following table shows all possible

   // combinations:

   //

   // index, journal, tmpfile

   // M      *        *       - index is missing    -> BUILD

   // I      *        *       - index is invalid    -> BUILD

   // D      *        *       - index is dirty      -> UPDATE

   // C      M        *       - index is dirty      -> UPDATE

   // C      I        *       - unexpected state    -> UPDATE

   // C      V        E       - unexpected state    -> UPDATE

   // C      V        M       - index is up to date -> READY

   //

   // where the letters mean:

   //   * - any state

   //   E - file exists

   //   M - file is missing

   //   I - data is invalid (parsing failed or hash didn't match)

   //   D - dirty (data in index file is correct, but dirty flag is set)

   //   C - clean (index file is clean)

   //   V - valid (data in journal file is correct)

   //

   // Note: We accept the data from journal only when the index is up to date as

   // a whole (i.e. C,V,M state).

   //

   // We rename the journal file to the temporary file as soon as possible after

   // initial test to ensure that we start update process on the next startup if

   // FF crashes during parsing of the index.

   //

   // Initiates reading index from disk.

   void ReadIndexFromDisk();

   // Starts reading data from index file.

   void StartReadingIndex();

   // Parses data read from index file.

   void ParseRecords();

   // Starts reading data from journal file.

   void StartReadingJournal();

   // Parses data read from journal file.

   void ParseJournal();

   // Merges entries from journal into mIndex.

   void MergeJournal();

   // In debug build this method checks that we have no fresh entry in mIndex

   // after we finish reading index and before we process pending operations.

   void EnsureNoFreshEntry();

   // In debug build this method is called after processing pending operations

   // to make sure mIndexStats contains correct information.

   void EnsureCorrectStats();

   static PLDHashOperator SumIndexStats(CacheIndexEntry *aEntry, void* aClosure);

   // Finalizes reading process.

   void FinishRead(bool aSucceeded);

   static PLDHashOperator ProcessJournalEntry(CacheIndexEntry *aEntry,

                                              void* aClosure);

   // Following methods perform updating and building of the index.

   // Timer callback that starts update or build process.

   static void DelayedUpdate(nsITimer *aTimer, void *aClosure);

   // Posts timer event that start update or build process.

   nsresult ScheduleUpdateTimer(uint32_t aDelay);

   nsresult SetupDirectoryEnumerator();

   void InitEntryFromDiskData(CacheIndexEntry *aEntry,

                              CacheFileMetadata *aMetaData,

                              int64_t aFileSize);

   // Returns true when either a timer is scheduled or event is posted.

   bool IsUpdatePending();

   // Iterates through all files in entries directory that we didn't create/open

   // during this session, parses them and adds the entries to the index.

   void BuildIndex();

   bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState = false);

   // Starts update or build process or fires a timer when it is too early after

   // startup.

   void StartUpdatingIndex(bool aRebuild);

   // Iterates through all files in entries directory that we didn't create/open

   // during this session and theirs last modified time is newer than timestamp

   // in the index header. Parses the files and adds the entries to the index.

   void UpdateIndex();

   // Finalizes update or build process.

   void FinishUpdate(bool aSucceeded);

   static PLDHashOperator RemoveNonFreshEntries(CacheIndexEntry *aEntry,

                                                void* aClosure);

   enum EState {

     // Initial state in which the index is not usable

     // Possible transitions:

     //  -> READING

     INITIAL  = 0,

     // Index is being read from the disk.

     // Possible transitions:

     //  -> INITIAL  - We failed to dispatch a read event.

     //  -> BUILDING - No or corrupted index file was found.

     //  -> UPDATING - No or corrupted journal file was found.

     //              - Dirty flag was set in index header.

     //  -> READY    - Index was read successfully or was interrupted by

     //                pre-shutdown.

     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.

     READING  = 1,

     // Index is being written to the disk.

     // Possible transitions:

     //  -> READY    - Writing of index finished or was interrupted by

     //                pre-shutdown..

     //  -> UPDATING - Writing of index finished, but index was found outdated

     //                during writing.

     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.

     WRITING  = 2,

     // Index is being build.

     // Possible transitions:

     //  -> READY    - Building of index finished or was interrupted by

     //                pre-shutdown.

     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.

     BUILDING = 3,

     // Index is being updated.

     // Possible transitions:

     //  -> READY    - Updating of index finished or was interrupted by

     //                pre-shutdown.

     //  -> SHUTDOWN - This could happen only in case of pre-shutdown failure.

     UPDATING = 4,

     // Index is ready.

     // Possible transitions:

     //  -> UPDATING - Index was found outdated.

     //  -> SHUTDOWN - Index is shutting down.

     READY    = 5,

     // Index is shutting down.

     SHUTDOWN = 6

   };

 #ifdef PR_LOGGING

   static char const * StateString(EState aState);

 #endif

   void ChangeState(EState aNewState);

   // Allocates and releases buffer used for reading and writing index.

   void AllocBuffer();

   void ReleaseBuffer();

   // Methods used by CacheIndexEntryAutoManage to keep the arrays up to date.

   void InsertRecordToFrecencyArray(CacheIndexRecord *aRecord);

   void InsertRecordToExpirationArray(CacheIndexRecord *aRecord);

   void RemoveRecordFromFrecencyArray(CacheIndexRecord *aRecord);

   void RemoveRecordFromExpirationArray(CacheIndexRecord *aRecord);

   // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date.

   void AddRecordToIterators(CacheIndexRecord *aRecord);

   void RemoveRecordFromIterators(CacheIndexRecord *aRecord);

   void ReplaceRecordInIterators(CacheIndexRecord *aOldRecord,

                                 CacheIndexRecord *aNewRecord);

   // Memory reporting (private part)

   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;

   static CacheIndex *gInstance;

   nsCOMPtr<nsIFile> mCacheDirectory;

   mozilla::Mutex mLock;

   EState         mState;

   // Timestamp of time when the index was initialized. We use it to delay

   // initial update or build of index.

   TimeStamp      mStartTime;

   // Set to true in PreShutdown(), it is checked on variaous places to prevent

   // starting any process (write, update, etc.) during shutdown.

   bool           mShuttingDown;

   // When set to true, update process should start as soon as possible. This

   // flag is set whenever we find some inconsistency which would be fixed by

   // update process. The flag is checked always when switching to READY state.

   // To make sure we start the update process as soon as possible, methods that

   // set this flag should also call StartUpdatingIndexIfNeeded() to cover the

   // case when we are currently in READY state.

   bool           mIndexNeedsUpdate;

   // Set at the beginning of RemoveAll() which clears the whole index. When

   // removing all entries we must stop any pending reading, writing, updating or

   // building operation. This flag is checked at various places and it prevents

   // we won't start another operation (e.g. canceling reading of the index would

   // normally start update or build process)

   bool           mRemovingAll;

   // Whether the index file on disk exists and is valid.

   bool           mIndexOnDiskIsValid;

   // When something goes wrong during updating or building process, we don't

   // mark index clean (and also don't write journal) to ensure that update or

   // build will be initiated on the next start.

   bool           mDontMarkIndexClean;

   // Timestamp value from index file. It is used during update process to skip

   // entries that were last modified before this timestamp.

   uint32_t       mIndexTimeStamp;

   // Timestamp of last time the index was dumped to disk.

   // NOTE: The index might not be necessarily dumped at this time. The value

   // is used to schedule next dump of the index.

   TimeStamp      mLastDumpTime;

   // Timer of delayed update/build.

   nsCOMPtr<nsITimer> mUpdateTimer;

   // True when build or update event is posted

   bool               mUpdateEventPending;

   // Helper members used when reading/writing index from/to disk.

   // Contains number of entries that should be skipped:

   //  - in hashtable when writing index because they were already written

   //  - in index file when reading index because they were already read

   uint32_t                  mSkipEntries;

   // Number of entries that should be written to disk. This is number of entries

   // in hashtable that are initialized and are not marked as removed when writing

   // begins.

   uint32_t                  mProcessEntries;

   char                     *mRWBuf;

   uint32_t                  mRWBufSize;

   uint32_t                  mRWBufPos;

   nsRefPtr<CacheHash>       mRWHash;

   // Reading of journal succeeded if true.

   bool                      mJournalReadSuccessfully;

   // Handle used for writing and reading index file.

   nsRefPtr<CacheFileHandle> mIndexHandle;

   // Handle used for reading journal file.

   nsRefPtr<CacheFileHandle> mJournalHandle;

   // Used to check the existence of the file during reading process.

   nsRefPtr<CacheFileHandle> mTmpHandle;

   nsRefPtr<FileOpenHelper>  mIndexFileOpener;

   nsRefPtr<FileOpenHelper>  mJournalFileOpener;

   nsRefPtr<FileOpenHelper>  mTmpFileOpener;

   // Directory enumerator used when building and updating index.

   nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator;

   // Main index hashtable.

   nsTHashtable<CacheIndexEntry> mIndex;

   // We cannot add, remove or change any entry in mIndex in states READING and

   // WRITING. We track all changes in mPendingUpdates during these states.

   nsTHashtable<CacheIndexEntry> mPendingUpdates;

   // Contains information statistics for mIndex + mPendingUpdates.

   CacheIndexStats               mIndexStats;

   // When reading journal, we must first parse the whole file and apply the

   // changes iff the journal was read successfully. mTmpJournal is used to store

   // entries from the journal file. We throw away all these entries if parsing

   // of the journal fails or the hash does not match.

   nsTHashtable<CacheIndexEntry> mTmpJournal;

   // Arrays that keep entry records ordered by eviction preference. When looking

   // for an entry to evict, we first try to find an expired entry. If there is

   // no expired entry, we take the entry with lowest valid frecency. Zero

   // frecency is an initial value and such entries are stored at the end of the

   // array. Uninitialized entries and entries marked as deleted are not present

   // in these arrays.

   nsTArray<CacheIndexRecord *>  mFrecencyArray;

   nsTArray<CacheIndexRecord *>  mExpirationArray;

   nsTArray<CacheIndexIterator *> mIterators;

   class DiskConsumptionObserver : public nsRunnable

   {

   public:

     static DiskConsumptionObserver* Init(nsICacheStorageConsumptionObserver* aObserver)

     {

       nsWeakPtr observer = do_GetWeakReference(aObserver);

       if (!observer)

         return nullptr;

       return new DiskConsumptionObserver(observer);

     }

     void OnDiskConsumption(int64_t aSize)

     {

       mSize = aSize;

       NS_DispatchToMainThread(this);

     }

   private:

     DiskConsumptionObserver(nsWeakPtr const &aWeakObserver)

       : mObserver(aWeakObserver) { }

     virtual ~DiskConsumptionObserver() { }

     NS_IMETHODIMP Run()

     {

       MOZ_ASSERT(NS_IsMainThread());

       nsCOMPtr<nsICacheStorageConsumptionObserver> observer =

         do_QueryReferent(mObserver);

       if (observer) {

         observer->OnNetworkCacheDiskConsumption(mSize);

       }

       return NS_OK;

     }

     nsWeakPtr mObserver;

     int64_t mSize;

   };

   // List of async observers that want to get disk consumption information

   nsTArray<nsRefPtr<DiskConsumptionObserver> > mDiskConsumptionObservers;

 };

 class CacheIndexAutoLock {

 public:

   CacheIndexAutoLock(CacheIndex *aIndex)

     : mIndex(aIndex)

     , mLocked(true)

   {

     mIndex->Lock();

   }

   ~CacheIndexAutoLock()

   {

     if (mLocked) {

       mIndex->Unlock();

     }

   }

   void Lock()

   {

     MOZ_ASSERT(!mLocked);

     mIndex->Lock();

     mLocked = true;

   }

   void Unlock()

   {

     MOZ_ASSERT(mLocked);

     mIndex->Unlock();

     mLocked = false;

   }

 private:

   nsRefPtr<CacheIndex> mIndex;

   bool mLocked;

 };

 class CacheIndexAutoUnlock {

 public:

   CacheIndexAutoUnlock(CacheIndex *aIndex)

     : mIndex(aIndex)

     , mLocked(false)

   {

     mIndex->Unlock();

   }

   ~CacheIndexAutoUnlock()

   {

     if (!mLocked) {

       mIndex->Lock();

     }

   }

   void Lock()

   {

     MOZ_ASSERT(!mLocked);

     mIndex->Lock();

     mLocked = true;

   }

   void Unlock()

   {

     MOZ_ASSERT(mLocked);

     mIndex->Unlock();

     mLocked = false;

   }

 private:

   nsRefPtr<CacheIndex> mIndex;

   bool mLocked;

 };

 } // net

 } // mozilla

 #endif