michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: #ifndef CacheIndex__h__ michael@0: #define CacheIndex__h__ michael@0: michael@0: #include "CacheLog.h" michael@0: #include "CacheFileIOManager.h" michael@0: #include "nsIRunnable.h" michael@0: #include "CacheHashUtils.h" michael@0: #include "nsICacheStorageService.h" michael@0: #include "nsICacheEntry.h" michael@0: #include "nsILoadContextInfo.h" michael@0: #include "nsTHashtable.h" michael@0: #include "nsThreadUtils.h" michael@0: #include "nsWeakReference.h" michael@0: #include "mozilla/SHA1.h" michael@0: #include "mozilla/Mutex.h" michael@0: #include "mozilla/Endian.h" michael@0: #include "mozilla/TimeStamp.h" michael@0: michael@0: class nsIFile; michael@0: class nsIDirectoryEnumerator; michael@0: class nsITimer; michael@0: michael@0: michael@0: #ifdef DEBUG michael@0: #define DEBUG_STATS 1 michael@0: #endif michael@0: michael@0: namespace mozilla { michael@0: namespace net { michael@0: michael@0: class CacheFileMetadata; michael@0: class FileOpenHelper; michael@0: class CacheIndexIterator; michael@0: michael@0: typedef struct { michael@0: // Version of the index. The index must be ignored and deleted when the file michael@0: // on disk was written with a newer version. michael@0: uint32_t mVersion; michael@0: michael@0: // Timestamp of time when the last successful write of the index started. michael@0: // During update process we use this timestamp for a quick validation of entry michael@0: // files. If last modified time of the file is lower than this timestamp, we michael@0: // skip parsing of such file since the information in index should be up to michael@0: // date. michael@0: uint32_t mTimeStamp; michael@0: michael@0: // We set this flag as soon as possible after parsing index during startup michael@0: // and clean it after we write journal to disk during shutdown. We ignore the michael@0: // journal and start update process whenever this flag is set during index michael@0: // parsing. michael@0: uint32_t mIsDirty; michael@0: } CacheIndexHeader; michael@0: michael@0: struct CacheIndexRecord { michael@0: SHA1Sum::Hash mHash; michael@0: uint32_t mFrecency; michael@0: uint32_t mExpirationTime; michael@0: uint32_t mAppId; michael@0: michael@0: /* michael@0: * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized michael@0: * 0100 0000 0000 0000 0000 0000 0000 0000 : anonymous michael@0: * 0010 0000 0000 0000 0000 0000 0000 0000 : inBrowser michael@0: * 0001 0000 0000 0000 0000 0000 0000 0000 : removed michael@0: * 0000 1000 0000 0000 0000 0000 0000 0000 : dirty michael@0: * 0000 0100 0000 0000 0000 0000 0000 0000 : fresh michael@0: * 0000 0011 0000 0000 0000 0000 0000 0000 : reserved michael@0: * 0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB) michael@0: */ michael@0: uint32_t mFlags; michael@0: michael@0: CacheIndexRecord() michael@0: : mFrecency(0) michael@0: , mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME) michael@0: , mAppId(nsILoadContextInfo::NO_APP_ID) michael@0: , mFlags(0) michael@0: {} michael@0: }; michael@0: michael@0: class CacheIndexEntry : public PLDHashEntryHdr michael@0: { michael@0: public: michael@0: typedef const SHA1Sum::Hash& KeyType; michael@0: typedef const SHA1Sum::Hash* KeyTypePointer; michael@0: michael@0: CacheIndexEntry(KeyTypePointer aKey) michael@0: { michael@0: MOZ_COUNT_CTOR(CacheIndexEntry); michael@0: mRec = new CacheIndexRecord(); michael@0: LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec.get())); michael@0: memcpy(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)); michael@0: } michael@0: CacheIndexEntry(const CacheIndexEntry& aOther) michael@0: { michael@0: NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!"); michael@0: } michael@0: ~CacheIndexEntry() michael@0: { michael@0: MOZ_COUNT_DTOR(CacheIndexEntry); michael@0: LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]", michael@0: mRec.get())); michael@0: } michael@0: michael@0: // KeyEquals(): does this entry match this key? michael@0: bool KeyEquals(KeyTypePointer aKey) const michael@0: { michael@0: return memcmp(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0; michael@0: } michael@0: michael@0: // KeyToPointer(): Convert KeyType to KeyTypePointer michael@0: static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; } michael@0: michael@0: // HashKey(): calculate the hash number michael@0: static PLDHashNumber HashKey(KeyTypePointer aKey) michael@0: { michael@0: return (reinterpret_cast(aKey))[0]; michael@0: } michael@0: michael@0: // ALLOW_MEMMOVE can we move this class with memmove(), or do we have michael@0: // to use the copy constructor? michael@0: enum { ALLOW_MEMMOVE = true }; michael@0: michael@0: bool operator==(const CacheIndexEntry& aOther) const michael@0: { michael@0: return KeyEquals(&aOther.mRec->mHash); michael@0: } michael@0: michael@0: CacheIndexEntry& operator=(const CacheIndexEntry& aOther) michael@0: { michael@0: MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash, michael@0: sizeof(SHA1Sum::Hash)) == 0); michael@0: mRec->mFrecency = aOther.mRec->mFrecency; michael@0: mRec->mExpirationTime = aOther.mRec->mExpirationTime; michael@0: mRec->mAppId = aOther.mRec->mAppId; michael@0: mRec->mFlags = aOther.mRec->mFlags; michael@0: return *this; michael@0: } michael@0: michael@0: void InitNew() michael@0: { michael@0: mRec->mFrecency = 0; michael@0: mRec->mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME; michael@0: mRec->mAppId = nsILoadContextInfo::NO_APP_ID; michael@0: mRec->mFlags = 0; michael@0: } michael@0: michael@0: void Init(uint32_t aAppId, bool aAnonymous, bool aInBrowser) michael@0: { michael@0: MOZ_ASSERT(mRec->mFrecency == 0); michael@0: MOZ_ASSERT(mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME); michael@0: MOZ_ASSERT(mRec->mAppId == nsILoadContextInfo::NO_APP_ID); michael@0: // When we init the entry it must be fresh and may be dirty michael@0: MOZ_ASSERT((mRec->mFlags & ~kDirtyMask) == kFreshMask); michael@0: michael@0: mRec->mAppId = aAppId; michael@0: mRec->mFlags |= kInitializedMask; michael@0: if (aAnonymous) { michael@0: mRec->mFlags |= kAnonymousMask; michael@0: } michael@0: if (aInBrowser) { michael@0: mRec->mFlags |= kInBrowserMask; michael@0: } michael@0: } michael@0: michael@0: const SHA1Sum::Hash * Hash() { return &mRec->mHash; } michael@0: michael@0: bool IsInitialized() { return !!(mRec->mFlags & kInitializedMask); } michael@0: michael@0: uint32_t AppId() { return mRec->mAppId; } michael@0: bool Anonymous() { return !!(mRec->mFlags & kAnonymousMask); } michael@0: bool InBrowser() { return !!(mRec->mFlags & kInBrowserMask); } michael@0: michael@0: bool IsRemoved() { return !!(mRec->mFlags & kRemovedMask); } michael@0: void MarkRemoved() { mRec->mFlags |= kRemovedMask; } michael@0: michael@0: bool IsDirty() { return !!(mRec->mFlags & kDirtyMask); } michael@0: void MarkDirty() { mRec->mFlags |= kDirtyMask; } michael@0: void ClearDirty() { mRec->mFlags &= ~kDirtyMask; } michael@0: michael@0: bool IsFresh() { return !!(mRec->mFlags & kFreshMask); } michael@0: void MarkFresh() { mRec->mFlags |= kFreshMask; } michael@0: michael@0: void SetFrecency(uint32_t aFrecency) { mRec->mFrecency = aFrecency; } michael@0: uint32_t GetFrecency() { return mRec->mFrecency; } michael@0: michael@0: void SetExpirationTime(uint32_t aExpirationTime) michael@0: { michael@0: mRec->mExpirationTime = aExpirationTime; michael@0: } michael@0: uint32_t GetExpirationTime() { return mRec->mExpirationTime; } michael@0: michael@0: // Sets filesize in kilobytes. michael@0: void SetFileSize(uint32_t aFileSize) michael@0: { michael@0: if (aFileSize > kFileSizeMask) { michael@0: LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, " michael@0: "truncating to %u", kFileSizeMask)); michael@0: aFileSize = kFileSizeMask; michael@0: } michael@0: mRec->mFlags &= ~kFileSizeMask; michael@0: mRec->mFlags |= aFileSize; michael@0: } michael@0: // Returns filesize in kilobytes. michael@0: uint32_t GetFileSize() { return mRec->mFlags & kFileSizeMask; } michael@0: bool IsFileEmpty() { return GetFileSize() == 0; } michael@0: michael@0: void WriteToBuf(void *aBuf) michael@0: { michael@0: CacheIndexRecord *dst = reinterpret_cast(aBuf); michael@0: michael@0: // Copy the whole record to the buffer. michael@0: memcpy(aBuf, mRec, sizeof(CacheIndexRecord)); michael@0: michael@0: // Dirty and fresh flags should never go to disk, since they make sense only michael@0: // during current session. michael@0: dst->mFlags &= ~kDirtyMask; michael@0: dst->mFlags &= ~kFreshMask; michael@0: michael@0: #if defined(IS_LITTLE_ENDIAN) michael@0: // Data in the buffer are in machine byte order and we want them in network michael@0: // byte order. michael@0: NetworkEndian::writeUint32(&dst->mFrecency, dst->mFrecency); michael@0: NetworkEndian::writeUint32(&dst->mExpirationTime, dst->mExpirationTime); michael@0: NetworkEndian::writeUint32(&dst->mAppId, dst->mAppId); michael@0: NetworkEndian::writeUint32(&dst->mFlags, dst->mFlags); michael@0: #endif michael@0: } michael@0: michael@0: void ReadFromBuf(void *aBuf) michael@0: { michael@0: CacheIndexRecord *src= reinterpret_cast(aBuf); michael@0: MOZ_ASSERT(memcmp(&mRec->mHash, &src->mHash, michael@0: sizeof(SHA1Sum::Hash)) == 0); michael@0: michael@0: mRec->mFrecency = NetworkEndian::readUint32(&src->mFrecency); michael@0: mRec->mExpirationTime = NetworkEndian::readUint32(&src->mExpirationTime); michael@0: mRec->mAppId = NetworkEndian::readUint32(&src->mAppId); michael@0: mRec->mFlags = NetworkEndian::readUint32(&src->mFlags); michael@0: } michael@0: michael@0: void Log() { michael@0: LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u," michael@0: " initialized=%u, removed=%u, dirty=%u, anonymous=%u, inBrowser=%u, " michael@0: "appId=%u, frecency=%u, expirationTime=%u, size=%u]", michael@0: this, LOGSHA1(mRec->mHash), IsFresh(), IsInitialized(), IsRemoved(), michael@0: IsDirty(), Anonymous(), InBrowser(), AppId(), GetFrecency(), michael@0: GetExpirationTime(), GetFileSize())); michael@0: } michael@0: michael@0: static bool RecordMatchesLoadContextInfo(CacheIndexRecord *aRec, michael@0: nsILoadContextInfo *aInfo) michael@0: { michael@0: if (!aInfo->IsPrivate() && michael@0: aInfo->AppId() == aRec->mAppId && michael@0: aInfo->IsAnonymous() == !!(aRec->mFlags & kAnonymousMask) && michael@0: aInfo->IsInBrowserElement() == !!(aRec->mFlags & kInBrowserMask)) { michael@0: return true; michael@0: } michael@0: michael@0: return false; michael@0: } michael@0: michael@0: // Memory reporting michael@0: size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const michael@0: { michael@0: return mallocSizeOf(mRec.get()); michael@0: } michael@0: michael@0: size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const michael@0: { michael@0: return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf); michael@0: } michael@0: michael@0: private: michael@0: friend class CacheIndex; michael@0: friend class CacheIndexEntryAutoManage; michael@0: michael@0: static const uint32_t kInitializedMask = 0x80000000; michael@0: static const uint32_t kAnonymousMask = 0x40000000; michael@0: static const uint32_t kInBrowserMask = 0x20000000; michael@0: michael@0: // This flag is set when the entry was removed. We need to keep this michael@0: // information in memory until we write the index file. michael@0: static const uint32_t kRemovedMask = 0x10000000; michael@0: michael@0: // This flag is set when the information in memory is not in sync with the michael@0: // information in index file on disk. michael@0: static const uint32_t kDirtyMask = 0x08000000; michael@0: michael@0: // This flag is set when the information about the entry is fresh, i.e. michael@0: // we've created or opened this entry during this session, or we've seen michael@0: // this entry during update or build process. michael@0: static const uint32_t kFreshMask = 0x04000000; michael@0: michael@0: static const uint32_t kReservedMask = 0x03000000; michael@0: michael@0: // FileSize in kilobytes michael@0: static const uint32_t kFileSizeMask = 0x00FFFFFF; michael@0: michael@0: nsAutoPtr mRec; michael@0: }; michael@0: michael@0: class CacheIndexStats michael@0: { michael@0: public: michael@0: CacheIndexStats() michael@0: : mCount(0) michael@0: , mNotInitialized(0) michael@0: , mRemoved(0) michael@0: , mDirty(0) michael@0: , mFresh(0) michael@0: , mEmpty(0) michael@0: , mSize(0) michael@0: #ifdef DEBUG michael@0: , mStateLogged(false) michael@0: , mDisableLogging(false) michael@0: #endif michael@0: { michael@0: } michael@0: michael@0: bool operator==(const CacheIndexStats& aOther) const michael@0: { michael@0: return michael@0: #ifdef DEBUG michael@0: aOther.mStateLogged == mStateLogged && michael@0: #endif michael@0: aOther.mCount == mCount && michael@0: aOther.mNotInitialized == mNotInitialized && michael@0: aOther.mRemoved == mRemoved && michael@0: aOther.mDirty == mDirty && michael@0: aOther.mFresh == mFresh && michael@0: aOther.mEmpty == mEmpty && michael@0: aOther.mSize == mSize; michael@0: } michael@0: michael@0: #ifdef DEBUG michael@0: void DisableLogging() { michael@0: mDisableLogging = true; michael@0: } michael@0: #endif michael@0: michael@0: void Log() { michael@0: LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, " michael@0: "dirty=%u, fresh=%u, empty=%u, size=%u]", mCount, mNotInitialized, michael@0: mRemoved, mDirty, mFresh, mEmpty, mSize)); michael@0: } michael@0: michael@0: void Clear() { michael@0: MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!"); michael@0: michael@0: mCount = 0; michael@0: mNotInitialized = 0; michael@0: mRemoved = 0; michael@0: mDirty = 0; michael@0: mFresh = 0; michael@0: mEmpty = 0; michael@0: mSize = 0; michael@0: } michael@0: michael@0: #ifdef DEBUG michael@0: bool StateLogged() { michael@0: return mStateLogged; michael@0: } michael@0: #endif michael@0: michael@0: uint32_t Count() { michael@0: MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!"); michael@0: return mCount; michael@0: } michael@0: michael@0: uint32_t Dirty() { michael@0: MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!"); michael@0: return mDirty; michael@0: } michael@0: michael@0: uint32_t Fresh() { michael@0: MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!"); michael@0: return mFresh; michael@0: } michael@0: michael@0: uint32_t ActiveEntriesCount() { michael@0: MOZ_ASSERT(!mStateLogged, "CacheIndexStats::ActiveEntriesCount() - state " michael@0: "logged!"); michael@0: return mCount - mRemoved - mNotInitialized - mEmpty; michael@0: } michael@0: michael@0: uint32_t Size() { michael@0: MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!"); michael@0: return mSize; michael@0: } michael@0: michael@0: void BeforeChange(CacheIndexEntry *aEntry) { michael@0: #ifdef DEBUG_STATS michael@0: if (!mDisableLogging) { michael@0: LOG(("CacheIndexStats::BeforeChange()")); michael@0: Log(); michael@0: } michael@0: #endif michael@0: michael@0: MOZ_ASSERT(!mStateLogged, "CacheIndexStats::BeforeChange() - state " michael@0: "logged!"); michael@0: #ifdef DEBUG michael@0: mStateLogged = true; michael@0: #endif michael@0: if (aEntry) { michael@0: MOZ_ASSERT(mCount); michael@0: mCount--; michael@0: if (aEntry->IsDirty()) { michael@0: MOZ_ASSERT(mDirty); michael@0: mDirty--; michael@0: } michael@0: if (aEntry->IsFresh()) { michael@0: MOZ_ASSERT(mFresh); michael@0: mFresh--; michael@0: } michael@0: if (aEntry->IsRemoved()) { michael@0: MOZ_ASSERT(mRemoved); michael@0: mRemoved--; michael@0: } else { michael@0: if (!aEntry->IsInitialized()) { michael@0: MOZ_ASSERT(mNotInitialized); michael@0: mNotInitialized--; michael@0: } else { michael@0: if (aEntry->IsFileEmpty()) { michael@0: MOZ_ASSERT(mEmpty); michael@0: mEmpty--; michael@0: } else { michael@0: MOZ_ASSERT(mSize >= aEntry->GetFileSize()); michael@0: mSize -= aEntry->GetFileSize(); michael@0: } michael@0: } michael@0: } michael@0: } michael@0: } michael@0: michael@0: void AfterChange(CacheIndexEntry *aEntry) { michael@0: MOZ_ASSERT(mStateLogged, "CacheIndexStats::AfterChange() - state not " michael@0: "logged!"); michael@0: #ifdef DEBUG michael@0: mStateLogged = false; michael@0: #endif michael@0: if (aEntry) { michael@0: ++mCount; michael@0: if (aEntry->IsDirty()) { michael@0: mDirty++; michael@0: } michael@0: if (aEntry->IsFresh()) { michael@0: mFresh++; michael@0: } michael@0: if (aEntry->IsRemoved()) { michael@0: mRemoved++; michael@0: } else { michael@0: if (!aEntry->IsInitialized()) { michael@0: mNotInitialized++; michael@0: } else { michael@0: if (aEntry->IsFileEmpty()) { michael@0: mEmpty++; michael@0: } else { michael@0: mSize += aEntry->GetFileSize(); michael@0: } michael@0: } michael@0: } michael@0: } michael@0: michael@0: #ifdef DEBUG_STATS michael@0: if (!mDisableLogging) { michael@0: LOG(("CacheIndexStats::AfterChange()")); michael@0: Log(); michael@0: } michael@0: #endif michael@0: } michael@0: michael@0: private: michael@0: uint32_t mCount; michael@0: uint32_t mNotInitialized; michael@0: uint32_t mRemoved; michael@0: uint32_t mDirty; michael@0: uint32_t mFresh; michael@0: uint32_t mEmpty; michael@0: uint32_t mSize; michael@0: #ifdef DEBUG michael@0: // We completely remove the data about an entry from the stats in michael@0: // BeforeChange() and set this flag to true. The entry is then modified, michael@0: // deleted or created and the data is again put into the stats and this flag michael@0: // set to false. Statistics must not be read during this time since the michael@0: // information is not correct. michael@0: bool mStateLogged; michael@0: michael@0: // Disables logging in this instance of CacheIndexStats michael@0: bool mDisableLogging; michael@0: #endif michael@0: }; michael@0: michael@0: class CacheIndex : public CacheFileIOListener michael@0: , public nsIRunnable michael@0: { michael@0: public: michael@0: NS_DECL_THREADSAFE_ISUPPORTS michael@0: NS_DECL_NSIRUNNABLE michael@0: michael@0: CacheIndex(); michael@0: michael@0: static nsresult Init(nsIFile *aCacheDirectory); michael@0: static nsresult PreShutdown(); michael@0: static nsresult Shutdown(); michael@0: michael@0: // Following methods can be called only on IO thread. michael@0: michael@0: // Add entry to the index. The entry shouldn't be present in index. This michael@0: // method is called whenever a new handle for a new entry file is created. The michael@0: // newly created entry is not initialized and it must be either initialized michael@0: // with InitEntry() or removed with RemoveEntry(). michael@0: static nsresult AddEntry(const SHA1Sum::Hash *aHash); michael@0: michael@0: // Inform index about an existing entry that should be present in index. This michael@0: // method is called whenever a new handle for an existing entry file is michael@0: // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry() michael@0: // must be called on the entry, since the entry is not initizlized if the michael@0: // index is outdated. michael@0: static nsresult EnsureEntryExists(const SHA1Sum::Hash *aHash); michael@0: michael@0: // Initialize the entry. It MUST be present in index. Call to AddEntry() or michael@0: // EnsureEntryExists() must precede the call to this method. michael@0: static nsresult InitEntry(const SHA1Sum::Hash *aHash, michael@0: uint32_t aAppId, michael@0: bool aAnonymous, michael@0: bool aInBrowser); michael@0: michael@0: // Remove entry from index. The entry should be present in index. michael@0: static nsresult RemoveEntry(const SHA1Sum::Hash *aHash); michael@0: michael@0: // Update some information in entry. The entry MUST be present in index and michael@0: // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to michael@0: // InitEntry() must precede the call to this method. michael@0: // Pass nullptr if the value didn't change. michael@0: static nsresult UpdateEntry(const SHA1Sum::Hash *aHash, michael@0: const uint32_t *aFrecency, michael@0: const uint32_t *aExpirationTime, michael@0: const uint32_t *aSize); michael@0: michael@0: // Remove all entries from the index. Called when clearing the whole cache. michael@0: static nsresult RemoveAll(); michael@0: michael@0: enum EntryStatus { michael@0: EXISTS = 0, michael@0: DOES_NOT_EXIST = 1, michael@0: DO_NOT_KNOW = 2 michael@0: }; michael@0: michael@0: // Returns status of the entry in index for the given key. It can be called michael@0: // on any thread. michael@0: static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval); michael@0: michael@0: // Returns a hash of the least important entry that should be evicted if the michael@0: // cache size is over limit and also returns a total number of all entries in michael@0: // the index. michael@0: static nsresult GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt); michael@0: michael@0: // Returns cache size in kB. michael@0: static nsresult GetCacheSize(uint32_t *_retval); michael@0: michael@0: // Asynchronously gets the disk cache size, used for display in the UI. michael@0: static nsresult AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver); michael@0: michael@0: // Returns an iterator that returns entries matching a given context that were michael@0: // present in the index at the time this method was called. If aAddNew is true michael@0: // then the iterator will also return entries created after this call. michael@0: // NOTE: When some entry is removed from index it is removed also from the michael@0: // iterator regardless what aAddNew was passed. michael@0: static nsresult GetIterator(nsILoadContextInfo *aInfo, bool aAddNew, michael@0: CacheIndexIterator **_retval); michael@0: michael@0: // Returns true if we _think_ that the index is up to date. I.e. the state is michael@0: // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false. michael@0: static nsresult IsUpToDate(bool *_retval); michael@0: michael@0: // Memory reporting michael@0: static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf); michael@0: static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf); michael@0: michael@0: private: michael@0: friend class CacheIndexEntryAutoManage; michael@0: friend class CacheIndexAutoLock; michael@0: friend class CacheIndexAutoUnlock; michael@0: friend class FileOpenHelper; michael@0: friend class CacheIndexIterator; michael@0: michael@0: virtual ~CacheIndex(); michael@0: michael@0: NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult); michael@0: nsresult OnFileOpenedInternal(FileOpenHelper *aOpener, michael@0: CacheFileHandle *aHandle, nsresult aResult); michael@0: NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf, michael@0: nsresult aResult); michael@0: NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult); michael@0: NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult); michael@0: NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult); michael@0: NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult); michael@0: michael@0: void Lock(); michael@0: void Unlock(); michael@0: void AssertOwnsLock(); michael@0: michael@0: nsresult InitInternal(nsIFile *aCacheDirectory); michael@0: void PreShutdownInternal(); michael@0: michael@0: // This method returns false when index is not initialized or is shut down. michael@0: bool IsIndexUsable(); michael@0: michael@0: // This method checks whether the entry has the same values of appId, michael@0: // isAnonymous and isInBrowser. We don't expect to find a collision since michael@0: // these values are part of the key that we hash and we use a strong hash michael@0: // function. michael@0: static bool IsCollision(CacheIndexEntry *aEntry, michael@0: uint32_t aAppId, michael@0: bool aAnonymous, michael@0: bool aInBrowser); michael@0: michael@0: // Checks whether any of the information about the entry has changed. michael@0: static bool HasEntryChanged(CacheIndexEntry *aEntry, michael@0: const uint32_t *aFrecency, michael@0: const uint32_t *aExpirationTime, michael@0: const uint32_t *aSize); michael@0: michael@0: // Merge all pending operations from mPendingUpdates into mIndex. michael@0: void ProcessPendingOperations(); michael@0: static PLDHashOperator UpdateEntryInIndex(CacheIndexEntry *aEntry, michael@0: void* aClosure); michael@0: michael@0: // Following methods perform writing of the index file. michael@0: // michael@0: // The index is written periodically, but not earlier than once in michael@0: // kMinDumpInterval and there must be at least kMinUnwrittenChanges michael@0: // differences between index on disk and in memory. Index is always first michael@0: // written to a temporary file and the old index file is replaced when the michael@0: // writing process succeeds. michael@0: // michael@0: // Starts writing of index when both limits (minimal delay between writes and michael@0: // minimum number of changes in index) were exceeded. michael@0: bool WriteIndexToDiskIfNeeded(); michael@0: // Starts writing of index file. michael@0: void WriteIndexToDisk(); michael@0: // Serializes part of mIndex hashtable to the write buffer a writes the buffer michael@0: // to the file. michael@0: void WriteRecords(); michael@0: // Finalizes writing process. michael@0: void FinishWrite(bool aSucceeded); michael@0: michael@0: static PLDHashOperator CopyRecordsToRWBuf(CacheIndexEntry *aEntry, michael@0: void* aClosure); michael@0: static PLDHashOperator ApplyIndexChanges(CacheIndexEntry *aEntry, michael@0: void* aClosure); michael@0: michael@0: // Following methods perform writing of the journal during shutdown. All these michael@0: // methods must be called only during shutdown since they write/delete files michael@0: // directly on the main thread instead of using CacheFileIOManager that does michael@0: // it asynchronously on IO thread. Journal contains only entries that are michael@0: // dirty, i.e. changes that are not present in the index file on the disk. michael@0: // When the log is written successfully, the dirty flag in index file is michael@0: // cleared. michael@0: nsresult GetFile(const nsACString &aName, nsIFile **_retval); michael@0: nsresult RemoveFile(const nsACString &aName); michael@0: void RemoveIndexFromDisk(); michael@0: // Writes journal to the disk and clears dirty flag in index header. michael@0: nsresult WriteLogToDisk(); michael@0: michael@0: static PLDHashOperator WriteEntryToLog(CacheIndexEntry *aEntry, michael@0: void* aClosure); michael@0: michael@0: // Following methods perform reading of the index from the disk. michael@0: // michael@0: // Index is read at startup just after initializing the CacheIndex. There are michael@0: // 3 files used when manipulating with index: index file, journal file and michael@0: // a temporary file. All files contain the hash of the data, so we can check michael@0: // whether the content is valid and complete. Index file contains also a dirty michael@0: // flag in the index header which is unset on a clean shutdown. During opening michael@0: // and reading of the files we determine the status of the whole index from michael@0: // the states of the separate files. Following table shows all possible michael@0: // combinations: michael@0: // michael@0: // index, journal, tmpfile michael@0: // M * * - index is missing -> BUILD michael@0: // I * * - index is invalid -> BUILD michael@0: // D * * - index is dirty -> UPDATE michael@0: // C M * - index is dirty -> UPDATE michael@0: // C I * - unexpected state -> UPDATE michael@0: // C V E - unexpected state -> UPDATE michael@0: // C V M - index is up to date -> READY michael@0: // michael@0: // where the letters mean: michael@0: // * - any state michael@0: // E - file exists michael@0: // M - file is missing michael@0: // I - data is invalid (parsing failed or hash didn't match) michael@0: // D - dirty (data in index file is correct, but dirty flag is set) michael@0: // C - clean (index file is clean) michael@0: // V - valid (data in journal file is correct) michael@0: // michael@0: // Note: We accept the data from journal only when the index is up to date as michael@0: // a whole (i.e. C,V,M state). michael@0: // michael@0: // We rename the journal file to the temporary file as soon as possible after michael@0: // initial test to ensure that we start update process on the next startup if michael@0: // FF crashes during parsing of the index. michael@0: // michael@0: // Initiates reading index from disk. michael@0: void ReadIndexFromDisk(); michael@0: // Starts reading data from index file. michael@0: void StartReadingIndex(); michael@0: // Parses data read from index file. michael@0: void ParseRecords(); michael@0: // Starts reading data from journal file. michael@0: void StartReadingJournal(); michael@0: // Parses data read from journal file. michael@0: void ParseJournal(); michael@0: // Merges entries from journal into mIndex. michael@0: void MergeJournal(); michael@0: // In debug build this method checks that we have no fresh entry in mIndex michael@0: // after we finish reading index and before we process pending operations. michael@0: void EnsureNoFreshEntry(); michael@0: // In debug build this method is called after processing pending operations michael@0: // to make sure mIndexStats contains correct information. michael@0: void EnsureCorrectStats(); michael@0: static PLDHashOperator SumIndexStats(CacheIndexEntry *aEntry, void* aClosure); michael@0: // Finalizes reading process. michael@0: void FinishRead(bool aSucceeded); michael@0: michael@0: static PLDHashOperator ProcessJournalEntry(CacheIndexEntry *aEntry, michael@0: void* aClosure); michael@0: michael@0: // Following methods perform updating and building of the index. michael@0: // Timer callback that starts update or build process. michael@0: static void DelayedUpdate(nsITimer *aTimer, void *aClosure); michael@0: // Posts timer event that start update or build process. michael@0: nsresult ScheduleUpdateTimer(uint32_t aDelay); michael@0: nsresult SetupDirectoryEnumerator(); michael@0: void InitEntryFromDiskData(CacheIndexEntry *aEntry, michael@0: CacheFileMetadata *aMetaData, michael@0: int64_t aFileSize); michael@0: // Returns true when either a timer is scheduled or event is posted. michael@0: bool IsUpdatePending(); michael@0: // Iterates through all files in entries directory that we didn't create/open michael@0: // during this session, parses them and adds the entries to the index. michael@0: void BuildIndex(); michael@0: michael@0: bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState = false); michael@0: // Starts update or build process or fires a timer when it is too early after michael@0: // startup. michael@0: void StartUpdatingIndex(bool aRebuild); michael@0: // Iterates through all files in entries directory that we didn't create/open michael@0: // during this session and theirs last modified time is newer than timestamp michael@0: // in the index header. Parses the files and adds the entries to the index. michael@0: void UpdateIndex(); michael@0: // Finalizes update or build process. michael@0: void FinishUpdate(bool aSucceeded); michael@0: michael@0: static PLDHashOperator RemoveNonFreshEntries(CacheIndexEntry *aEntry, michael@0: void* aClosure); michael@0: michael@0: enum EState { michael@0: // Initial state in which the index is not usable michael@0: // Possible transitions: michael@0: // -> READING michael@0: INITIAL = 0, michael@0: michael@0: // Index is being read from the disk. michael@0: // Possible transitions: michael@0: // -> INITIAL - We failed to dispatch a read event. michael@0: // -> BUILDING - No or corrupted index file was found. michael@0: // -> UPDATING - No or corrupted journal file was found. michael@0: // - Dirty flag was set in index header. michael@0: // -> READY - Index was read successfully or was interrupted by michael@0: // pre-shutdown. michael@0: // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. michael@0: READING = 1, michael@0: michael@0: // Index is being written to the disk. michael@0: // Possible transitions: michael@0: // -> READY - Writing of index finished or was interrupted by michael@0: // pre-shutdown.. michael@0: // -> UPDATING - Writing of index finished, but index was found outdated michael@0: // during writing. michael@0: // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. michael@0: WRITING = 2, michael@0: michael@0: // Index is being build. michael@0: // Possible transitions: michael@0: // -> READY - Building of index finished or was interrupted by michael@0: // pre-shutdown. michael@0: // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. michael@0: BUILDING = 3, michael@0: michael@0: // Index is being updated. michael@0: // Possible transitions: michael@0: // -> READY - Updating of index finished or was interrupted by michael@0: // pre-shutdown. michael@0: // -> SHUTDOWN - This could happen only in case of pre-shutdown failure. michael@0: UPDATING = 4, michael@0: michael@0: // Index is ready. michael@0: // Possible transitions: michael@0: // -> UPDATING - Index was found outdated. michael@0: // -> SHUTDOWN - Index is shutting down. michael@0: READY = 5, michael@0: michael@0: // Index is shutting down. michael@0: SHUTDOWN = 6 michael@0: }; michael@0: michael@0: #ifdef PR_LOGGING michael@0: static char const * StateString(EState aState); michael@0: #endif michael@0: void ChangeState(EState aNewState); michael@0: michael@0: // Allocates and releases buffer used for reading and writing index. michael@0: void AllocBuffer(); michael@0: void ReleaseBuffer(); michael@0: michael@0: // Methods used by CacheIndexEntryAutoManage to keep the arrays up to date. michael@0: void InsertRecordToFrecencyArray(CacheIndexRecord *aRecord); michael@0: void InsertRecordToExpirationArray(CacheIndexRecord *aRecord); michael@0: void RemoveRecordFromFrecencyArray(CacheIndexRecord *aRecord); michael@0: void RemoveRecordFromExpirationArray(CacheIndexRecord *aRecord); michael@0: michael@0: // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date. michael@0: void AddRecordToIterators(CacheIndexRecord *aRecord); michael@0: void RemoveRecordFromIterators(CacheIndexRecord *aRecord); michael@0: void ReplaceRecordInIterators(CacheIndexRecord *aOldRecord, michael@0: CacheIndexRecord *aNewRecord); michael@0: michael@0: // Memory reporting (private part) michael@0: size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const; michael@0: michael@0: static CacheIndex *gInstance; michael@0: michael@0: nsCOMPtr mCacheDirectory; michael@0: michael@0: mozilla::Mutex mLock; michael@0: EState mState; michael@0: // Timestamp of time when the index was initialized. We use it to delay michael@0: // initial update or build of index. michael@0: TimeStamp mStartTime; michael@0: // Set to true in PreShutdown(), it is checked on variaous places to prevent michael@0: // starting any process (write, update, etc.) during shutdown. michael@0: bool mShuttingDown; michael@0: // When set to true, update process should start as soon as possible. This michael@0: // flag is set whenever we find some inconsistency which would be fixed by michael@0: // update process. The flag is checked always when switching to READY state. michael@0: // To make sure we start the update process as soon as possible, methods that michael@0: // set this flag should also call StartUpdatingIndexIfNeeded() to cover the michael@0: // case when we are currently in READY state. michael@0: bool mIndexNeedsUpdate; michael@0: // Set at the beginning of RemoveAll() which clears the whole index. When michael@0: // removing all entries we must stop any pending reading, writing, updating or michael@0: // building operation. This flag is checked at various places and it prevents michael@0: // we won't start another operation (e.g. canceling reading of the index would michael@0: // normally start update or build process) michael@0: bool mRemovingAll; michael@0: // Whether the index file on disk exists and is valid. michael@0: bool mIndexOnDiskIsValid; michael@0: // When something goes wrong during updating or building process, we don't michael@0: // mark index clean (and also don't write journal) to ensure that update or michael@0: // build will be initiated on the next start. michael@0: bool mDontMarkIndexClean; michael@0: // Timestamp value from index file. It is used during update process to skip michael@0: // entries that were last modified before this timestamp. michael@0: uint32_t mIndexTimeStamp; michael@0: // Timestamp of last time the index was dumped to disk. michael@0: // NOTE: The index might not be necessarily dumped at this time. The value michael@0: // is used to schedule next dump of the index. michael@0: TimeStamp mLastDumpTime; michael@0: michael@0: // Timer of delayed update/build. michael@0: nsCOMPtr mUpdateTimer; michael@0: // True when build or update event is posted michael@0: bool mUpdateEventPending; michael@0: michael@0: // Helper members used when reading/writing index from/to disk. michael@0: // Contains number of entries that should be skipped: michael@0: // - in hashtable when writing index because they were already written michael@0: // - in index file when reading index because they were already read michael@0: uint32_t mSkipEntries; michael@0: // Number of entries that should be written to disk. This is number of entries michael@0: // in hashtable that are initialized and are not marked as removed when writing michael@0: // begins. michael@0: uint32_t mProcessEntries; michael@0: char *mRWBuf; michael@0: uint32_t mRWBufSize; michael@0: uint32_t mRWBufPos; michael@0: nsRefPtr mRWHash; michael@0: michael@0: // Reading of journal succeeded if true. michael@0: bool mJournalReadSuccessfully; michael@0: michael@0: // Handle used for writing and reading index file. michael@0: nsRefPtr mIndexHandle; michael@0: // Handle used for reading journal file. michael@0: nsRefPtr mJournalHandle; michael@0: // Used to check the existence of the file during reading process. michael@0: nsRefPtr mTmpHandle; michael@0: michael@0: nsRefPtr mIndexFileOpener; michael@0: nsRefPtr mJournalFileOpener; michael@0: nsRefPtr mTmpFileOpener; michael@0: michael@0: // Directory enumerator used when building and updating index. michael@0: nsCOMPtr mDirEnumerator; michael@0: michael@0: // Main index hashtable. michael@0: nsTHashtable mIndex; michael@0: michael@0: // We cannot add, remove or change any entry in mIndex in states READING and michael@0: // WRITING. We track all changes in mPendingUpdates during these states. michael@0: nsTHashtable mPendingUpdates; michael@0: michael@0: // Contains information statistics for mIndex + mPendingUpdates. michael@0: CacheIndexStats mIndexStats; michael@0: michael@0: // When reading journal, we must first parse the whole file and apply the michael@0: // changes iff the journal was read successfully. mTmpJournal is used to store michael@0: // entries from the journal file. We throw away all these entries if parsing michael@0: // of the journal fails or the hash does not match. michael@0: nsTHashtable mTmpJournal; michael@0: michael@0: // Arrays that keep entry records ordered by eviction preference. When looking michael@0: // for an entry to evict, we first try to find an expired entry. If there is michael@0: // no expired entry, we take the entry with lowest valid frecency. Zero michael@0: // frecency is an initial value and such entries are stored at the end of the michael@0: // array. Uninitialized entries and entries marked as deleted are not present michael@0: // in these arrays. michael@0: nsTArray mFrecencyArray; michael@0: nsTArray mExpirationArray; michael@0: michael@0: nsTArray mIterators; michael@0: michael@0: class DiskConsumptionObserver : public nsRunnable michael@0: { michael@0: public: michael@0: static DiskConsumptionObserver* Init(nsICacheStorageConsumptionObserver* aObserver) michael@0: { michael@0: nsWeakPtr observer = do_GetWeakReference(aObserver); michael@0: if (!observer) michael@0: return nullptr; michael@0: michael@0: return new DiskConsumptionObserver(observer); michael@0: } michael@0: michael@0: void OnDiskConsumption(int64_t aSize) michael@0: { michael@0: mSize = aSize; michael@0: NS_DispatchToMainThread(this); michael@0: } michael@0: michael@0: private: michael@0: DiskConsumptionObserver(nsWeakPtr const &aWeakObserver) michael@0: : mObserver(aWeakObserver) { } michael@0: virtual ~DiskConsumptionObserver() { } michael@0: michael@0: NS_IMETHODIMP Run() michael@0: { michael@0: MOZ_ASSERT(NS_IsMainThread()); michael@0: michael@0: nsCOMPtr observer = michael@0: do_QueryReferent(mObserver); michael@0: michael@0: if (observer) { michael@0: observer->OnNetworkCacheDiskConsumption(mSize); michael@0: } michael@0: michael@0: return NS_OK; michael@0: } michael@0: michael@0: nsWeakPtr mObserver; michael@0: int64_t mSize; michael@0: }; michael@0: michael@0: // List of async observers that want to get disk consumption information michael@0: nsTArray > mDiskConsumptionObservers; michael@0: }; michael@0: michael@0: class CacheIndexAutoLock { michael@0: public: michael@0: CacheIndexAutoLock(CacheIndex *aIndex) michael@0: : mIndex(aIndex) michael@0: , mLocked(true) michael@0: { michael@0: mIndex->Lock(); michael@0: } michael@0: ~CacheIndexAutoLock() michael@0: { michael@0: if (mLocked) { michael@0: mIndex->Unlock(); michael@0: } michael@0: } michael@0: void Lock() michael@0: { michael@0: MOZ_ASSERT(!mLocked); michael@0: mIndex->Lock(); michael@0: mLocked = true; michael@0: } michael@0: void Unlock() michael@0: { michael@0: MOZ_ASSERT(mLocked); michael@0: mIndex->Unlock(); michael@0: mLocked = false; michael@0: } michael@0: michael@0: private: michael@0: nsRefPtr mIndex; michael@0: bool mLocked; michael@0: }; michael@0: michael@0: class CacheIndexAutoUnlock { michael@0: public: michael@0: CacheIndexAutoUnlock(CacheIndex *aIndex) michael@0: : mIndex(aIndex) michael@0: , mLocked(false) michael@0: { michael@0: mIndex->Unlock(); michael@0: } michael@0: ~CacheIndexAutoUnlock() michael@0: { michael@0: if (!mLocked) { michael@0: mIndex->Lock(); michael@0: } michael@0: } michael@0: void Lock() michael@0: { michael@0: MOZ_ASSERT(!mLocked); michael@0: mIndex->Lock(); michael@0: mLocked = true; michael@0: } michael@0: void Unlock() michael@0: { michael@0: MOZ_ASSERT(mLocked); michael@0: mIndex->Unlock(); michael@0: mLocked = false; michael@0: } michael@0: michael@0: private: michael@0: nsRefPtr mIndex; michael@0: bool mLocked; michael@0: }; michael@0: michael@0: } // net michael@0: } // mozilla michael@0: michael@0: #endif