diff -r 000000000000 -r 6474c204b198 netwerk/cache2/CacheIndex.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/netwerk/cache2/CacheIndex.cpp Wed Dec 31 06:09:35 2014 +0100 @@ -0,0 +1,3562 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CacheIndex.h" + +#include "CacheLog.h" +#include "CacheFileIOManager.h" +#include "CacheFileMetadata.h" +#include "CacheIndexIterator.h" +#include "CacheIndexContextIterator.h" +#include "nsThreadUtils.h" +#include "nsISimpleEnumerator.h" +#include "nsIDirectoryEnumerator.h" +#include "nsISizeOf.h" +#include "nsPrintfCString.h" +#include "mozilla/DebugOnly.h" +#include "prinrval.h" +#include "nsIFile.h" +#include "nsITimer.h" +#include "mozilla/AutoRestore.h" +#include + + +#define kMinUnwrittenChanges 300 +#define kMinDumpInterval 20000 // in milliseconds +#define kMaxBufSize 16384 +#define kIndexVersion 0x00000001 +#define kUpdateIndexStartDelay 50000 // in milliseconds + +const char kIndexName[] = "index"; +const char kTempIndexName[] = "index.tmp"; +const char kJournalName[] = "index.log"; + +namespace mozilla { +namespace net { + +/** + * This helper class is responsible for keeping CacheIndex::mIndexStats, + * CacheIndex::mFrecencyArray and CacheIndex::mExpirationArray up to date. + */ +class CacheIndexEntryAutoManage +{ +public: + CacheIndexEntryAutoManage(const SHA1Sum::Hash *aHash, CacheIndex *aIndex) + : mIndex(aIndex) + , mOldRecord(nullptr) + , mOldFrecency(0) + , mOldExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME) + , mDoNotSearchInIndex(false) + , mDoNotSearchInUpdates(false) + { + mIndex->AssertOwnsLock(); + + mHash = aHash; + CacheIndexEntry *entry = FindEntry(); + mIndex->mIndexStats.BeforeChange(entry); + if (entry && entry->IsInitialized() && !entry->IsRemoved()) { + mOldRecord = entry->mRec; + mOldFrecency = entry->mRec->mFrecency; + mOldExpirationTime = entry->mRec->mExpirationTime; + } + } + + ~CacheIndexEntryAutoManage() + { + mIndex->AssertOwnsLock(); + + CacheIndexEntry *entry = FindEntry(); + mIndex->mIndexStats.AfterChange(entry); + if (!entry || !entry->IsInitialized() || entry->IsRemoved()) { + entry = nullptr; + } + + if (entry && !mOldRecord) { + mIndex->InsertRecordToFrecencyArray(entry->mRec); + mIndex->InsertRecordToExpirationArray(entry->mRec); + mIndex->AddRecordToIterators(entry->mRec); + } else if (!entry && mOldRecord) { + mIndex->RemoveRecordFromFrecencyArray(mOldRecord); + mIndex->RemoveRecordFromExpirationArray(mOldRecord); + mIndex->RemoveRecordFromIterators(mOldRecord); + } else if (entry && mOldRecord) { + bool replaceFrecency = false; + bool replaceExpiration = false; + + if (entry->mRec != mOldRecord) { + // record has a different address, we have to replace it + replaceFrecency = replaceExpiration = true; + mIndex->ReplaceRecordInIterators(mOldRecord, entry->mRec); + } else { + if (entry->mRec->mFrecency == 0 && + entry->mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME) { + // This is a special case when we want to make sure that the entry is + // placed at the end of the lists even when the values didn't change. + replaceFrecency = replaceExpiration = true; + } else { + if (entry->mRec->mFrecency != mOldFrecency) { + replaceFrecency = true; + } + if (entry->mRec->mExpirationTime != mOldExpirationTime) { + replaceExpiration = true; + } + } + } + + if (replaceFrecency) { + mIndex->RemoveRecordFromFrecencyArray(mOldRecord); + mIndex->InsertRecordToFrecencyArray(entry->mRec); + } + if (replaceExpiration) { + mIndex->RemoveRecordFromExpirationArray(mOldRecord); + mIndex->InsertRecordToExpirationArray(entry->mRec); + } + } else { + // both entries were removed or not initialized, do nothing + } + } + + // We cannot rely on nsTHashtable::GetEntry() in case we are enumerating the + // entries and returning PL_DHASH_REMOVE. Destructor is called before the + // entry is removed. Caller must call one of following methods to skip + // lookup in the hashtable. + void DoNotSearchInIndex() { mDoNotSearchInIndex = true; } + void DoNotSearchInUpdates() { mDoNotSearchInUpdates = true; } + +private: + CacheIndexEntry * FindEntry() + { + CacheIndexEntry *entry = nullptr; + + switch (mIndex->mState) { + case CacheIndex::READING: + case CacheIndex::WRITING: + if (!mDoNotSearchInUpdates) { + entry = mIndex->mPendingUpdates.GetEntry(*mHash); + } + // no break + case CacheIndex::BUILDING: + case CacheIndex::UPDATING: + case CacheIndex::READY: + if (!entry && !mDoNotSearchInIndex) { + entry = mIndex->mIndex.GetEntry(*mHash); + } + break; + case CacheIndex::INITIAL: + case CacheIndex::SHUTDOWN: + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + return entry; + } + + const SHA1Sum::Hash *mHash; + nsRefPtr mIndex; + CacheIndexRecord *mOldRecord; + uint32_t mOldFrecency; + uint32_t mOldExpirationTime; + bool mDoNotSearchInIndex; + bool mDoNotSearchInUpdates; +}; + +class FileOpenHelper : public CacheFileIOListener +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + + FileOpenHelper(CacheIndex* aIndex) + : mIndex(aIndex) + , mCanceled(false) + {} + + virtual ~FileOpenHelper() {} + + void Cancel() { + mIndex->AssertOwnsLock(); + mCanceled = true; + } + +private: + NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult); + NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf, + nsresult aResult) { + MOZ_CRASH("FileOpenHelper::OnDataWritten should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, + nsresult aResult) { + MOZ_CRASH("FileOpenHelper::OnDataRead should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) { + MOZ_CRASH("FileOpenHelper::OnFileDoomed should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) { + MOZ_CRASH("FileOpenHelper::OnEOFSet should not be called!"); + return NS_ERROR_UNEXPECTED; + } + NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) { + MOZ_CRASH("FileOpenHelper::OnFileRenamed should not be called!"); + return NS_ERROR_UNEXPECTED; + } + + nsRefPtr mIndex; + bool mCanceled; +}; + +NS_IMETHODIMP FileOpenHelper::OnFileOpened(CacheFileHandle *aHandle, + nsresult aResult) +{ + CacheIndexAutoLock lock(mIndex); + + if (mCanceled) { + if (aHandle) { + CacheFileIOManager::DoomFile(aHandle, nullptr); + } + + return NS_OK; + } + + mIndex->OnFileOpenedInternal(this, aHandle, aResult); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(FileOpenHelper, CacheFileIOListener); + + +CacheIndex * CacheIndex::gInstance = nullptr; + + +NS_IMPL_ADDREF(CacheIndex) +NS_IMPL_RELEASE(CacheIndex) + +NS_INTERFACE_MAP_BEGIN(CacheIndex) + NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener) + NS_INTERFACE_MAP_ENTRY(nsIRunnable) +NS_INTERFACE_MAP_END_THREADSAFE + + +CacheIndex::CacheIndex() + : mLock("CacheFile.mLock") + , mState(INITIAL) + , mShuttingDown(false) + , mIndexNeedsUpdate(false) + , mRemovingAll(false) + , mIndexOnDiskIsValid(false) + , mDontMarkIndexClean(false) + , mIndexTimeStamp(0) + , mUpdateEventPending(false) + , mSkipEntries(0) + , mProcessEntries(0) + , mRWBuf(nullptr) + , mRWBufSize(0) + , mRWBufPos(0) + , mJournalReadSuccessfully(false) +{ + LOG(("CacheIndex::CacheIndex [this=%p]", this)); + MOZ_COUNT_CTOR(CacheIndex); + MOZ_ASSERT(!gInstance, "multiple CacheIndex instances!"); +} + +CacheIndex::~CacheIndex() +{ + LOG(("CacheIndex::~CacheIndex [this=%p]", this)); + MOZ_COUNT_DTOR(CacheIndex); + + ReleaseBuffer(); +} + +void +CacheIndex::Lock() +{ + mLock.Lock(); + + MOZ_ASSERT(!mIndexStats.StateLogged()); +} + +void +CacheIndex::Unlock() +{ + MOZ_ASSERT(!mIndexStats.StateLogged()); + + mLock.Unlock(); +} + +inline void +CacheIndex::AssertOwnsLock() +{ + mLock.AssertCurrentThreadOwns(); +} + +// static +nsresult +CacheIndex::Init(nsIFile *aCacheDirectory) +{ + LOG(("CacheIndex::Init()")); + + MOZ_ASSERT(NS_IsMainThread()); + + if (gInstance) { + return NS_ERROR_ALREADY_INITIALIZED; + } + + nsRefPtr idx = new CacheIndex(); + + CacheIndexAutoLock lock(idx); + + nsresult rv = idx->InitInternal(aCacheDirectory); + NS_ENSURE_SUCCESS(rv, rv); + + idx.swap(gInstance); + return NS_OK; +} + +nsresult +CacheIndex::InitInternal(nsIFile *aCacheDirectory) +{ + nsresult rv; + + rv = aCacheDirectory->Clone(getter_AddRefs(mCacheDirectory)); + NS_ENSURE_SUCCESS(rv, rv); + + mStartTime = TimeStamp::NowLoRes(); + + ReadIndexFromDisk(); + + return NS_OK; +} + +// static +nsresult +CacheIndex::PreShutdown() +{ + LOG(("CacheIndex::PreShutdown() [gInstance=%p]", gInstance)); + + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + CacheIndexAutoLock lock(index); + + LOG(("CacheIndex::PreShutdown() - [state=%d, indexOnDiskIsValid=%d, " + "dontMarkIndexClean=%d]", index->mState, index->mIndexOnDiskIsValid, + index->mDontMarkIndexClean)); + + LOG(("CacheIndex::PreShutdown() - Closing iterators.")); + for (uint32_t i = 0; i < index->mIterators.Length(); ) { + rv = index->mIterators[i]->CloseInternal(NS_ERROR_FAILURE); + if (NS_FAILED(rv)) { + // CacheIndexIterator::CloseInternal() removes itself from mIteratos iff + // it returns success. + LOG(("CacheIndex::PreShutdown() - Failed to remove iterator %p. " + "[rv=0x%08x]", rv)); + i++; + } + } + + index->mShuttingDown = true; + + if (index->mState == READY) { + return NS_OK; // nothing to do + } + + nsCOMPtr event; + event = NS_NewRunnableMethod(index, &CacheIndex::PreShutdownInternal); + + nsCOMPtr ioTarget = CacheFileIOManager::IOTarget(); + MOZ_ASSERT(ioTarget); + + // PreShutdownInternal() will be executed before any queued event on INDEX + // level. That's OK since we don't want to wait for any operation in progess. + // We need to interrupt it and save journal as quickly as possible. + rv = ioTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL); + if (NS_FAILED(rv)) { + NS_WARNING("CacheIndex::PreShutdown() - Can't dispatch event"); + LOG(("CacheIndex::PreShutdown() - Can't dispatch event" )); + return rv; + } + + return NS_OK; +} + +void +CacheIndex::PreShutdownInternal() +{ + CacheIndexAutoLock lock(this); + + LOG(("CacheIndex::PreShutdownInternal() - [state=%d, indexOnDiskIsValid=%d, " + "dontMarkIndexClean=%d]", mState, mIndexOnDiskIsValid, + mDontMarkIndexClean)); + + MOZ_ASSERT(mShuttingDown); + + if (mUpdateTimer) { + mUpdateTimer = nullptr; + } + + switch (mState) { + case WRITING: + FinishWrite(false); + break; + case READY: + // nothing to do, write the journal in Shutdown() + break; + case READING: + FinishRead(false); + break; + case BUILDING: + case UPDATING: + FinishUpdate(false); + break; + default: + MOZ_ASSERT(false, "Implement me!"); + } + + // We should end up in READY state + MOZ_ASSERT(mState == READY); +} + +// static +nsresult +CacheIndex::Shutdown() +{ + LOG(("CacheIndex::Shutdown() [gInstance=%p]", gInstance)); + + MOZ_ASSERT(NS_IsMainThread()); + + nsRefPtr index; + index.swap(gInstance); + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + CacheIndexAutoLock lock(index); + + bool sanitize = CacheObserver::ClearCacheOnShutdown(); + + LOG(("CacheIndex::Shutdown() - [state=%d, indexOnDiskIsValid=%d, " + "dontMarkIndexClean=%d, sanitize=%d]", index->mState, + index->mIndexOnDiskIsValid, index->mDontMarkIndexClean, sanitize)); + + MOZ_ASSERT(index->mShuttingDown); + + EState oldState = index->mState; + index->ChangeState(SHUTDOWN); + + if (oldState != READY) { + LOG(("CacheIndex::Shutdown() - Unexpected state. Did posting of " + "PreShutdownInternal() fail?")); + } + + switch (oldState) { + case WRITING: + index->FinishWrite(false); + // no break + case READY: + if (index->mIndexOnDiskIsValid && !index->mDontMarkIndexClean) { + if (!sanitize && NS_FAILED(index->WriteLogToDisk())) { + index->RemoveIndexFromDisk(); + } + } else { + index->RemoveIndexFromDisk(); + } + break; + case READING: + index->FinishRead(false); + break; + case BUILDING: + case UPDATING: + index->FinishUpdate(false); + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + if (sanitize) { + index->RemoveIndexFromDisk(); + } + + return NS_OK; +} + +// static +nsresult +CacheIndex::AddEntry(const SHA1Sum::Hash *aHash) +{ + LOG(("CacheIndex::AddEntry() [hash=%08x%08x%08x%08x%08x]", LOGSHA1(aHash))); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + // Getters in CacheIndexStats assert when mStateLogged is true since the + // information is incomplete between calls to BeforeChange() and AfterChange() + // (i.e. while CacheIndexEntryAutoManage exists). We need to check whether + // non-fresh entries exists outside the scope of CacheIndexEntryAutoManage. + bool updateIfNonFreshEntriesExist = false; + + { + CacheIndexEntryAutoManage entryMng(aHash, index); + + CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash); + bool entryRemoved = entry && entry->IsRemoved(); + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + + if (entry && !entryRemoved) { + // Found entry in index that shouldn't exist. + + if (entry->IsFresh()) { + // Someone removed the file on disk while FF is running. Update + // process can fix only non-fresh entries (i.e. entries that were not + // added within this session). Start update only if we have such + // entries. + // + // TODO: This should be very rare problem. If it turns out not to be + // true, change the update process so that it also iterates all + // initialized non-empty entries and checks whether the file exists. + + LOG(("CacheIndex::AddEntry() - Cache file was removed outside FF " + "process!")); + + updateIfNonFreshEntriesExist = true; + } else if (index->mState == READY) { + // Index is outdated, update it. + LOG(("CacheIndex::AddEntry() - Found entry that shouldn't exist, " + "update is needed")); + index->mIndexNeedsUpdate = true; + } else { + // We cannot be here when building index since all entries are fresh + // during building. + MOZ_ASSERT(index->mState == UPDATING); + } + } + + if (!entry) { + entry = index->mIndex.PutEntry(*aHash); + } + } else { // WRITING, READING + CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash); + bool updatedRemoved = updated && updated->IsRemoved(); + + if ((updated && !updatedRemoved) || + (!updated && entry && !entryRemoved && entry->IsFresh())) { + // Fresh entry found, so the file was removed outside FF + LOG(("CacheIndex::AddEntry() - Cache file was removed outside FF " + "process!")); + + updateIfNonFreshEntriesExist = true; + } else if (!updated && entry && !entryRemoved) { + if (index->mState == WRITING) { + LOG(("CacheIndex::AddEntry() - Found entry that shouldn't exist, " + "update is needed")); + index->mIndexNeedsUpdate = true; + } + // Ignore if state is READING since the index information is partial + } + + updated = index->mPendingUpdates.PutEntry(*aHash); + entry = updated; + } + + entry->InitNew(); + entry->MarkDirty(); + entry->MarkFresh(); + } + + if (updateIfNonFreshEntriesExist && + index->mIndexStats.Count() != index->mIndexStats.Fresh()) { + index->mIndexNeedsUpdate = true; + } + + index->StartUpdatingIndexIfNeeded(); + index->WriteIndexToDiskIfNeeded(); + + return NS_OK; +} + +// static +nsresult +CacheIndex::EnsureEntryExists(const SHA1Sum::Hash *aHash) +{ + LOG(("CacheIndex::EnsureEntryExists() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(aHash))); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index); + + CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash); + bool entryRemoved = entry && entry->IsRemoved(); + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + + if (!entry || entryRemoved) { + if (entryRemoved && entry->IsFresh()) { + // This could happen only if somebody copies files to the entries + // directory while FF is running. + LOG(("CacheIndex::EnsureEntryExists() - Cache file was added outside " + "FF process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (index->mState == READY || + (entryRemoved && !entry->IsFresh())) { + // Removed non-fresh entries can be present as a result of + // ProcessJournalEntry() + LOG(("CacheIndex::EnsureEntryExists() - Didn't find entry that should" + " exist, update is needed")); + index->mIndexNeedsUpdate = true; + } + + if (!entry) { + entry = index->mIndex.PutEntry(*aHash); + } + entry->InitNew(); + entry->MarkDirty(); + } + entry->MarkFresh(); + } else { // WRITING, READING + CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash); + bool updatedRemoved = updated && updated->IsRemoved(); + + if (updatedRemoved || + (!updated && entryRemoved && entry->IsFresh())) { + // Fresh information about missing entry found. This could happen only + // if somebody copies files to the entries directory while FF is running. + LOG(("CacheIndex::EnsureEntryExists() - Cache file was added outside " + "FF process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (!updated && (!entry || entryRemoved)) { + if (index->mState == WRITING) { + LOG(("CacheIndex::EnsureEntryExists() - Didn't find entry that should" + " exist, update is needed")); + index->mIndexNeedsUpdate = true; + } + // Ignore if state is READING since the index information is partial + } + + // We don't need entryRemoved and updatedRemoved info anymore + if (entryRemoved) entry = nullptr; + if (updatedRemoved) updated = nullptr; + + if (updated) { + updated->MarkFresh(); + } else { + if (!entry) { + // Create a new entry + updated = index->mPendingUpdates.PutEntry(*aHash); + updated->InitNew(); + updated->MarkFresh(); + updated->MarkDirty(); + } else { + if (!entry->IsFresh()) { + // To mark the entry fresh we must make a copy of index entry + // since the index is read-only. + updated = index->mPendingUpdates.PutEntry(*aHash); + *updated = *entry; + updated->MarkFresh(); + } + } + } + } + } + + index->StartUpdatingIndexIfNeeded(); + index->WriteIndexToDiskIfNeeded(); + + return NS_OK; +} + +// static +nsresult +CacheIndex::InitEntry(const SHA1Sum::Hash *aHash, + uint32_t aAppId, + bool aAnonymous, + bool aInBrowser) +{ + LOG(("CacheIndex::InitEntry() [hash=%08x%08x%08x%08x%08x, appId=%u, " + "anonymous=%d, inBrowser=%d]", LOGSHA1(aHash), aAppId, aAnonymous, + aInBrowser)); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index); + + CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash); + bool reinitEntry = false; + + if (entry && entry->IsRemoved()) { + entry = nullptr; + } + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + MOZ_ASSERT(entry); + MOZ_ASSERT(entry->IsFresh()); + + if (IsCollision(entry, aAppId, aAnonymous, aInBrowser)) { + index->mIndexNeedsUpdate = true; // TODO Does this really help in case of collision? + reinitEntry = true; + } else { + if (entry->IsInitialized()) { + return NS_OK; + } + } + } else { + CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash); + DebugOnly removed = updated && updated->IsRemoved(); + + MOZ_ASSERT(updated || !removed); + MOZ_ASSERT(updated || entry); + + if (updated) { + MOZ_ASSERT(updated->IsFresh()); + + if (IsCollision(updated, aAppId, aAnonymous, aInBrowser)) { + index->mIndexNeedsUpdate = true; + reinitEntry = true; + } else { + if (updated->IsInitialized()) { + return NS_OK; + } + } + entry = updated; + } else { + MOZ_ASSERT(entry->IsFresh()); + + if (IsCollision(entry, aAppId, aAnonymous, aInBrowser)) { + index->mIndexNeedsUpdate = true; + reinitEntry = true; + } else { + if (entry->IsInitialized()) { + return NS_OK; + } + } + + // make a copy of a read-only entry + updated = index->mPendingUpdates.PutEntry(*aHash); + *updated = *entry; + entry = updated; + } + } + + if (reinitEntry) { + // There is a collision and we are going to rewrite this entry. Initialize + // it as a new entry. + entry->InitNew(); + entry->MarkFresh(); + } + entry->Init(aAppId, aAnonymous, aInBrowser); + entry->MarkDirty(); + } + + index->StartUpdatingIndexIfNeeded(); + index->WriteIndexToDiskIfNeeded(); + + return NS_OK; +} + +// static +nsresult +CacheIndex::RemoveEntry(const SHA1Sum::Hash *aHash) +{ + LOG(("CacheIndex::RemoveEntry() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(aHash))); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index); + + CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash); + bool entryRemoved = entry && entry->IsRemoved(); + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + + if (!entry || entryRemoved) { + if (entryRemoved && entry->IsFresh()) { + // This could happen only if somebody copies files to the entries + // directory while FF is running. + LOG(("CacheIndex::RemoveEntry() - Cache file was added outside FF " + "process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (index->mState == READY || + (entryRemoved && !entry->IsFresh())) { + // Removed non-fresh entries can be present as a result of + // ProcessJournalEntry() + LOG(("CacheIndex::RemoveEntry() - Didn't find entry that should exist" + ", update is needed")); + index->mIndexNeedsUpdate = true; + } + } else { + if (entry) { + if (!entry->IsDirty() && entry->IsFileEmpty()) { + index->mIndex.RemoveEntry(*aHash); + entry = nullptr; + } else { + entry->MarkRemoved(); + entry->MarkDirty(); + entry->MarkFresh(); + } + } + } + } else { // WRITING, READING + CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash); + bool updatedRemoved = updated && updated->IsRemoved(); + + if (updatedRemoved || + (!updated && entryRemoved && entry->IsFresh())) { + // Fresh information about missing entry found. This could happen only + // if somebody copies files to the entries directory while FF is running. + LOG(("CacheIndex::RemoveEntry() - Cache file was added outside FF " + "process! Update is needed.")); + index->mIndexNeedsUpdate = true; + } else if (!updated && (!entry || entryRemoved)) { + if (index->mState == WRITING) { + LOG(("CacheIndex::RemoveEntry() - Didn't find entry that should exist" + ", update is needed")); + index->mIndexNeedsUpdate = true; + } + // Ignore if state is READING since the index information is partial + } + + if (!updated) { + updated = index->mPendingUpdates.PutEntry(*aHash); + updated->InitNew(); + } + + updated->MarkRemoved(); + updated->MarkDirty(); + updated->MarkFresh(); + } + } + + index->StartUpdatingIndexIfNeeded(); + index->WriteIndexToDiskIfNeeded(); + + return NS_OK; +} + +// static +nsresult +CacheIndex::UpdateEntry(const SHA1Sum::Hash *aHash, + const uint32_t *aFrecency, + const uint32_t *aExpirationTime, + const uint32_t *aSize) +{ + LOG(("CacheIndex::UpdateEntry() [hash=%08x%08x%08x%08x%08x, " + "frecency=%s, expirationTime=%s, size=%s]", LOGSHA1(aHash), + aFrecency ? nsPrintfCString("%u", *aFrecency).get() : "", + aExpirationTime ? nsPrintfCString("%u", *aExpirationTime).get() : "", + aSize ? nsPrintfCString("%u", *aSize).get() : "")); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + { + CacheIndexEntryAutoManage entryMng(aHash, index); + + CacheIndexEntry *entry = index->mIndex.GetEntry(*aHash); + + if (entry && entry->IsRemoved()) { + entry = nullptr; + } + + if (index->mState == READY || index->mState == UPDATING || + index->mState == BUILDING) { + MOZ_ASSERT(index->mPendingUpdates.Count() == 0); + MOZ_ASSERT(entry); + + if (!HasEntryChanged(entry, aFrecency, aExpirationTime, aSize)) { + return NS_OK; + } + } else { + CacheIndexEntry *updated = index->mPendingUpdates.GetEntry(*aHash); + DebugOnly removed = updated && updated->IsRemoved(); + + MOZ_ASSERT(updated || !removed); + MOZ_ASSERT(updated || entry); + + if (!updated) { + if (entry && + HasEntryChanged(entry, aFrecency, aExpirationTime, aSize)) { + // make a copy of a read-only entry + updated = index->mPendingUpdates.PutEntry(*aHash); + *updated = *entry; + entry = updated; + } else { + return NS_ERROR_NOT_AVAILABLE; + } + } else { + entry = updated; + } + } + + MOZ_ASSERT(entry->IsFresh()); + MOZ_ASSERT(entry->IsInitialized()); + entry->MarkDirty(); + + if (aFrecency) { + entry->SetFrecency(*aFrecency); + } + + if (aExpirationTime) { + entry->SetExpirationTime(*aExpirationTime); + } + + if (aSize) { + entry->SetFileSize(*aSize); + } + } + + index->WriteIndexToDiskIfNeeded(); + + return NS_OK; +} + +// static +nsresult +CacheIndex::RemoveAll() +{ + LOG(("CacheIndex::RemoveAll()")); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + MOZ_ASSERT(CacheFileIOManager::IsOnIOThread()); + + nsCOMPtr file; + + { + CacheIndexAutoLock lock(index); + + MOZ_ASSERT(!index->mRemovingAll); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + AutoRestore saveRemovingAll(index->mRemovingAll); + index->mRemovingAll = true; + + // Doom index and journal handles but don't null them out since this will be + // done in FinishWrite/FinishRead methods. + if (index->mIndexHandle) { + CacheFileIOManager::DoomFile(index->mIndexHandle, nullptr); + } else { + // We don't have a handle to index file, so get the file here, but delete + // it outside the lock. Ignore the result since this is not fatal. + index->GetFile(NS_LITERAL_CSTRING(kIndexName), getter_AddRefs(file)); + } + + if (index->mJournalHandle) { + CacheFileIOManager::DoomFile(index->mJournalHandle, nullptr); + } + + switch (index->mState) { + case WRITING: + index->FinishWrite(false); + break; + case READY: + // nothing to do + break; + case READING: + index->FinishRead(false); + break; + case BUILDING: + case UPDATING: + index->FinishUpdate(false); + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + // We should end up in READY state + MOZ_ASSERT(index->mState == READY); + + // There should not be any handle + MOZ_ASSERT(!index->mIndexHandle); + MOZ_ASSERT(!index->mJournalHandle); + + index->mIndexOnDiskIsValid = false; + index->mIndexNeedsUpdate = false; + + index->mIndexStats.Clear(); + index->mFrecencyArray.Clear(); + index->mExpirationArray.Clear(); + index->mIndex.Clear(); + } + + if (file) { + // Ignore the result. The file might not exist and the failure is not fatal. + file->Remove(false); + } + + return NS_OK; +} + +// static +nsresult +CacheIndex::HasEntry(const nsACString &aKey, EntryStatus *_retval) +{ + LOG(("CacheIndex::HasEntry() [key=%s]", PromiseFlatCString(aKey).get())); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + SHA1Sum sum; + SHA1Sum::Hash hash; + sum.update(aKey.BeginReading(), aKey.Length()); + sum.finish(hash); + + CacheIndexEntry *entry = nullptr; + + switch (index->mState) { + case READING: + case WRITING: + entry = index->mPendingUpdates.GetEntry(hash); + // no break + case BUILDING: + case UPDATING: + case READY: + if (!entry) { + entry = index->mIndex.GetEntry(hash); + } + break; + case INITIAL: + case SHUTDOWN: + MOZ_ASSERT(false, "Unexpected state!"); + } + + if (!entry) { + if (index->mState == READY || index->mState == WRITING) { + *_retval = DOES_NOT_EXIST; + } else { + *_retval = DO_NOT_KNOW; + } + } else { + if (entry->IsRemoved()) { + if (entry->IsFresh()) { + *_retval = DOES_NOT_EXIST; + } else { + *_retval = DO_NOT_KNOW; + } + } else { + *_retval = EXISTS; + } + } + + LOG(("CacheIndex::HasEntry() - result is %u", *_retval)); + return NS_OK; +} + +// static +nsresult +CacheIndex::GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt) +{ + LOG(("CacheIndex::GetEntryForEviction()")); + + nsRefPtr index = gInstance; + + if (!index) + return NS_ERROR_NOT_INITIALIZED; + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + MOZ_ASSERT(index->mFrecencyArray.Length() == + index->mExpirationArray.Length()); + + if (index->mExpirationArray.Length() == 0) + return NS_ERROR_NOT_AVAILABLE; + + uint32_t now = PR_Now() / PR_USEC_PER_SEC; + if (index->mExpirationArray[0]->mExpirationTime < now) { + memcpy(aHash, &index->mExpirationArray[0]->mHash, sizeof(SHA1Sum::Hash)); + *aCnt = index->mExpirationArray.Length(); + LOG(("CacheIndex::GetEntryForEviction() - returning entry from expiration " + "array [hash=%08x%08x%08x%08x%08x, cnt=%u, expTime=%u, now=%u, " + "frecency=%u]", LOGSHA1(aHash), *aCnt, + index->mExpirationArray[0]->mExpirationTime, now, + index->mExpirationArray[0]->mFrecency)); + } + else { + memcpy(aHash, &index->mFrecencyArray[0]->mHash, sizeof(SHA1Sum::Hash)); + *aCnt = index->mFrecencyArray.Length(); + LOG(("CacheIndex::GetEntryForEviction() - returning entry from frecency " + "array [hash=%08x%08x%08x%08x%08x, cnt=%u, expTime=%u, now=%u, " + "frecency=%u]", LOGSHA1(aHash), *aCnt, + index->mExpirationArray[0]->mExpirationTime, now, + index->mExpirationArray[0]->mFrecency)); + } + + return NS_OK; +} + +// static +nsresult +CacheIndex::GetCacheSize(uint32_t *_retval) +{ + LOG(("CacheIndex::GetCacheSize()")); + + nsRefPtr index = gInstance; + + if (!index) + return NS_ERROR_NOT_INITIALIZED; + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + *_retval = index->mIndexStats.Size(); + LOG(("CacheIndex::GetCacheSize() - returning %u", *_retval)); + return NS_OK; +} + +// static +nsresult +CacheIndex::AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver) +{ + LOG(("CacheIndex::AsyncGetDiskConsumption()")); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsRefPtr observer = + DiskConsumptionObserver::Init(aObserver); + + NS_ENSURE_ARG(observer); + + if (index->mState == READY || index->mState == WRITING) { + LOG(("CacheIndex::AsyncGetDiskConsumption - calling immediately")); + // Safe to call the callback under the lock, + // we always post to the main thread. + observer->OnDiskConsumption(index->mIndexStats.Size() << 10); + return NS_OK; + } + + LOG(("CacheIndex::AsyncGetDiskConsumption - remembering callback")); + // Will be called when the index get to the READY state. + index->mDiskConsumptionObservers.AppendElement(observer); + + return NS_OK; +} + +// static +nsresult +CacheIndex::GetIterator(nsILoadContextInfo *aInfo, bool aAddNew, + CacheIndexIterator **_retval) +{ + LOG(("CacheIndex::GetIterator() [info=%p, addNew=%d]", aInfo, aAddNew)); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsRefPtr iter; + if (aInfo) { + iter = new CacheIndexContextIterator(index, aAddNew, aInfo); + } else { + iter = new CacheIndexIterator(index, aAddNew); + } + + iter->AddRecords(index->mFrecencyArray); + + index->mIterators.AppendElement(iter); + iter.swap(*_retval); + return NS_OK; +} + +// static +nsresult +CacheIndex::IsUpToDate(bool *_retval) +{ + LOG(("CacheIndex::IsUpToDate()")); + + nsRefPtr index = gInstance; + + if (!index) { + return NS_ERROR_NOT_INITIALIZED; + } + + CacheIndexAutoLock lock(index); + + if (!index->IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + *_retval = (index->mState == READY || index->mState == WRITING) && + !index->mIndexNeedsUpdate && !index->mShuttingDown; + + LOG(("CacheIndex::IsUpToDate() - returning %p", *_retval)); + return NS_OK; +} + +bool +CacheIndex::IsIndexUsable() +{ + MOZ_ASSERT(mState != INITIAL); + + switch (mState) { + case INITIAL: + case SHUTDOWN: + return false; + + case READING: + case WRITING: + case BUILDING: + case UPDATING: + case READY: + break; + } + + return true; +} + +// static +bool +CacheIndex::IsCollision(CacheIndexEntry *aEntry, + uint32_t aAppId, + bool aAnonymous, + bool aInBrowser) +{ + if (!aEntry->IsInitialized()) { + return false; + } + + if (aEntry->AppId() != aAppId || aEntry->Anonymous() != aAnonymous || + aEntry->InBrowser() != aInBrowser) { + LOG(("CacheIndex::IsCollision() - Collision detected for entry hash=%08x" + "%08x%08x%08x%08x, expected values: appId=%u, anonymous=%d, " + "inBrowser=%d; actual values: appId=%u, anonymous=%d, inBrowser=%d]", + LOGSHA1(aEntry->Hash()), aAppId, aAnonymous, aInBrowser, + aEntry->AppId(), aEntry->Anonymous(), aEntry->InBrowser())); + return true; + } + + return false; +} + +// static +bool +CacheIndex::HasEntryChanged(CacheIndexEntry *aEntry, + const uint32_t *aFrecency, + const uint32_t *aExpirationTime, + const uint32_t *aSize) +{ + if (aFrecency && *aFrecency != aEntry->GetFrecency()) { + return true; + } + + if (aExpirationTime && *aExpirationTime != aEntry->GetExpirationTime()) { + return true; + } + + if (aSize && + (*aSize & CacheIndexEntry::kFileSizeMask) != aEntry->GetFileSize()) { + return true; + } + + return false; +} + +void +CacheIndex::ProcessPendingOperations() +{ + LOG(("CacheIndex::ProcessPendingOperations()")); + + AssertOwnsLock(); + + mPendingUpdates.EnumerateEntries(&CacheIndex::UpdateEntryInIndex, this); + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + + EnsureCorrectStats(); +} + +// static +PLDHashOperator +CacheIndex::UpdateEntryInIndex(CacheIndexEntry *aEntry, void* aClosure) +{ + CacheIndex *index = static_cast(aClosure); + + LOG(("CacheFile::UpdateEntryInIndex() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(aEntry->Hash()))); + + MOZ_ASSERT(aEntry->IsFresh()); + MOZ_ASSERT(aEntry->IsDirty()); + + CacheIndexEntry *entry = index->mIndex.GetEntry(*aEntry->Hash()); + + CacheIndexEntryAutoManage emng(aEntry->Hash(), index); + emng.DoNotSearchInUpdates(); + + if (aEntry->IsRemoved()) { + if (entry) { + if (entry->IsRemoved()) { + MOZ_ASSERT(entry->IsFresh()); + MOZ_ASSERT(entry->IsDirty()); + } else if (!entry->IsDirty() && entry->IsFileEmpty()) { + // Entries with empty file are not stored in index on disk. Just remove + // the entry, but only in case the entry is not dirty, i.e. the entry + // file was empty when we wrote the index. + index->mIndex.RemoveEntry(*aEntry->Hash()); + entry = nullptr; + } else { + entry->MarkRemoved(); + entry->MarkDirty(); + entry->MarkFresh(); + } + } + + return PL_DHASH_REMOVE; + } + + entry = index->mIndex.PutEntry(*aEntry->Hash()); + *entry = *aEntry; + + return PL_DHASH_REMOVE; +} + +bool +CacheIndex::WriteIndexToDiskIfNeeded() +{ + if (mState != READY || mShuttingDown) { + return false; + } + + if (!mLastDumpTime.IsNull() && + (TimeStamp::NowLoRes() - mLastDumpTime).ToMilliseconds() < + kMinDumpInterval) { + return false; + } + + if (mIndexStats.Dirty() < kMinUnwrittenChanges) { + return false; + } + + WriteIndexToDisk(); + return true; +} + +void +CacheIndex::WriteIndexToDisk() +{ + LOG(("CacheIndex::WriteIndexToDisk()")); + mIndexStats.Log(); + + nsresult rv; + + AssertOwnsLock(); + MOZ_ASSERT(mState == READY); + MOZ_ASSERT(!mRWBuf); + MOZ_ASSERT(!mRWHash); + + ChangeState(WRITING); + + mProcessEntries = mIndexStats.ActiveEntriesCount(); + + mIndexFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kTempIndexName), + CacheFileIOManager::SPECIAL_FILE | + CacheFileIOManager::CREATE, + true, + mIndexFileOpener); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::WriteIndexToDisk() - Can't open file [rv=0x%08x]", rv)); + FinishWrite(false); + return; + } + + // Write index header to a buffer, it will be written to disk together with + // records in WriteRecords() once we open the file successfully. + AllocBuffer(); + mRWHash = new CacheHash(); + + CacheIndexHeader *hdr = reinterpret_cast(mRWBuf); + NetworkEndian::writeUint32(&hdr->mVersion, kIndexVersion); + NetworkEndian::writeUint32(&hdr->mTimeStamp, + static_cast(PR_Now() / PR_USEC_PER_SEC)); + NetworkEndian::writeUint32(&hdr->mIsDirty, 1); + + mRWBufPos = sizeof(CacheIndexHeader); + mSkipEntries = 0; +} + +namespace { // anon + +struct WriteRecordsHelper +{ + char *mBuf; + uint32_t mSkip; + uint32_t mProcessMax; + uint32_t mProcessed; +#ifdef DEBUG + bool mHasMore; +#endif +}; + +} // anon + +void +CacheIndex::WriteRecords() +{ + LOG(("CacheIndex::WriteRecords()")); + + nsresult rv; + + AssertOwnsLock(); + MOZ_ASSERT(mState == WRITING); + + int64_t fileOffset; + + if (mSkipEntries) { + MOZ_ASSERT(mRWBufPos == 0); + fileOffset = sizeof(CacheIndexHeader); + fileOffset += sizeof(CacheIndexRecord) * mSkipEntries; + } else { + MOZ_ASSERT(mRWBufPos == sizeof(CacheIndexHeader)); + fileOffset = 0; + } + uint32_t hashOffset = mRWBufPos; + + WriteRecordsHelper data; + data.mBuf = mRWBuf + mRWBufPos; + data.mSkip = mSkipEntries; + data.mProcessMax = (mRWBufSize - mRWBufPos) / sizeof(CacheIndexRecord); + MOZ_ASSERT(data.mProcessMax != 0 || mProcessEntries == 0); // TODO make sure we can write an empty index + data.mProcessed = 0; +#ifdef DEBUG + data.mHasMore = false; +#endif + + mIndex.EnumerateEntries(&CacheIndex::CopyRecordsToRWBuf, &data); + MOZ_ASSERT(mRWBufPos != static_cast(data.mBuf - mRWBuf) || + mProcessEntries == 0); + mRWBufPos = data.mBuf - mRWBuf; + mSkipEntries += data.mProcessed; + MOZ_ASSERT(mSkipEntries <= mProcessEntries); + + mRWHash->Update(mRWBuf + hashOffset, mRWBufPos - hashOffset); + + if (mSkipEntries == mProcessEntries) { + MOZ_ASSERT(!data.mHasMore); + + // We've processed all records + if (mRWBufPos + sizeof(CacheHash::Hash32_t) > mRWBufSize) { + // realloc buffer to spare another write cycle + mRWBufSize = mRWBufPos + sizeof(CacheHash::Hash32_t); + mRWBuf = static_cast(moz_xrealloc(mRWBuf, mRWBufSize)); + } + + NetworkEndian::writeUint32(mRWBuf + mRWBufPos, mRWHash->GetHash()); + mRWBufPos += sizeof(CacheHash::Hash32_t); + } else { + MOZ_ASSERT(data.mHasMore); + } + + rv = CacheFileIOManager::Write(mIndexHandle, fileOffset, mRWBuf, mRWBufPos, + mSkipEntries == mProcessEntries, this); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::WriteRecords() - CacheFileIOManager::Write() failed " + "synchronously [rv=0x%08x]", rv)); + FinishWrite(false); + } + + mRWBufPos = 0; +} + +void +CacheIndex::FinishWrite(bool aSucceeded) +{ + LOG(("CacheIndex::FinishWrite() [succeeded=%d]", aSucceeded)); + + MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == WRITING); + + AssertOwnsLock(); + + mIndexHandle = nullptr; + mRWHash = nullptr; + ReleaseBuffer(); + + if (aSucceeded) { + // Opening of the file must not be in progress if writing succeeded. + MOZ_ASSERT(!mIndexFileOpener); + + mIndex.EnumerateEntries(&CacheIndex::ApplyIndexChanges, this); + mIndexOnDiskIsValid = true; + } else { + if (mIndexFileOpener) { + // If opening of the file is still in progress (e.g. WRITE process was + // canceled by RemoveAll()) then we need to cancel the opener to make sure + // that OnFileOpenedInternal() won't be called. + mIndexFileOpener->Cancel(); + mIndexFileOpener = nullptr; + } + } + + ProcessPendingOperations(); + mIndexStats.Log(); + + if (mState == WRITING) { + ChangeState(READY); + mLastDumpTime = TimeStamp::NowLoRes(); + } +} + +// static +PLDHashOperator +CacheIndex::CopyRecordsToRWBuf(CacheIndexEntry *aEntry, void* aClosure) +{ + if (aEntry->IsRemoved()) { + return PL_DHASH_NEXT; + } + + if (!aEntry->IsInitialized()) { + return PL_DHASH_NEXT; + } + + if (aEntry->IsFileEmpty()) { + return PL_DHASH_NEXT; + } + + WriteRecordsHelper *data = static_cast(aClosure); + if (data->mSkip) { + data->mSkip--; + return PL_DHASH_NEXT; + } + + if (data->mProcessed == data->mProcessMax) { +#ifdef DEBUG + data->mHasMore = true; +#endif + return PL_DHASH_STOP; + } + + aEntry->WriteToBuf(data->mBuf); + data->mBuf += sizeof(CacheIndexRecord); + data->mProcessed++; + + return PL_DHASH_NEXT; +} + +// static +PLDHashOperator +CacheIndex::ApplyIndexChanges(CacheIndexEntry *aEntry, void* aClosure) +{ + CacheIndex *index = static_cast(aClosure); + + CacheIndexEntryAutoManage emng(aEntry->Hash(), index); + + if (aEntry->IsRemoved()) { + emng.DoNotSearchInIndex(); + return PL_DHASH_REMOVE; + } + + if (aEntry->IsDirty()) { + aEntry->ClearDirty(); + } + + return PL_DHASH_NEXT; +} + +nsresult +CacheIndex::GetFile(const nsACString &aName, nsIFile **_retval) +{ + nsresult rv; + + nsCOMPtr file; + rv = mCacheDirectory->Clone(getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = file->AppendNative(aName); + NS_ENSURE_SUCCESS(rv, rv); + + file.swap(*_retval); + return NS_OK; +} + +nsresult +CacheIndex::RemoveFile(const nsACString &aName) +{ + MOZ_ASSERT(mState == SHUTDOWN); + + nsresult rv; + + nsCOMPtr file; + rv = GetFile(aName, getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + bool exists; + rv = file->Exists(&exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (exists) { + rv = file->Remove(false); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::RemoveFile() - Cannot remove old entry file from disk." + "[name=%s]", PromiseFlatCString(aName).get())); + NS_WARNING("Cannot remove old entry file from the disk"); + return rv; + } + } + + return NS_OK; +} + +void +CacheIndex::RemoveIndexFromDisk() +{ + LOG(("CacheIndex::RemoveIndexFromDisk()")); + + RemoveFile(NS_LITERAL_CSTRING(kIndexName)); + RemoveFile(NS_LITERAL_CSTRING(kTempIndexName)); + RemoveFile(NS_LITERAL_CSTRING(kJournalName)); +} + +class WriteLogHelper +{ +public: + WriteLogHelper(PRFileDesc *aFD) + : mStatus(NS_OK) + , mFD(aFD) + , mBufSize(kMaxBufSize) + , mBufPos(0) + { + mHash = new CacheHash(); + mBuf = static_cast(moz_xmalloc(mBufSize)); + } + + ~WriteLogHelper() { + free(mBuf); + } + + nsresult AddEntry(CacheIndexEntry *aEntry); + nsresult Finish(); + +private: + + nsresult FlushBuffer(); + + nsresult mStatus; + PRFileDesc *mFD; + char *mBuf; + uint32_t mBufSize; + int32_t mBufPos; + nsRefPtr mHash; +}; + +nsresult +WriteLogHelper::AddEntry(CacheIndexEntry *aEntry) +{ + nsresult rv; + + if (NS_FAILED(mStatus)) { + return mStatus; + } + + if (mBufPos + sizeof(CacheIndexRecord) > mBufSize) { + mHash->Update(mBuf, mBufPos); + + rv = FlushBuffer(); + if (NS_FAILED(rv)) { + mStatus = rv; + return rv; + } + MOZ_ASSERT(mBufPos + sizeof(CacheIndexRecord) <= mBufSize); + } + + aEntry->WriteToBuf(mBuf + mBufPos); + mBufPos += sizeof(CacheIndexRecord); + + return NS_OK; +} + +nsresult +WriteLogHelper::Finish() +{ + nsresult rv; + + if (NS_FAILED(mStatus)) { + return mStatus; + } + + mHash->Update(mBuf, mBufPos); + if (mBufPos + sizeof(CacheHash::Hash32_t) > mBufSize) { + rv = FlushBuffer(); + if (NS_FAILED(rv)) { + mStatus = rv; + return rv; + } + MOZ_ASSERT(mBufPos + sizeof(CacheHash::Hash32_t) <= mBufSize); + } + + NetworkEndian::writeUint32(mBuf + mBufPos, mHash->GetHash()); + mBufPos += sizeof(CacheHash::Hash32_t); + + rv = FlushBuffer(); + NS_ENSURE_SUCCESS(rv, rv); + + mStatus = NS_ERROR_UNEXPECTED; // Don't allow any other operation + return NS_OK; +} + +nsresult +WriteLogHelper::FlushBuffer() +{ + MOZ_ASSERT(NS_SUCCEEDED(mStatus)); + + int32_t bytesWritten = PR_Write(mFD, mBuf, mBufPos); + + if (bytesWritten != mBufPos) { + return NS_ERROR_FAILURE; + } + + mBufPos = 0; + return NS_OK; +} + +nsresult +CacheIndex::WriteLogToDisk() +{ + LOG(("CacheIndex::WriteLogToDisk()")); + + nsresult rv; + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + MOZ_ASSERT(mState == SHUTDOWN); + + RemoveFile(NS_LITERAL_CSTRING(kTempIndexName)); + + nsCOMPtr indexFile; + rv = GetFile(NS_LITERAL_CSTRING(kIndexName), getter_AddRefs(indexFile)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr logFile; + rv = GetFile(NS_LITERAL_CSTRING(kJournalName), getter_AddRefs(logFile)); + NS_ENSURE_SUCCESS(rv, rv); + + mIndexStats.Log(); + + PRFileDesc *fd = nullptr; + rv = logFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, + 0600, &fd); + NS_ENSURE_SUCCESS(rv, rv); + + WriteLogHelper wlh(fd); + mIndex.EnumerateEntries(&CacheIndex::WriteEntryToLog, &wlh); + + rv = wlh.Finish(); + PR_Close(fd); + NS_ENSURE_SUCCESS(rv, rv); + + rv = indexFile->OpenNSPRFileDesc(PR_RDWR, 0600, &fd); + NS_ENSURE_SUCCESS(rv, rv); + + CacheIndexHeader header; + int32_t bytesRead = PR_Read(fd, &header, sizeof(CacheIndexHeader)); + if (bytesRead != sizeof(CacheIndexHeader)) { + PR_Close(fd); + return NS_ERROR_FAILURE; + } + + NetworkEndian::writeUint32(&header.mIsDirty, 0); + + int64_t offset = PR_Seek64(fd, 0, PR_SEEK_SET); + if (offset == -1) { + PR_Close(fd); + return NS_ERROR_FAILURE; + } + + int32_t bytesWritten = PR_Write(fd, &header, sizeof(CacheIndexHeader)); + PR_Close(fd); + if (bytesWritten != sizeof(CacheIndexHeader)) { + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +// static +PLDHashOperator +CacheIndex::WriteEntryToLog(CacheIndexEntry *aEntry, void* aClosure) +{ + WriteLogHelper *wlh = static_cast(aClosure); + + if (aEntry->IsRemoved() || aEntry->IsDirty()) { + wlh->AddEntry(aEntry); + } + + return PL_DHASH_REMOVE; +} + +void +CacheIndex::ReadIndexFromDisk() +{ + LOG(("CacheIndex::ReadIndexFromDisk()")); + + nsresult rv; + + AssertOwnsLock(); + MOZ_ASSERT(mState == INITIAL); + + ChangeState(READING); + + mIndexFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kIndexName), + CacheFileIOManager::SPECIAL_FILE | + CacheFileIOManager::OPEN, + true, + mIndexFileOpener); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() " + "failed [rv=0x%08x, file=%s]", rv, kIndexName)); + FinishRead(false); + return; + } + + mJournalFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kJournalName), + CacheFileIOManager::SPECIAL_FILE | + CacheFileIOManager::OPEN, + true, + mJournalFileOpener); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() " + "failed [rv=0x%08x, file=%s]", rv, kJournalName)); + FinishRead(false); + } + + mTmpFileOpener = new FileOpenHelper(this); + rv = CacheFileIOManager::OpenFile(NS_LITERAL_CSTRING(kTempIndexName), + CacheFileIOManager::SPECIAL_FILE | + CacheFileIOManager::OPEN, + true, + mTmpFileOpener); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::ReadIndexFromDisk() - CacheFileIOManager::OpenFile() " + "failed [rv=0x%08x, file=%s]", rv, kTempIndexName)); + FinishRead(false); + } +} + +void +CacheIndex::StartReadingIndex() +{ + LOG(("CacheIndex::StartReadingIndex()")); + + nsresult rv; + + AssertOwnsLock(); + + MOZ_ASSERT(mIndexHandle); + MOZ_ASSERT(mState == READING); + MOZ_ASSERT(!mIndexOnDiskIsValid); + MOZ_ASSERT(!mDontMarkIndexClean); + MOZ_ASSERT(!mJournalReadSuccessfully); + MOZ_ASSERT(mIndexHandle->FileSize() >= 0); + + int64_t entriesSize = mIndexHandle->FileSize() - sizeof(CacheIndexHeader) - + sizeof(CacheHash::Hash32_t); + + if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) { + LOG(("CacheIndex::StartReadingIndex() - Index is corrupted")); + FinishRead(false); + return; + } + + AllocBuffer(); + mSkipEntries = 0; + mRWHash = new CacheHash(); + + mRWBufPos = std::min(mRWBufSize, + static_cast(mIndexHandle->FileSize())); + + rv = CacheFileIOManager::Read(mIndexHandle, 0, mRWBuf, mRWBufPos, true, this); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::StartReadingIndex() - CacheFileIOManager::Read() failed " + "synchronously [rv=0x%08x]", rv)); + FinishRead(false); + } +} + +void +CacheIndex::ParseRecords() +{ + LOG(("CacheIndex::ParseRecords()")); + + nsresult rv; + + AssertOwnsLock(); + + uint32_t entryCnt = (mIndexHandle->FileSize() - sizeof(CacheIndexHeader) - + sizeof(CacheHash::Hash32_t)) / sizeof(CacheIndexRecord); + uint32_t pos = 0; + + if (!mSkipEntries) { + CacheIndexHeader *hdr = reinterpret_cast( + moz_xmalloc(sizeof(CacheIndexHeader))); + memcpy(hdr, mRWBuf, sizeof(CacheIndexHeader)); + + if (NetworkEndian::readUint32(&hdr->mVersion) != kIndexVersion) { + free(hdr); + FinishRead(false); + return; + } + + mIndexTimeStamp = NetworkEndian::readUint32(&hdr->mTimeStamp); + + if (NetworkEndian::readUint32(&hdr->mIsDirty)) { + if (mJournalHandle) { + CacheFileIOManager::DoomFile(mJournalHandle, nullptr); + mJournalHandle = nullptr; + } + free(hdr); + } else { + NetworkEndian::writeUint32(&hdr->mIsDirty, 1); + + // Mark index dirty. The buffer is freed by CacheFileIOManager when + // nullptr is passed as the listener and the call doesn't fail + // synchronously. + rv = CacheFileIOManager::Write(mIndexHandle, 0, + reinterpret_cast(hdr), + sizeof(CacheIndexHeader), true, nullptr); + if (NS_FAILED(rv)) { + // This is not fatal, just free the memory + free(hdr); + } + } + + pos += sizeof(CacheIndexHeader); + } + + uint32_t hashOffset = pos; + + while (pos + sizeof(CacheIndexRecord) <= mRWBufPos && + mSkipEntries != entryCnt) { + CacheIndexRecord *rec = reinterpret_cast(mRWBuf + pos); + CacheIndexEntry tmpEntry(&rec->mHash); + tmpEntry.ReadFromBuf(mRWBuf + pos); + + if (tmpEntry.IsDirty() || !tmpEntry.IsInitialized() || + tmpEntry.IsFileEmpty() || tmpEntry.IsFresh() || tmpEntry.IsRemoved()) { + LOG(("CacheIndex::ParseRecords() - Invalid entry found in index, removing" + " whole index [dirty=%d, initialized=%d, fileEmpty=%d, fresh=%d, " + "removed=%d]", tmpEntry.IsDirty(), tmpEntry.IsInitialized(), + tmpEntry.IsFileEmpty(), tmpEntry.IsFresh(), tmpEntry.IsRemoved())); + FinishRead(false); + return; + } + + CacheIndexEntryAutoManage emng(tmpEntry.Hash(), this); + + CacheIndexEntry *entry = mIndex.PutEntry(*tmpEntry.Hash()); + *entry = tmpEntry; + + pos += sizeof(CacheIndexRecord); + mSkipEntries++; + } + + mRWHash->Update(mRWBuf + hashOffset, pos - hashOffset); + + if (pos != mRWBufPos) { + memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos); + mRWBufPos -= pos; + pos = 0; + } + + int64_t fileOffset = sizeof(CacheIndexHeader) + + mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos; + + MOZ_ASSERT(fileOffset <= mIndexHandle->FileSize()); + if (fileOffset == mIndexHandle->FileSize()) { + if (mRWHash->GetHash() != NetworkEndian::readUint32(mRWBuf)) { + LOG(("CacheIndex::ParseRecords() - Hash mismatch, [is %x, should be %x]", + mRWHash->GetHash(), + NetworkEndian::readUint32(mRWBuf))); + FinishRead(false); + return; + } + + mIndexOnDiskIsValid = true; + mJournalReadSuccessfully = false; + + if (mJournalHandle) { + StartReadingJournal(); + } else { + FinishRead(false); + } + + return; + } + + pos = mRWBufPos; + uint32_t toRead = std::min(mRWBufSize - pos, + static_cast(mIndexHandle->FileSize() - + fileOffset)); + mRWBufPos = pos + toRead; + + rv = CacheFileIOManager::Read(mIndexHandle, fileOffset, mRWBuf + pos, toRead, + true, this); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::ParseRecords() - CacheFileIOManager::Read() failed " + "synchronously [rv=0x%08x]", rv)); + FinishRead(false); + return; + } +} + +void +CacheIndex::StartReadingJournal() +{ + LOG(("CacheIndex::StartReadingJournal()")); + + nsresult rv; + + AssertOwnsLock(); + + MOZ_ASSERT(mJournalHandle); + MOZ_ASSERT(mIndexOnDiskIsValid); + MOZ_ASSERT(mTmpJournal.Count() == 0); + MOZ_ASSERT(mJournalHandle->FileSize() >= 0); + + int64_t entriesSize = mJournalHandle->FileSize() - + sizeof(CacheHash::Hash32_t); + + if (entriesSize < 0 || entriesSize % sizeof(CacheIndexRecord)) { + LOG(("CacheIndex::StartReadingJournal() - Journal is corrupted")); + FinishRead(false); + return; + } + + mSkipEntries = 0; + mRWHash = new CacheHash(); + + mRWBufPos = std::min(mRWBufSize, + static_cast(mJournalHandle->FileSize())); + + rv = CacheFileIOManager::Read(mJournalHandle, 0, mRWBuf, mRWBufPos, true, this); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::StartReadingJournal() - CacheFileIOManager::Read() failed" + " synchronously [rv=0x%08x]", rv)); + FinishRead(false); + } +} + +void +CacheIndex::ParseJournal() +{ + LOG(("CacheIndex::ParseRecords()")); + + nsresult rv; + + AssertOwnsLock(); + + uint32_t entryCnt = (mJournalHandle->FileSize() - + sizeof(CacheHash::Hash32_t)) / sizeof(CacheIndexRecord); + + uint32_t pos = 0; + + while (pos + sizeof(CacheIndexRecord) <= mRWBufPos && + mSkipEntries != entryCnt) { + CacheIndexRecord *rec = reinterpret_cast(mRWBuf + pos); + CacheIndexEntry tmpEntry(&rec->mHash); + tmpEntry.ReadFromBuf(mRWBuf + pos); + + CacheIndexEntry *entry = mTmpJournal.PutEntry(*tmpEntry.Hash()); + *entry = tmpEntry; + + if (entry->IsDirty() || entry->IsFresh()) { + LOG(("CacheIndex::ParseJournal() - Invalid entry found in journal, " + "ignoring whole journal [dirty=%d, fresh=%d]", entry->IsDirty(), + entry->IsFresh())); + FinishRead(false); + return; + } + + pos += sizeof(CacheIndexRecord); + mSkipEntries++; + } + + mRWHash->Update(mRWBuf, pos); + + if (pos != mRWBufPos) { + memmove(mRWBuf, mRWBuf + pos, mRWBufPos - pos); + mRWBufPos -= pos; + pos = 0; + } + + int64_t fileOffset = mSkipEntries * sizeof(CacheIndexRecord) + mRWBufPos; + + MOZ_ASSERT(fileOffset <= mJournalHandle->FileSize()); + if (fileOffset == mJournalHandle->FileSize()) { + if (mRWHash->GetHash() != NetworkEndian::readUint32(mRWBuf)) { + LOG(("CacheIndex::ParseJournal() - Hash mismatch, [is %x, should be %x]", + mRWHash->GetHash(), + NetworkEndian::readUint32(mRWBuf))); + FinishRead(false); + return; + } + + mJournalReadSuccessfully = true; + FinishRead(true); + return; + } + + pos = mRWBufPos; + uint32_t toRead = std::min(mRWBufSize - pos, + static_cast(mJournalHandle->FileSize() - + fileOffset)); + mRWBufPos = pos + toRead; + + rv = CacheFileIOManager::Read(mJournalHandle, fileOffset, mRWBuf + pos, + toRead, true, this); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::ParseJournal() - CacheFileIOManager::Read() failed " + "synchronously [rv=0x%08x]", rv)); + FinishRead(false); + return; + } +} + +void +CacheIndex::MergeJournal() +{ + LOG(("CacheIndex::MergeJournal()")); + + AssertOwnsLock(); + + mTmpJournal.EnumerateEntries(&CacheIndex::ProcessJournalEntry, this); + + MOZ_ASSERT(mTmpJournal.Count() == 0); +} + +// static +PLDHashOperator +CacheIndex::ProcessJournalEntry(CacheIndexEntry *aEntry, void* aClosure) +{ + CacheIndex *index = static_cast(aClosure); + + LOG(("CacheFile::ProcessJournalEntry() [hash=%08x%08x%08x%08x%08x]", + LOGSHA1(aEntry->Hash()))); + + CacheIndexEntry *entry = index->mIndex.GetEntry(*aEntry->Hash()); + + CacheIndexEntryAutoManage emng(aEntry->Hash(), index); + + if (aEntry->IsRemoved()) { + if (entry) { + entry->MarkRemoved(); + entry->MarkDirty(); + } + } else { + if (!entry) { + entry = index->mIndex.PutEntry(*aEntry->Hash()); + } + + *entry = *aEntry; + entry->MarkDirty(); + } + + return PL_DHASH_REMOVE; +} + +void +CacheIndex::EnsureNoFreshEntry() +{ +#ifdef DEBUG_STATS + CacheIndexStats debugStats; + debugStats.DisableLogging(); + mIndex.EnumerateEntries(&CacheIndex::SumIndexStats, &debugStats); + MOZ_ASSERT(debugStats.Fresh() == 0); +#endif +} + +void +CacheIndex::EnsureCorrectStats() +{ +#ifdef DEBUG_STATS + MOZ_ASSERT(mPendingUpdates.Count() == 0); + CacheIndexStats debugStats; + debugStats.DisableLogging(); + mIndex.EnumerateEntries(&CacheIndex::SumIndexStats, &debugStats); + MOZ_ASSERT(debugStats == mIndexStats); +#endif +} + +// static +PLDHashOperator +CacheIndex::SumIndexStats(CacheIndexEntry *aEntry, void* aClosure) +{ + CacheIndexStats *stats = static_cast(aClosure); + stats->BeforeChange(nullptr); + stats->AfterChange(aEntry); + return PL_DHASH_NEXT; +} + +void +CacheIndex::FinishRead(bool aSucceeded) +{ + LOG(("CacheIndex::FinishRead() [succeeded=%d]", aSucceeded)); + AssertOwnsLock(); + + MOZ_ASSERT((!aSucceeded && mState == SHUTDOWN) || mState == READING); + + MOZ_ASSERT( + // -> rebuild + (!aSucceeded && !mIndexOnDiskIsValid && !mJournalReadSuccessfully) || + // -> update + (!aSucceeded && mIndexOnDiskIsValid && !mJournalReadSuccessfully) || + // -> ready + (aSucceeded && mIndexOnDiskIsValid && mJournalReadSuccessfully)); + + if (mState == SHUTDOWN) { + RemoveFile(NS_LITERAL_CSTRING(kTempIndexName)); + RemoveFile(NS_LITERAL_CSTRING(kJournalName)); + } else { + if (mIndexHandle && !mIndexOnDiskIsValid) { + CacheFileIOManager::DoomFile(mIndexHandle, nullptr); + } + + if (mJournalHandle) { + CacheFileIOManager::DoomFile(mJournalHandle, nullptr); + } + } + + if (mIndexFileOpener) { + mIndexFileOpener->Cancel(); + mIndexFileOpener = nullptr; + } + if (mJournalFileOpener) { + mJournalFileOpener->Cancel(); + mJournalFileOpener = nullptr; + } + if (mTmpFileOpener) { + mTmpFileOpener->Cancel(); + mTmpFileOpener = nullptr; + } + + mIndexHandle = nullptr; + mJournalHandle = nullptr; + mRWHash = nullptr; + ReleaseBuffer(); + + if (mState == SHUTDOWN) { + return; + } + + if (!mIndexOnDiskIsValid) { + MOZ_ASSERT(mTmpJournal.Count() == 0); + EnsureNoFreshEntry(); + ProcessPendingOperations(); + // Remove all entries that we haven't seen during this session + mIndex.EnumerateEntries(&CacheIndex::RemoveNonFreshEntries, this); + StartUpdatingIndex(true); + return; + } + + if (!mJournalReadSuccessfully) { + mTmpJournal.Clear(); + EnsureNoFreshEntry(); + ProcessPendingOperations(); + StartUpdatingIndex(false); + return; + } + + MergeJournal(); + EnsureNoFreshEntry(); + ProcessPendingOperations(); + mIndexStats.Log(); + + ChangeState(READY); + mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately +} + +// static +void +CacheIndex::DelayedUpdate(nsITimer *aTimer, void *aClosure) +{ + LOG(("CacheIndex::DelayedUpdate()")); + + nsresult rv; + nsRefPtr index = gInstance; + + if (!index) { + return; + } + + CacheIndexAutoLock lock(index); + + index->mUpdateTimer = nullptr; + + if (!index->IsIndexUsable()) { + return; + } + + if (index->mState == READY && index->mShuttingDown) { + return; + } + + // mUpdateEventPending must be false here since StartUpdatingIndex() won't + // schedule timer if it is true. + MOZ_ASSERT(!index->mUpdateEventPending); + if (index->mState != BUILDING && index->mState != UPDATING) { + LOG(("CacheIndex::DelayedUpdate() - Update was canceled")); + return; + } + + // We need to redispatch to run with lower priority + nsRefPtr ioThread = CacheFileIOManager::IOThread(); + MOZ_ASSERT(ioThread); + + index->mUpdateEventPending = true; + rv = ioThread->Dispatch(index, CacheIOThread::INDEX); + if (NS_FAILED(rv)) { + index->mUpdateEventPending = false; + NS_WARNING("CacheIndex::DelayedUpdate() - Can't dispatch event"); + LOG(("CacheIndex::DelayedUpdate() - Can't dispatch event" )); + index->FinishUpdate(false); + } +} + +nsresult +CacheIndex::ScheduleUpdateTimer(uint32_t aDelay) +{ + LOG(("CacheIndex::ScheduleUpdateTimer() [delay=%u]", aDelay)); + + MOZ_ASSERT(!mUpdateTimer); + + nsresult rv; + + nsCOMPtr timer = do_CreateInstance("@mozilla.org/timer;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr ioTarget = CacheFileIOManager::IOTarget(); + MOZ_ASSERT(ioTarget); + + rv = timer->SetTarget(ioTarget); + NS_ENSURE_SUCCESS(rv, rv); + + rv = timer->InitWithFuncCallback(CacheIndex::DelayedUpdate, nullptr, + aDelay, nsITimer::TYPE_ONE_SHOT); + NS_ENSURE_SUCCESS(rv, rv); + + mUpdateTimer.swap(timer); + return NS_OK; +} + +nsresult +CacheIndex::SetupDirectoryEnumerator() +{ + MOZ_ASSERT(!NS_IsMainThread()); + MOZ_ASSERT(!mDirEnumerator); + + nsresult rv; + nsCOMPtr file; + + rv = mCacheDirectory->Clone(getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = file->AppendNative(NS_LITERAL_CSTRING(kEntriesDir)); + NS_ENSURE_SUCCESS(rv, rv); + + bool exists; + rv = file->Exists(&exists); + NS_ENSURE_SUCCESS(rv, rv); + + if (!exists) { + NS_WARNING("CacheIndex::SetupDirectoryEnumerator() - Entries directory " + "doesn't exist!"); + LOG(("CacheIndex::SetupDirectoryEnumerator() - Entries directory doesn't " + "exist!" )); + return NS_ERROR_UNEXPECTED; + } + + nsCOMPtr enumerator; + rv = file->GetDirectoryEntries(getter_AddRefs(enumerator)); + NS_ENSURE_SUCCESS(rv, rv); + + mDirEnumerator = do_QueryInterface(enumerator, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +void +CacheIndex::InitEntryFromDiskData(CacheIndexEntry *aEntry, + CacheFileMetadata *aMetaData, + int64_t aFileSize) +{ + aEntry->InitNew(); + aEntry->MarkDirty(); + aEntry->MarkFresh(); + aEntry->Init(aMetaData->AppId(), aMetaData->IsAnonymous(), + aMetaData->IsInBrowser()); + + uint32_t expirationTime; + aMetaData->GetExpirationTime(&expirationTime); + aEntry->SetExpirationTime(expirationTime); + + uint32_t frecency; + aMetaData->GetFrecency(&frecency); + aEntry->SetFrecency(frecency); + + aEntry->SetFileSize(static_cast( + std::min(static_cast(PR_UINT32_MAX), + (aFileSize + 0x3FF) >> 10))); +} + +bool +CacheIndex::IsUpdatePending() +{ + AssertOwnsLock(); + + if (mUpdateTimer || mUpdateEventPending) { + return true; + } + + return false; +} + +void +CacheIndex::BuildIndex() +{ + LOG(("CacheIndex::BuildIndex()")); + + AssertOwnsLock(); + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + + nsresult rv; + + if (!mDirEnumerator) { + { + // Do not do IO under the lock. + CacheIndexAutoUnlock unlock(this); + rv = SetupDirectoryEnumerator(); + } + if (mState == SHUTDOWN) { + // The index was shut down while we released the lock. FinishUpdate() was + // already called from Shutdown(), so just simply return here. + return; + } + + if (NS_FAILED(rv)) { + FinishUpdate(false); + return; + } + } + + while (true) { + if (CacheIOThread::YieldAndRerun()) { + LOG(("CacheIndex::BuildIndex() - Breaking loop for higher level events.")); + mUpdateEventPending = true; + return; + } + + nsCOMPtr file; + { + // Do not do IO under the lock. + CacheIndexAutoUnlock unlock(this); + rv = mDirEnumerator->GetNextFile(getter_AddRefs(file)); + } + if (mState == SHUTDOWN) { + return; + } + if (!file) { + FinishUpdate(NS_SUCCEEDED(rv)); + return; + } + + nsAutoCString leaf; + rv = file->GetNativeLeafName(leaf); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::BuildIndex() - GetNativeLeafName() failed! Skipping " + "file.")); + mDontMarkIndexClean = true; + continue; + } + + SHA1Sum::Hash hash; + rv = CacheFileIOManager::StrToHash(leaf, &hash); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::BuildIndex() - Filename is not a hash, removing file. " + "[name=%s]", leaf.get())); + file->Remove(false); + continue; + } + + CacheIndexEntry *entry = mIndex.GetEntry(hash); + if (entry && entry->IsRemoved()) { + LOG(("CacheIndex::BuildIndex() - Found file that should not exist. " + "[name=%s]", leaf.get())); + entry->Log(); + MOZ_ASSERT(entry->IsFresh()); + entry = nullptr; + } + +#ifdef DEBUG + nsRefPtr handle; + CacheFileIOManager::gInstance->mHandles.GetHandle(&hash, false, + getter_AddRefs(handle)); +#endif + + if (entry) { + // the entry is up to date + LOG(("CacheIndex::BuildIndex() - Skipping file because the entry is up to" + " date. [name=%s]", leaf.get())); + entry->Log(); + MOZ_ASSERT(entry->IsFresh()); // The entry must be from this session + // there must be an active CacheFile if the entry is not initialized + MOZ_ASSERT(entry->IsInitialized() || handle); + continue; + } + + MOZ_ASSERT(!handle); + + nsRefPtr meta = new CacheFileMetadata(); + int64_t size = 0; + + { + // Do not do IO under the lock. + CacheIndexAutoUnlock unlock(this); + rv = meta->SyncReadMetadata(file); + + if (NS_SUCCEEDED(rv)) { + rv = file->GetFileSize(&size); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::BuildIndex() - Cannot get filesize of file that was" + " successfully parsed. [name=%s]", leaf.get())); + } + } + } + if (mState == SHUTDOWN) { + return; + } + + // Nobody could add the entry while the lock was released since we modify + // the index only on IO thread and this loop is executed on IO thread too. + entry = mIndex.GetEntry(hash); + MOZ_ASSERT(!entry || entry->IsRemoved()); + + if (NS_FAILED(rv)) { + LOG(("CacheIndex::BuildIndex() - CacheFileMetadata::SyncReadMetadata() " + "failed, removing file. [name=%s]", leaf.get())); + file->Remove(false); + } else { + CacheIndexEntryAutoManage entryMng(&hash, this); + entry = mIndex.PutEntry(hash); + InitEntryFromDiskData(entry, meta, size); + LOG(("CacheIndex::BuildIndex() - Added entry to index. [hash=%s]", + leaf.get())); + entry->Log(); + } + } + + NS_NOTREACHED("We should never get here"); +} + +bool +CacheIndex::StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState) +{ + // Start updating process when we are in or we are switching to READY state + // and index needs update, but not during shutdown or when removing all + // entries. + if ((mState == READY || aSwitchingToReadyState) && mIndexNeedsUpdate && + !mShuttingDown && !mRemovingAll) { + LOG(("CacheIndex::StartUpdatingIndexIfNeeded() - starting update process")); + mIndexNeedsUpdate = false; + StartUpdatingIndex(false); + return true; + } + + return false; +} + +void +CacheIndex::StartUpdatingIndex(bool aRebuild) +{ + LOG(("CacheIndex::StartUpdatingIndex() [rebuild=%d]", aRebuild)); + + AssertOwnsLock(); + + nsresult rv; + + mIndexStats.Log(); + + ChangeState(aRebuild ? BUILDING : UPDATING); + mDontMarkIndexClean = false; + + if (mShuttingDown || mRemovingAll) { + FinishUpdate(false); + return; + } + + if (IsUpdatePending()) { + LOG(("CacheIndex::StartUpdatingIndex() - Update is already pending")); + return; + } + + uint32_t elapsed = (TimeStamp::NowLoRes() - mStartTime).ToMilliseconds(); + if (elapsed < kUpdateIndexStartDelay) { + LOG(("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, " + "scheduling timer to fire in %u ms.", elapsed, + kUpdateIndexStartDelay - elapsed)); + rv = ScheduleUpdateTimer(kUpdateIndexStartDelay - elapsed); + if (NS_SUCCEEDED(rv)) { + return; + } + + LOG(("CacheIndex::StartUpdatingIndex() - ScheduleUpdateTimer() failed. " + "Starting update immediately.")); + } else { + LOG(("CacheIndex::StartUpdatingIndex() - %u ms elapsed since startup, " + "starting update now.", elapsed)); + } + + nsRefPtr ioThread = CacheFileIOManager::IOThread(); + MOZ_ASSERT(ioThread); + + // We need to dispatch an event even if we are on IO thread since we need to + // update the index with the correct priority. + mUpdateEventPending = true; + rv = ioThread->Dispatch(this, CacheIOThread::INDEX); + if (NS_FAILED(rv)) { + mUpdateEventPending = false; + NS_WARNING("CacheIndex::StartUpdatingIndex() - Can't dispatch event"); + LOG(("CacheIndex::StartUpdatingIndex() - Can't dispatch event" )); + FinishUpdate(false); + } +} + +void +CacheIndex::UpdateIndex() +{ + LOG(("CacheIndex::UpdateIndex()")); + + AssertOwnsLock(); + + MOZ_ASSERT(mPendingUpdates.Count() == 0); + + nsresult rv; + + if (!mDirEnumerator) { + { + // Do not do IO under the lock. + CacheIndexAutoUnlock unlock(this); + rv = SetupDirectoryEnumerator(); + } + if (mState == SHUTDOWN) { + // The index was shut down while we released the lock. FinishUpdate() was + // already called from Shutdown(), so just simply return here. + return; + } + + if (NS_FAILED(rv)) { + FinishUpdate(false); + return; + } + } + + while (true) { + if (CacheIOThread::YieldAndRerun()) { + LOG(("CacheIndex::UpdateIndex() - Breaking loop for higher level " + "events.")); + mUpdateEventPending = true; + return; + } + + nsCOMPtr file; + { + // Do not do IO under the lock. + CacheIndexAutoUnlock unlock(this); + rv = mDirEnumerator->GetNextFile(getter_AddRefs(file)); + } + if (mState == SHUTDOWN) { + return; + } + if (!file) { + FinishUpdate(NS_SUCCEEDED(rv)); + return; + } + + nsAutoCString leaf; + rv = file->GetNativeLeafName(leaf); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::UpdateIndex() - GetNativeLeafName() failed! Skipping " + "file.")); + mDontMarkIndexClean = true; + continue; + } + + SHA1Sum::Hash hash; + rv = CacheFileIOManager::StrToHash(leaf, &hash); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::UpdateIndex() - Filename is not a hash, removing file. " + "[name=%s]", leaf.get())); + file->Remove(false); + continue; + } + + CacheIndexEntry *entry = mIndex.GetEntry(hash); + if (entry && entry->IsRemoved()) { + if (entry->IsFresh()) { + LOG(("CacheIndex::UpdateIndex() - Found file that should not exist. " + "[name=%s]", leaf.get())); + entry->Log(); + } + entry = nullptr; + } + +#ifdef DEBUG + nsRefPtr handle; + CacheFileIOManager::gInstance->mHandles.GetHandle(&hash, false, + getter_AddRefs(handle)); +#endif + + if (entry && entry->IsFresh()) { + // the entry is up to date + LOG(("CacheIndex::UpdateIndex() - Skipping file because the entry is up " + " to date. [name=%s]", leaf.get())); + entry->Log(); + // there must be an active CacheFile if the entry is not initialized + MOZ_ASSERT(entry->IsInitialized() || handle); + continue; + } + + MOZ_ASSERT(!handle); + + if (entry) { + PRTime lastModifiedTime; + { + // Do not do IO under the lock. + CacheIndexAutoUnlock unlock(this); + rv = file->GetLastModifiedTime(&lastModifiedTime); + } + if (mState == SHUTDOWN) { + return; + } + if (NS_FAILED(rv)) { + LOG(("CacheIndex::UpdateIndex() - Cannot get lastModifiedTime. " + "[name=%s]", leaf.get())); + // Assume the file is newer than index + } else { + if (mIndexTimeStamp > (lastModifiedTime / PR_MSEC_PER_SEC)) { + LOG(("CacheIndex::UpdateIndex() - Skipping file because of last " + "modified time. [name=%s, indexTimeStamp=%u, " + "lastModifiedTime=%u]", leaf.get(), mIndexTimeStamp, + lastModifiedTime / PR_MSEC_PER_SEC)); + + CacheIndexEntryAutoManage entryMng(&hash, this); + entry->MarkFresh(); + continue; + } + } + } + + nsRefPtr meta = new CacheFileMetadata(); + int64_t size = 0; + + { + // Do not do IO under the lock. + CacheIndexAutoUnlock unlock(this); + rv = meta->SyncReadMetadata(file); + + if (NS_SUCCEEDED(rv)) { + rv = file->GetFileSize(&size); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::UpdateIndex() - Cannot get filesize of file that " + "was successfully parsed. [name=%s]", leaf.get())); + } + } + } + if (mState == SHUTDOWN) { + return; + } + + // Nobody could add the entry while the lock was released since we modify + // the index only on IO thread and this loop is executed on IO thread too. + entry = mIndex.GetEntry(hash); + MOZ_ASSERT(!entry || !entry->IsFresh()); + + CacheIndexEntryAutoManage entryMng(&hash, this); + + if (NS_FAILED(rv)) { + LOG(("CacheIndex::UpdateIndex() - CacheFileMetadata::SyncReadMetadata() " + "failed, removing file. [name=%s]", leaf.get())); + file->Remove(false); + if (entry) { + entry->MarkRemoved(); + entry->MarkFresh(); + entry->MarkDirty(); + } + } else { + entry = mIndex.PutEntry(hash); + InitEntryFromDiskData(entry, meta, size); + LOG(("CacheIndex::UpdateIndex() - Added/updated entry to/in index. " + "[hash=%s]", leaf.get())); + entry->Log(); + } + } + + NS_NOTREACHED("We should never get here"); +} + +void +CacheIndex::FinishUpdate(bool aSucceeded) +{ + LOG(("CacheIndex::FinishUpdate() [succeeded=%d]", aSucceeded)); + + MOZ_ASSERT(mState == UPDATING || mState == BUILDING || + (!aSucceeded && mState == SHUTDOWN)); + + AssertOwnsLock(); + + if (mDirEnumerator) { + if (NS_IsMainThread()) { + LOG(("CacheIndex::FinishUpdate() - posting of PreShutdownInternal failed?" + " Cannot safely release mDirEnumerator, leaking it!")); + NS_WARNING(("CacheIndex::FinishUpdate() - Leaking mDirEnumerator!")); + // This can happen only in case dispatching event to IO thread failed in + // CacheIndex::PreShutdown(). + mDirEnumerator.forget(); // Leak it since dir enumerator is not threadsafe + } else { + mDirEnumerator->Close(); + mDirEnumerator = nullptr; + } + } + + if (!aSucceeded) { + mDontMarkIndexClean = true; + } + + if (mState == SHUTDOWN) { + return; + } + + if (mState == UPDATING && aSucceeded) { + // If we've iterated over all entries successfully then all entries that + // really exist on the disk are now marked as fresh. All non-fresh entries + // don't exist anymore and must be removed from the index. + mIndex.EnumerateEntries(&CacheIndex::RemoveNonFreshEntries, this); + } + + // Make sure we won't start update. If the build or update failed, there is no + // reason to believe that it will succeed next time. + mIndexNeedsUpdate = false; + + ChangeState(READY); + mLastDumpTime = TimeStamp::NowLoRes(); // Do not dump new index immediately +} + +// static +PLDHashOperator +CacheIndex::RemoveNonFreshEntries(CacheIndexEntry *aEntry, void* aClosure) +{ + if (aEntry->IsFresh()) { + return PL_DHASH_NEXT; + } + + LOG(("CacheFile::RemoveNonFreshEntries() - Removing entry. " + "[hash=%08x%08x%08x%08x%08x]", LOGSHA1(aEntry->Hash()))); + + CacheIndex *index = static_cast(aClosure); + + CacheIndexEntryAutoManage emng(aEntry->Hash(), index); + emng.DoNotSearchInIndex(); + + return PL_DHASH_REMOVE; +} + +#ifdef PR_LOGGING +// static +char const * +CacheIndex::StateString(EState aState) +{ + switch (aState) { + case INITIAL: return "INITIAL"; + case READING: return "READING"; + case WRITING: return "WRITING"; + case BUILDING: return "BUILDING"; + case UPDATING: return "UPDATING"; + case READY: return "READY"; + case SHUTDOWN: return "SHUTDOWN"; + } + + MOZ_ASSERT(false, "Unexpected state!"); + return "?"; +} +#endif + +void +CacheIndex::ChangeState(EState aNewState) +{ + LOG(("CacheIndex::ChangeState() changing state %s -> %s", StateString(mState), + StateString(aNewState))); + + // All pending updates should be processed before changing state + MOZ_ASSERT(mPendingUpdates.Count() == 0); + + // PreShutdownInternal() should change the state to READY from every state. It + // may go through different states, but once we are in READY state the only + // possible transition is to SHUTDOWN state. + MOZ_ASSERT(!mShuttingDown || mState != READY || aNewState == SHUTDOWN); + + // Start updating process when switching to READY state if needed + if (aNewState == READY && StartUpdatingIndexIfNeeded(true)) { + return; + } + + // Try to evict entries over limit everytime we're leaving state READING, + // BUILDING or UPDATING, but not during shutdown or when removing all + // entries. + if (!mShuttingDown && !mRemovingAll && aNewState != SHUTDOWN && + (mState == READING || mState == BUILDING || mState == UPDATING)) { + CacheFileIOManager::EvictIfOverLimit(); + } + + mState = aNewState; + + if (mState != SHUTDOWN) { + CacheFileIOManager::CacheIndexStateChanged(); + } + + if (mState == READY && mDiskConsumptionObservers.Length()) { + for (uint32_t i = 0; i < mDiskConsumptionObservers.Length(); ++i) { + DiskConsumptionObserver* o = mDiskConsumptionObservers[i]; + // Safe to call under the lock. We always post to the main thread. + o->OnDiskConsumption(mIndexStats.Size() << 10); + } + + mDiskConsumptionObservers.Clear(); + } +} + +void +CacheIndex::AllocBuffer() +{ + switch (mState) { + case WRITING: + mRWBufSize = sizeof(CacheIndexHeader) + sizeof(CacheHash::Hash32_t) + + mProcessEntries * sizeof(CacheIndexRecord); + if (mRWBufSize > kMaxBufSize) { + mRWBufSize = kMaxBufSize; + } + break; + case READING: + mRWBufSize = kMaxBufSize; + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + mRWBuf = static_cast(moz_xmalloc(mRWBufSize)); +} + +void +CacheIndex::ReleaseBuffer() +{ + if (!mRWBuf) { + return; + } + + free(mRWBuf); + mRWBuf = nullptr; + mRWBufSize = 0; + mRWBufPos = 0; +} + +namespace { // anon + +class FrecencyComparator +{ +public: + bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const { + return a->mFrecency == b->mFrecency; + } + bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const { + // Place entries with frecency 0 at the end of the array. + if (a->mFrecency == 0) { + return false; + } + if (b->mFrecency == 0) { + return true; + } + return a->mFrecency < b->mFrecency; + } +}; + +class ExpirationComparator +{ +public: + bool Equals(CacheIndexRecord* a, CacheIndexRecord* b) const { + return a->mExpirationTime == b->mExpirationTime; + } + bool LessThan(CacheIndexRecord* a, CacheIndexRecord* b) const { + return a->mExpirationTime < b->mExpirationTime; + } +}; + +} // anon + +void +CacheIndex::InsertRecordToFrecencyArray(CacheIndexRecord *aRecord) +{ + LOG(("CacheIndex::InsertRecordToFrecencyArray() [record=%p, hash=%08x%08x%08x" + "%08x%08x]", aRecord, LOGSHA1(aRecord->mHash))); + + MOZ_ASSERT(!mFrecencyArray.Contains(aRecord)); + mFrecencyArray.InsertElementSorted(aRecord, FrecencyComparator()); +} + +void +CacheIndex::InsertRecordToExpirationArray(CacheIndexRecord *aRecord) +{ + LOG(("CacheIndex::InsertRecordToExpirationArray() [record=%p, hash=%08x%08x" + "%08x%08x%08x]", aRecord, LOGSHA1(aRecord->mHash))); + + MOZ_ASSERT(!mExpirationArray.Contains(aRecord)); + mExpirationArray.InsertElementSorted(aRecord, ExpirationComparator()); +} + +void +CacheIndex::RemoveRecordFromFrecencyArray(CacheIndexRecord *aRecord) +{ + LOG(("CacheIndex::RemoveRecordFromFrecencyArray() [record=%p]", aRecord)); + + DebugOnly removed; + removed = mFrecencyArray.RemoveElement(aRecord); + MOZ_ASSERT(removed); +} + +void +CacheIndex::RemoveRecordFromExpirationArray(CacheIndexRecord *aRecord) +{ + LOG(("CacheIndex::RemoveRecordFromExpirationArray() [record=%p]", aRecord)); + + DebugOnly removed; + removed = mExpirationArray.RemoveElement(aRecord); + MOZ_ASSERT(removed); +} + +void +CacheIndex::AddRecordToIterators(CacheIndexRecord *aRecord) +{ + AssertOwnsLock(); + + for (uint32_t i = 0; i < mIterators.Length(); ++i) { + // Add a new record only when iterator is supposed to be updated. + if (mIterators[i]->ShouldBeNewAdded()) { + mIterators[i]->AddRecord(aRecord); + } + } +} + +void +CacheIndex::RemoveRecordFromIterators(CacheIndexRecord *aRecord) +{ + AssertOwnsLock(); + + for (uint32_t i = 0; i < mIterators.Length(); ++i) { + // Remove the record from iterator always, it makes no sence to return + // non-existing entries. Also the pointer to the record is no longer valid + // once the entry is removed from index. + mIterators[i]->RemoveRecord(aRecord); + } +} + +void +CacheIndex::ReplaceRecordInIterators(CacheIndexRecord *aOldRecord, + CacheIndexRecord *aNewRecord) +{ + AssertOwnsLock(); + + for (uint32_t i = 0; i < mIterators.Length(); ++i) { + // We have to replace the record always since the pointer is no longer + // valid after this point. NOTE: Replacing the record doesn't mean that + // a new entry was added, it just means that the data in the entry was + // changed (e.g. a file size) and we had to track this change in + // mPendingUpdates since mIndex was read-only. + mIterators[i]->ReplaceRecord(aOldRecord, aNewRecord); + } +} + +nsresult +CacheIndex::Run() +{ + LOG(("CacheIndex::Run()")); + + CacheIndexAutoLock lock(this); + + if (!IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (mState == READY && mShuttingDown) { + return NS_OK; + } + + mUpdateEventPending = false; + + switch (mState) { + case BUILDING: + BuildIndex(); + break; + case UPDATING: + UpdateIndex(); + break; + default: + LOG(("CacheIndex::Run() - Update/Build was canceled")); + } + + return NS_OK; +} + +nsresult +CacheIndex::OnFileOpenedInternal(FileOpenHelper *aOpener, + CacheFileHandle *aHandle, nsresult aResult) +{ + LOG(("CacheIndex::OnFileOpenedInternal() [opener=%p, handle=%p, " + "result=0x%08x]", aOpener, aHandle, aResult)); + + nsresult rv; + + AssertOwnsLock(); + + if (!IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (mState == READY && mShuttingDown) { + return NS_OK; + } + + switch (mState) { + case WRITING: + MOZ_ASSERT(aOpener == mIndexFileOpener); + mIndexFileOpener = nullptr; + + if (NS_FAILED(aResult)) { + LOG(("CacheIndex::OnFileOpenedInternal() - Can't open index file for " + "writing [rv=0x%08x]", aResult)); + FinishWrite(false); + } else { + mIndexHandle = aHandle; + WriteRecords(); + } + break; + case READING: + if (aOpener == mIndexFileOpener) { + mIndexFileOpener = nullptr; + + if (NS_SUCCEEDED(aResult)) { + if (aHandle->FileSize() == 0) { + FinishRead(false); + CacheFileIOManager::DoomFile(aHandle, nullptr); + break; + } else { + mIndexHandle = aHandle; + } + } else { + FinishRead(false); + break; + } + } else if (aOpener == mJournalFileOpener) { + mJournalFileOpener = nullptr; + mJournalHandle = aHandle; + } else if (aOpener == mTmpFileOpener) { + mTmpFileOpener = nullptr; + mTmpHandle = aHandle; + } else { + MOZ_ASSERT(false, "Unexpected state!"); + } + + if (mIndexFileOpener || mJournalFileOpener || mTmpFileOpener) { + // Some opener still didn't finish + break; + } + + // We fail and cancel all other openers when we opening index file fails. + MOZ_ASSERT(mIndexHandle); + + if (mTmpHandle) { + CacheFileIOManager::DoomFile(mTmpHandle, nullptr); + mTmpHandle = nullptr; + + if (mJournalHandle) { // this shouldn't normally happen + LOG(("CacheIndex::OnFileOpenedInternal() - Unexpected state, all " + "files [%s, %s, %s] should never exist. Removing whole index.", + kIndexName, kJournalName, kTempIndexName)); + FinishRead(false); + break; + } + } + + if (mJournalHandle) { + // Rename journal to make sure we update index on next start in case + // firefox crashes + rv = CacheFileIOManager::RenameFile( + mJournalHandle, NS_LITERAL_CSTRING(kTempIndexName), this); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::OnFileOpenedInternal() - CacheFileIOManager::" + "RenameFile() failed synchronously [rv=0x%08x]", rv)); + FinishRead(false); + break; + } + } else { + StartReadingIndex(); + } + + break; + default: + MOZ_ASSERT(false, "Unexpected state!"); + } + + return NS_OK; +} + +nsresult +CacheIndex::OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) +{ + MOZ_CRASH("CacheIndex::OnFileOpened should not be called!"); + return NS_ERROR_UNEXPECTED; +} + +nsresult +CacheIndex::OnDataWritten(CacheFileHandle *aHandle, const char *aBuf, + nsresult aResult) +{ + LOG(("CacheIndex::OnDataWritten() [handle=%p, result=0x%08x]", aHandle, + aResult)); + + nsresult rv; + + CacheIndexAutoLock lock(this); + + if (!IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (mState == READY && mShuttingDown) { + return NS_OK; + } + + switch (mState) { + case WRITING: + if (mIndexHandle != aHandle) { + LOG(("CacheIndex::OnDataWritten() - ignoring notification since it " + "belongs to previously canceled operation [state=%d]", mState)); + break; + } + + if (NS_FAILED(aResult)) { + FinishWrite(false); + } else { + if (mSkipEntries == mProcessEntries) { + rv = CacheFileIOManager::RenameFile(mIndexHandle, + NS_LITERAL_CSTRING(kIndexName), + this); + if (NS_FAILED(rv)) { + LOG(("CacheIndex::OnDataWritten() - CacheFileIOManager::" + "RenameFile() failed synchronously [rv=0x%08x]", rv)); + FinishWrite(false); + } + } else { + WriteRecords(); + } + } + break; + default: + // Writing was canceled. + LOG(("CacheIndex::OnDataWritten() - ignoring notification since the " + "operation was previously canceled [state=%d]", mState)); + } + + return NS_OK; +} + +nsresult +CacheIndex::OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult) +{ + LOG(("CacheIndex::OnDataRead() [handle=%p, result=0x%08x]", aHandle, + aResult)); + + CacheIndexAutoLock lock(this); + + if (!IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + switch (mState) { + case READING: + MOZ_ASSERT(mIndexHandle == aHandle || mJournalHandle == aHandle); + + if (NS_FAILED(aResult)) { + FinishRead(false); + } else { + if (!mIndexOnDiskIsValid) { + ParseRecords(); + } else { + ParseJournal(); + } + } + break; + default: + // Reading was canceled. + LOG(("CacheIndex::OnDataRead() - ignoring notification since the " + "operation was previously canceled [state=%d]", mState)); + } + + return NS_OK; +} + +nsresult +CacheIndex::OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) +{ + MOZ_CRASH("CacheIndex::OnFileDoomed should not be called!"); + return NS_ERROR_UNEXPECTED; +} + +nsresult +CacheIndex::OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) +{ + MOZ_CRASH("CacheIndex::OnEOFSet should not be called!"); + return NS_ERROR_UNEXPECTED; +} + +nsresult +CacheIndex::OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) +{ + LOG(("CacheIndex::OnFileRenamed() [handle=%p, result=0x%08x]", aHandle, + aResult)); + + CacheIndexAutoLock lock(this); + + if (!IsIndexUsable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (mState == READY && mShuttingDown) { + return NS_OK; + } + + switch (mState) { + case WRITING: + // This is a result of renaming the new index written to tmpfile to index + // file. This is the last step when writing the index and the whole + // writing process is successful iff renaming was successful. + + if (mIndexHandle != aHandle) { + LOG(("CacheIndex::OnFileRenamed() - ignoring notification since it " + "belongs to previously canceled operation [state=%d]", mState)); + break; + } + + FinishWrite(NS_SUCCEEDED(aResult)); + break; + case READING: + // This is a result of renaming journal file to tmpfile. It is renamed + // before we start reading index and journal file and it should normally + // succeed. If it fails give up reading of index. + + if (mJournalHandle != aHandle) { + LOG(("CacheIndex::OnFileRenamed() - ignoring notification since it " + "belongs to previously canceled operation [state=%d]", mState)); + break; + } + + if (NS_FAILED(aResult)) { + FinishRead(false); + } else { + StartReadingIndex(); + } + break; + default: + // Reading/writing was canceled. + LOG(("CacheIndex::OnFileRenamed() - ignoring notification since the " + "operation was previously canceled [state=%d]", mState)); + } + + return NS_OK; +} + +// Memory reporting + +namespace { // anon + +size_t +CollectIndexEntryMemory(CacheIndexEntry* aEntry, + mozilla::MallocSizeOf mallocSizeOf, + void *arg) +{ + return aEntry->SizeOfExcludingThis(mallocSizeOf); +} + +} // anon + +size_t +CacheIndex::SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const +{ + CacheIndexAutoLock lock(const_cast(this)); + + size_t n = 0; + nsCOMPtr sizeOf; + + // mIndexHandle and mJournalHandle are reported via SizeOfHandlesRunnable + // in CacheFileIOManager::SizeOfExcludingThisInternal as part of special + // handles array. + + sizeOf = do_QueryInterface(mCacheDirectory); + if (sizeOf) { + n += sizeOf->SizeOfIncludingThis(mallocSizeOf); + } + + sizeOf = do_QueryInterface(mUpdateTimer); + if (sizeOf) { + n += sizeOf->SizeOfIncludingThis(mallocSizeOf); + } + + n += mallocSizeOf(mRWBuf); + n += mallocSizeOf(mRWHash); + + n += mIndex.SizeOfExcludingThis(&CollectIndexEntryMemory, mallocSizeOf); + n += mPendingUpdates.SizeOfExcludingThis(&CollectIndexEntryMemory, mallocSizeOf); + n += mTmpJournal.SizeOfExcludingThis(&CollectIndexEntryMemory, mallocSizeOf); + + // mFrecencyArray and mExpirationArray items are reported by + // mIndex/mPendingUpdates + n += mFrecencyArray.SizeOfExcludingThis(mallocSizeOf); + n += mExpirationArray.SizeOfExcludingThis(mallocSizeOf); + n += mDiskConsumptionObservers.SizeOfExcludingThis(mallocSizeOf); + + return n; +} + +// static +size_t +CacheIndex::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) +{ + if (!gInstance) + return 0; + + return gInstance->SizeOfExcludingThisInternal(mallocSizeOf); +} + +// static +size_t +CacheIndex::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) +{ + return mallocSizeOf(gInstance) + SizeOfExcludingThis(mallocSizeOf); +} + +} // net +} // mozilla