1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/netwerk/cache2/CacheIndex.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,1052 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +#ifndef CacheIndex__h__
1.9 +#define CacheIndex__h__
1.10 +
1.11 +#include "CacheLog.h"
1.12 +#include "CacheFileIOManager.h"
1.13 +#include "nsIRunnable.h"
1.14 +#include "CacheHashUtils.h"
1.15 +#include "nsICacheStorageService.h"
1.16 +#include "nsICacheEntry.h"
1.17 +#include "nsILoadContextInfo.h"
1.18 +#include "nsTHashtable.h"
1.19 +#include "nsThreadUtils.h"
1.20 +#include "nsWeakReference.h"
1.21 +#include "mozilla/SHA1.h"
1.22 +#include "mozilla/Mutex.h"
1.23 +#include "mozilla/Endian.h"
1.24 +#include "mozilla/TimeStamp.h"
1.25 +
1.26 +class nsIFile;
1.27 +class nsIDirectoryEnumerator;
1.28 +class nsITimer;
1.29 +
1.30 +
1.31 +#ifdef DEBUG
1.32 +#define DEBUG_STATS 1
1.33 +#endif
1.34 +
1.35 +namespace mozilla {
1.36 +namespace net {
1.37 +
1.38 +class CacheFileMetadata;
1.39 +class FileOpenHelper;
1.40 +class CacheIndexIterator;
1.41 +
1.42 +typedef struct {
1.43 + // Version of the index. The index must be ignored and deleted when the file
1.44 + // on disk was written with a newer version.
1.45 + uint32_t mVersion;
1.46 +
1.47 + // Timestamp of time when the last successful write of the index started.
1.48 + // During update process we use this timestamp for a quick validation of entry
1.49 + // files. If last modified time of the file is lower than this timestamp, we
1.50 + // skip parsing of such file since the information in index should be up to
1.51 + // date.
1.52 + uint32_t mTimeStamp;
1.53 +
1.54 + // We set this flag as soon as possible after parsing index during startup
1.55 + // and clean it after we write journal to disk during shutdown. We ignore the
1.56 + // journal and start update process whenever this flag is set during index
1.57 + // parsing.
1.58 + uint32_t mIsDirty;
1.59 +} CacheIndexHeader;
1.60 +
1.61 +struct CacheIndexRecord {
1.62 + SHA1Sum::Hash mHash;
1.63 + uint32_t mFrecency;
1.64 + uint32_t mExpirationTime;
1.65 + uint32_t mAppId;
1.66 +
1.67 + /*
1.68 + * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized
1.69 + * 0100 0000 0000 0000 0000 0000 0000 0000 : anonymous
1.70 + * 0010 0000 0000 0000 0000 0000 0000 0000 : inBrowser
1.71 + * 0001 0000 0000 0000 0000 0000 0000 0000 : removed
1.72 + * 0000 1000 0000 0000 0000 0000 0000 0000 : dirty
1.73 + * 0000 0100 0000 0000 0000 0000 0000 0000 : fresh
1.74 + * 0000 0011 0000 0000 0000 0000 0000 0000 : reserved
1.75 + * 0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB)
1.76 + */
1.77 + uint32_t mFlags;
1.78 +
1.79 + CacheIndexRecord()
1.80 + : mFrecency(0)
1.81 + , mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME)
1.82 + , mAppId(nsILoadContextInfo::NO_APP_ID)
1.83 + , mFlags(0)
1.84 + {}
1.85 +};
1.86 +
1.87 +class CacheIndexEntry : public PLDHashEntryHdr
1.88 +{
1.89 +public:
1.90 + typedef const SHA1Sum::Hash& KeyType;
1.91 + typedef const SHA1Sum::Hash* KeyTypePointer;
1.92 +
1.93 + CacheIndexEntry(KeyTypePointer aKey)
1.94 + {
1.95 + MOZ_COUNT_CTOR(CacheIndexEntry);
1.96 + mRec = new CacheIndexRecord();
1.97 + LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec.get()));
1.98 + memcpy(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash));
1.99 + }
1.100 + CacheIndexEntry(const CacheIndexEntry& aOther)
1.101 + {
1.102 + NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!");
1.103 + }
1.104 + ~CacheIndexEntry()
1.105 + {
1.106 + MOZ_COUNT_DTOR(CacheIndexEntry);
1.107 + LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]",
1.108 + mRec.get()));
1.109 + }
1.110 +
1.111 + // KeyEquals(): does this entry match this key?
1.112 + bool KeyEquals(KeyTypePointer aKey) const
1.113 + {
1.114 + return memcmp(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0;
1.115 + }
1.116 +
1.117 + // KeyToPointer(): Convert KeyType to KeyTypePointer
1.118 + static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }
1.119 +
1.120 + // HashKey(): calculate the hash number
1.121 + static PLDHashNumber HashKey(KeyTypePointer aKey)
1.122 + {
1.123 + return (reinterpret_cast<const uint32_t *>(aKey))[0];
1.124 + }
1.125 +
1.126 + // ALLOW_MEMMOVE can we move this class with memmove(), or do we have
1.127 + // to use the copy constructor?
1.128 + enum { ALLOW_MEMMOVE = true };
1.129 +
1.130 + bool operator==(const CacheIndexEntry& aOther) const
1.131 + {
1.132 + return KeyEquals(&aOther.mRec->mHash);
1.133 + }
1.134 +
1.135 + CacheIndexEntry& operator=(const CacheIndexEntry& aOther)
1.136 + {
1.137 + MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash,
1.138 + sizeof(SHA1Sum::Hash)) == 0);
1.139 + mRec->mFrecency = aOther.mRec->mFrecency;
1.140 + mRec->mExpirationTime = aOther.mRec->mExpirationTime;
1.141 + mRec->mAppId = aOther.mRec->mAppId;
1.142 + mRec->mFlags = aOther.mRec->mFlags;
1.143 + return *this;
1.144 + }
1.145 +
1.146 + void InitNew()
1.147 + {
1.148 + mRec->mFrecency = 0;
1.149 + mRec->mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;
1.150 + mRec->mAppId = nsILoadContextInfo::NO_APP_ID;
1.151 + mRec->mFlags = 0;
1.152 + }
1.153 +
1.154 + void Init(uint32_t aAppId, bool aAnonymous, bool aInBrowser)
1.155 + {
1.156 + MOZ_ASSERT(mRec->mFrecency == 0);
1.157 + MOZ_ASSERT(mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME);
1.158 + MOZ_ASSERT(mRec->mAppId == nsILoadContextInfo::NO_APP_ID);
1.159 + // When we init the entry it must be fresh and may be dirty
1.160 + MOZ_ASSERT((mRec->mFlags & ~kDirtyMask) == kFreshMask);
1.161 +
1.162 + mRec->mAppId = aAppId;
1.163 + mRec->mFlags |= kInitializedMask;
1.164 + if (aAnonymous) {
1.165 + mRec->mFlags |= kAnonymousMask;
1.166 + }
1.167 + if (aInBrowser) {
1.168 + mRec->mFlags |= kInBrowserMask;
1.169 + }
1.170 + }
1.171 +
1.172 + const SHA1Sum::Hash * Hash() { return &mRec->mHash; }
1.173 +
1.174 + bool IsInitialized() { return !!(mRec->mFlags & kInitializedMask); }
1.175 +
1.176 + uint32_t AppId() { return mRec->mAppId; }
1.177 + bool Anonymous() { return !!(mRec->mFlags & kAnonymousMask); }
1.178 + bool InBrowser() { return !!(mRec->mFlags & kInBrowserMask); }
1.179 +
1.180 + bool IsRemoved() { return !!(mRec->mFlags & kRemovedMask); }
1.181 + void MarkRemoved() { mRec->mFlags |= kRemovedMask; }
1.182 +
1.183 + bool IsDirty() { return !!(mRec->mFlags & kDirtyMask); }
1.184 + void MarkDirty() { mRec->mFlags |= kDirtyMask; }
1.185 + void ClearDirty() { mRec->mFlags &= ~kDirtyMask; }
1.186 +
1.187 + bool IsFresh() { return !!(mRec->mFlags & kFreshMask); }
1.188 + void MarkFresh() { mRec->mFlags |= kFreshMask; }
1.189 +
1.190 + void SetFrecency(uint32_t aFrecency) { mRec->mFrecency = aFrecency; }
1.191 + uint32_t GetFrecency() { return mRec->mFrecency; }
1.192 +
1.193 + void SetExpirationTime(uint32_t aExpirationTime)
1.194 + {
1.195 + mRec->mExpirationTime = aExpirationTime;
1.196 + }
1.197 + uint32_t GetExpirationTime() { return mRec->mExpirationTime; }
1.198 +
1.199 + // Sets filesize in kilobytes.
1.200 + void SetFileSize(uint32_t aFileSize)
1.201 + {
1.202 + if (aFileSize > kFileSizeMask) {
1.203 + LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, "
1.204 + "truncating to %u", kFileSizeMask));
1.205 + aFileSize = kFileSizeMask;
1.206 + }
1.207 + mRec->mFlags &= ~kFileSizeMask;
1.208 + mRec->mFlags |= aFileSize;
1.209 + }
1.210 + // Returns filesize in kilobytes.
1.211 + uint32_t GetFileSize() { return mRec->mFlags & kFileSizeMask; }
1.212 + bool IsFileEmpty() { return GetFileSize() == 0; }
1.213 +
1.214 + void WriteToBuf(void *aBuf)
1.215 + {
1.216 + CacheIndexRecord *dst = reinterpret_cast<CacheIndexRecord *>(aBuf);
1.217 +
1.218 + // Copy the whole record to the buffer.
1.219 + memcpy(aBuf, mRec, sizeof(CacheIndexRecord));
1.220 +
1.221 + // Dirty and fresh flags should never go to disk, since they make sense only
1.222 + // during current session.
1.223 + dst->mFlags &= ~kDirtyMask;
1.224 + dst->mFlags &= ~kFreshMask;
1.225 +
1.226 +#if defined(IS_LITTLE_ENDIAN)
1.227 + // Data in the buffer are in machine byte order and we want them in network
1.228 + // byte order.
1.229 + NetworkEndian::writeUint32(&dst->mFrecency, dst->mFrecency);
1.230 + NetworkEndian::writeUint32(&dst->mExpirationTime, dst->mExpirationTime);
1.231 + NetworkEndian::writeUint32(&dst->mAppId, dst->mAppId);
1.232 + NetworkEndian::writeUint32(&dst->mFlags, dst->mFlags);
1.233 +#endif
1.234 + }
1.235 +
1.236 + void ReadFromBuf(void *aBuf)
1.237 + {
1.238 + CacheIndexRecord *src= reinterpret_cast<CacheIndexRecord *>(aBuf);
1.239 + MOZ_ASSERT(memcmp(&mRec->mHash, &src->mHash,
1.240 + sizeof(SHA1Sum::Hash)) == 0);
1.241 +
1.242 + mRec->mFrecency = NetworkEndian::readUint32(&src->mFrecency);
1.243 + mRec->mExpirationTime = NetworkEndian::readUint32(&src->mExpirationTime);
1.244 + mRec->mAppId = NetworkEndian::readUint32(&src->mAppId);
1.245 + mRec->mFlags = NetworkEndian::readUint32(&src->mFlags);
1.246 + }
1.247 +
1.248 + void Log() {
1.249 + LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u,"
1.250 + " initialized=%u, removed=%u, dirty=%u, anonymous=%u, inBrowser=%u, "
1.251 + "appId=%u, frecency=%u, expirationTime=%u, size=%u]",
1.252 + this, LOGSHA1(mRec->mHash), IsFresh(), IsInitialized(), IsRemoved(),
1.253 + IsDirty(), Anonymous(), InBrowser(), AppId(), GetFrecency(),
1.254 + GetExpirationTime(), GetFileSize()));
1.255 + }
1.256 +
1.257 + static bool RecordMatchesLoadContextInfo(CacheIndexRecord *aRec,
1.258 + nsILoadContextInfo *aInfo)
1.259 + {
1.260 + if (!aInfo->IsPrivate() &&
1.261 + aInfo->AppId() == aRec->mAppId &&
1.262 + aInfo->IsAnonymous() == !!(aRec->mFlags & kAnonymousMask) &&
1.263 + aInfo->IsInBrowserElement() == !!(aRec->mFlags & kInBrowserMask)) {
1.264 + return true;
1.265 + }
1.266 +
1.267 + return false;
1.268 + }
1.269 +
1.270 + // Memory reporting
1.271 + size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const
1.272 + {
1.273 + return mallocSizeOf(mRec.get());
1.274 + }
1.275 +
1.276 + size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
1.277 + {
1.278 + return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
1.279 + }
1.280 +
1.281 +private:
1.282 + friend class CacheIndex;
1.283 + friend class CacheIndexEntryAutoManage;
1.284 +
1.285 + static const uint32_t kInitializedMask = 0x80000000;
1.286 + static const uint32_t kAnonymousMask = 0x40000000;
1.287 + static const uint32_t kInBrowserMask = 0x20000000;
1.288 +
1.289 + // This flag is set when the entry was removed. We need to keep this
1.290 + // information in memory until we write the index file.
1.291 + static const uint32_t kRemovedMask = 0x10000000;
1.292 +
1.293 + // This flag is set when the information in memory is not in sync with the
1.294 + // information in index file on disk.
1.295 + static const uint32_t kDirtyMask = 0x08000000;
1.296 +
1.297 + // This flag is set when the information about the entry is fresh, i.e.
1.298 + // we've created or opened this entry during this session, or we've seen
1.299 + // this entry during update or build process.
1.300 + static const uint32_t kFreshMask = 0x04000000;
1.301 +
1.302 + static const uint32_t kReservedMask = 0x03000000;
1.303 +
1.304 + // FileSize in kilobytes
1.305 + static const uint32_t kFileSizeMask = 0x00FFFFFF;
1.306 +
1.307 + nsAutoPtr<CacheIndexRecord> mRec;
1.308 +};
1.309 +
1.310 +class CacheIndexStats
1.311 +{
1.312 +public:
1.313 + CacheIndexStats()
1.314 + : mCount(0)
1.315 + , mNotInitialized(0)
1.316 + , mRemoved(0)
1.317 + , mDirty(0)
1.318 + , mFresh(0)
1.319 + , mEmpty(0)
1.320 + , mSize(0)
1.321 +#ifdef DEBUG
1.322 + , mStateLogged(false)
1.323 + , mDisableLogging(false)
1.324 +#endif
1.325 + {
1.326 + }
1.327 +
1.328 + bool operator==(const CacheIndexStats& aOther) const
1.329 + {
1.330 + return
1.331 +#ifdef DEBUG
1.332 + aOther.mStateLogged == mStateLogged &&
1.333 +#endif
1.334 + aOther.mCount == mCount &&
1.335 + aOther.mNotInitialized == mNotInitialized &&
1.336 + aOther.mRemoved == mRemoved &&
1.337 + aOther.mDirty == mDirty &&
1.338 + aOther.mFresh == mFresh &&
1.339 + aOther.mEmpty == mEmpty &&
1.340 + aOther.mSize == mSize;
1.341 + }
1.342 +
1.343 +#ifdef DEBUG
1.344 + void DisableLogging() {
1.345 + mDisableLogging = true;
1.346 + }
1.347 +#endif
1.348 +
1.349 + void Log() {
1.350 + LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, "
1.351 + "dirty=%u, fresh=%u, empty=%u, size=%u]", mCount, mNotInitialized,
1.352 + mRemoved, mDirty, mFresh, mEmpty, mSize));
1.353 + }
1.354 +
1.355 + void Clear() {
1.356 + MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!");
1.357 +
1.358 + mCount = 0;
1.359 + mNotInitialized = 0;
1.360 + mRemoved = 0;
1.361 + mDirty = 0;
1.362 + mFresh = 0;
1.363 + mEmpty = 0;
1.364 + mSize = 0;
1.365 + }
1.366 +
1.367 +#ifdef DEBUG
1.368 + bool StateLogged() {
1.369 + return mStateLogged;
1.370 + }
1.371 +#endif
1.372 +
1.373 + uint32_t Count() {
1.374 + MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!");
1.375 + return mCount;
1.376 + }
1.377 +
1.378 + uint32_t Dirty() {
1.379 + MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!");
1.380 + return mDirty;
1.381 + }
1.382 +
1.383 + uint32_t Fresh() {
1.384 + MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!");
1.385 + return mFresh;
1.386 + }
1.387 +
1.388 + uint32_t ActiveEntriesCount() {
1.389 + MOZ_ASSERT(!mStateLogged, "CacheIndexStats::ActiveEntriesCount() - state "
1.390 + "logged!");
1.391 + return mCount - mRemoved - mNotInitialized - mEmpty;
1.392 + }
1.393 +
1.394 + uint32_t Size() {
1.395 + MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!");
1.396 + return mSize;
1.397 + }
1.398 +
1.399 + void BeforeChange(CacheIndexEntry *aEntry) {
1.400 +#ifdef DEBUG_STATS
1.401 + if (!mDisableLogging) {
1.402 + LOG(("CacheIndexStats::BeforeChange()"));
1.403 + Log();
1.404 + }
1.405 +#endif
1.406 +
1.407 + MOZ_ASSERT(!mStateLogged, "CacheIndexStats::BeforeChange() - state "
1.408 + "logged!");
1.409 +#ifdef DEBUG
1.410 + mStateLogged = true;
1.411 +#endif
1.412 + if (aEntry) {
1.413 + MOZ_ASSERT(mCount);
1.414 + mCount--;
1.415 + if (aEntry->IsDirty()) {
1.416 + MOZ_ASSERT(mDirty);
1.417 + mDirty--;
1.418 + }
1.419 + if (aEntry->IsFresh()) {
1.420 + MOZ_ASSERT(mFresh);
1.421 + mFresh--;
1.422 + }
1.423 + if (aEntry->IsRemoved()) {
1.424 + MOZ_ASSERT(mRemoved);
1.425 + mRemoved--;
1.426 + } else {
1.427 + if (!aEntry->IsInitialized()) {
1.428 + MOZ_ASSERT(mNotInitialized);
1.429 + mNotInitialized--;
1.430 + } else {
1.431 + if (aEntry->IsFileEmpty()) {
1.432 + MOZ_ASSERT(mEmpty);
1.433 + mEmpty--;
1.434 + } else {
1.435 + MOZ_ASSERT(mSize >= aEntry->GetFileSize());
1.436 + mSize -= aEntry->GetFileSize();
1.437 + }
1.438 + }
1.439 + }
1.440 + }
1.441 + }
1.442 +
1.443 + void AfterChange(CacheIndexEntry *aEntry) {
1.444 + MOZ_ASSERT(mStateLogged, "CacheIndexStats::AfterChange() - state not "
1.445 + "logged!");
1.446 +#ifdef DEBUG
1.447 + mStateLogged = false;
1.448 +#endif
1.449 + if (aEntry) {
1.450 + ++mCount;
1.451 + if (aEntry->IsDirty()) {
1.452 + mDirty++;
1.453 + }
1.454 + if (aEntry->IsFresh()) {
1.455 + mFresh++;
1.456 + }
1.457 + if (aEntry->IsRemoved()) {
1.458 + mRemoved++;
1.459 + } else {
1.460 + if (!aEntry->IsInitialized()) {
1.461 + mNotInitialized++;
1.462 + } else {
1.463 + if (aEntry->IsFileEmpty()) {
1.464 + mEmpty++;
1.465 + } else {
1.466 + mSize += aEntry->GetFileSize();
1.467 + }
1.468 + }
1.469 + }
1.470 + }
1.471 +
1.472 +#ifdef DEBUG_STATS
1.473 + if (!mDisableLogging) {
1.474 + LOG(("CacheIndexStats::AfterChange()"));
1.475 + Log();
1.476 + }
1.477 +#endif
1.478 + }
1.479 +
1.480 +private:
1.481 + uint32_t mCount;
1.482 + uint32_t mNotInitialized;
1.483 + uint32_t mRemoved;
1.484 + uint32_t mDirty;
1.485 + uint32_t mFresh;
1.486 + uint32_t mEmpty;
1.487 + uint32_t mSize;
1.488 +#ifdef DEBUG
1.489 + // We completely remove the data about an entry from the stats in
1.490 + // BeforeChange() and set this flag to true. The entry is then modified,
1.491 + // deleted or created and the data is again put into the stats and this flag
1.492 + // set to false. Statistics must not be read during this time since the
1.493 + // information is not correct.
1.494 + bool mStateLogged;
1.495 +
1.496 + // Disables logging in this instance of CacheIndexStats
1.497 + bool mDisableLogging;
1.498 +#endif
1.499 +};
1.500 +
1.501 +class CacheIndex : public CacheFileIOListener
1.502 + , public nsIRunnable
1.503 +{
1.504 +public:
1.505 + NS_DECL_THREADSAFE_ISUPPORTS
1.506 + NS_DECL_NSIRUNNABLE
1.507 +
1.508 + CacheIndex();
1.509 +
1.510 + static nsresult Init(nsIFile *aCacheDirectory);
1.511 + static nsresult PreShutdown();
1.512 + static nsresult Shutdown();
1.513 +
1.514 + // Following methods can be called only on IO thread.
1.515 +
1.516 + // Add entry to the index. The entry shouldn't be present in index. This
1.517 + // method is called whenever a new handle for a new entry file is created. The
1.518 + // newly created entry is not initialized and it must be either initialized
1.519 + // with InitEntry() or removed with RemoveEntry().
1.520 + static nsresult AddEntry(const SHA1Sum::Hash *aHash);
1.521 +
1.522 + // Inform index about an existing entry that should be present in index. This
1.523 + // method is called whenever a new handle for an existing entry file is
1.524 + // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry()
1.525 + // must be called on the entry, since the entry is not initizlized if the
1.526 + // index is outdated.
1.527 + static nsresult EnsureEntryExists(const SHA1Sum::Hash *aHash);
1.528 +
1.529 + // Initialize the entry. It MUST be present in index. Call to AddEntry() or
1.530 + // EnsureEntryExists() must precede the call to this method.
1.531 + static nsresult InitEntry(const SHA1Sum::Hash *aHash,
1.532 + uint32_t aAppId,
1.533 + bool aAnonymous,
1.534 + bool aInBrowser);
1.535 +
1.536 + // Remove entry from index. The entry should be present in index.
1.537 + static nsresult RemoveEntry(const SHA1Sum::Hash *aHash);
1.538 +
1.539 + // Update some information in entry. The entry MUST be present in index and
1.540 + // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to
1.541 + // InitEntry() must precede the call to this method.
1.542 + // Pass nullptr if the value didn't change.
1.543 + static nsresult UpdateEntry(const SHA1Sum::Hash *aHash,
1.544 + const uint32_t *aFrecency,
1.545 + const uint32_t *aExpirationTime,
1.546 + const uint32_t *aSize);
1.547 +
1.548 + // Remove all entries from the index. Called when clearing the whole cache.
1.549 + static nsresult RemoveAll();
1.550 +
1.551 + enum EntryStatus {
1.552 + EXISTS = 0,
1.553 + DOES_NOT_EXIST = 1,
1.554 + DO_NOT_KNOW = 2
1.555 + };
1.556 +
1.557 + // Returns status of the entry in index for the given key. It can be called
1.558 + // on any thread.
1.559 + static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval);
1.560 +
1.561 + // Returns a hash of the least important entry that should be evicted if the
1.562 + // cache size is over limit and also returns a total number of all entries in
1.563 + // the index.
1.564 + static nsresult GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt);
1.565 +
1.566 + // Returns cache size in kB.
1.567 + static nsresult GetCacheSize(uint32_t *_retval);
1.568 +
1.569 + // Asynchronously gets the disk cache size, used for display in the UI.
1.570 + static nsresult AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver);
1.571 +
1.572 + // Returns an iterator that returns entries matching a given context that were
1.573 + // present in the index at the time this method was called. If aAddNew is true
1.574 + // then the iterator will also return entries created after this call.
1.575 + // NOTE: When some entry is removed from index it is removed also from the
1.576 + // iterator regardless what aAddNew was passed.
1.577 + static nsresult GetIterator(nsILoadContextInfo *aInfo, bool aAddNew,
1.578 + CacheIndexIterator **_retval);
1.579 +
1.580 + // Returns true if we _think_ that the index is up to date. I.e. the state is
1.581 + // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false.
1.582 + static nsresult IsUpToDate(bool *_retval);
1.583 +
1.584 + // Memory reporting
1.585 + static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
1.586 + static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
1.587 +
1.588 +private:
1.589 + friend class CacheIndexEntryAutoManage;
1.590 + friend class CacheIndexAutoLock;
1.591 + friend class CacheIndexAutoUnlock;
1.592 + friend class FileOpenHelper;
1.593 + friend class CacheIndexIterator;
1.594 +
1.595 + virtual ~CacheIndex();
1.596 +
1.597 + NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult);
1.598 + nsresult OnFileOpenedInternal(FileOpenHelper *aOpener,
1.599 + CacheFileHandle *aHandle, nsresult aResult);
1.600 + NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
1.601 + nsresult aResult);
1.602 + NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult);
1.603 + NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
1.604 + NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult);
1.605 + NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult);
1.606 +
1.607 + void Lock();
1.608 + void Unlock();
1.609 + void AssertOwnsLock();
1.610 +
1.611 + nsresult InitInternal(nsIFile *aCacheDirectory);
1.612 + void PreShutdownInternal();
1.613 +
1.614 + // This method returns false when index is not initialized or is shut down.
1.615 + bool IsIndexUsable();
1.616 +
1.617 + // This method checks whether the entry has the same values of appId,
1.618 + // isAnonymous and isInBrowser. We don't expect to find a collision since
1.619 + // these values are part of the key that we hash and we use a strong hash
1.620 + // function.
1.621 + static bool IsCollision(CacheIndexEntry *aEntry,
1.622 + uint32_t aAppId,
1.623 + bool aAnonymous,
1.624 + bool aInBrowser);
1.625 +
1.626 + // Checks whether any of the information about the entry has changed.
1.627 + static bool HasEntryChanged(CacheIndexEntry *aEntry,
1.628 + const uint32_t *aFrecency,
1.629 + const uint32_t *aExpirationTime,
1.630 + const uint32_t *aSize);
1.631 +
1.632 + // Merge all pending operations from mPendingUpdates into mIndex.
1.633 + void ProcessPendingOperations();
1.634 + static PLDHashOperator UpdateEntryInIndex(CacheIndexEntry *aEntry,
1.635 + void* aClosure);
1.636 +
1.637 + // Following methods perform writing of the index file.
1.638 + //
1.639 + // The index is written periodically, but not earlier than once in
1.640 + // kMinDumpInterval and there must be at least kMinUnwrittenChanges
1.641 + // differences between index on disk and in memory. Index is always first
1.642 + // written to a temporary file and the old index file is replaced when the
1.643 + // writing process succeeds.
1.644 + //
1.645 + // Starts writing of index when both limits (minimal delay between writes and
1.646 + // minimum number of changes in index) were exceeded.
1.647 + bool WriteIndexToDiskIfNeeded();
1.648 + // Starts writing of index file.
1.649 + void WriteIndexToDisk();
1.650 + // Serializes part of mIndex hashtable to the write buffer a writes the buffer
1.651 + // to the file.
1.652 + void WriteRecords();
1.653 + // Finalizes writing process.
1.654 + void FinishWrite(bool aSucceeded);
1.655 +
1.656 + static PLDHashOperator CopyRecordsToRWBuf(CacheIndexEntry *aEntry,
1.657 + void* aClosure);
1.658 + static PLDHashOperator ApplyIndexChanges(CacheIndexEntry *aEntry,
1.659 + void* aClosure);
1.660 +
1.661 + // Following methods perform writing of the journal during shutdown. All these
1.662 + // methods must be called only during shutdown since they write/delete files
1.663 + // directly on the main thread instead of using CacheFileIOManager that does
1.664 + // it asynchronously on IO thread. Journal contains only entries that are
1.665 + // dirty, i.e. changes that are not present in the index file on the disk.
1.666 + // When the log is written successfully, the dirty flag in index file is
1.667 + // cleared.
1.668 + nsresult GetFile(const nsACString &aName, nsIFile **_retval);
1.669 + nsresult RemoveFile(const nsACString &aName);
1.670 + void RemoveIndexFromDisk();
1.671 + // Writes journal to the disk and clears dirty flag in index header.
1.672 + nsresult WriteLogToDisk();
1.673 +
1.674 + static PLDHashOperator WriteEntryToLog(CacheIndexEntry *aEntry,
1.675 + void* aClosure);
1.676 +
1.677 + // Following methods perform reading of the index from the disk.
1.678 + //
1.679 + // Index is read at startup just after initializing the CacheIndex. There are
1.680 + // 3 files used when manipulating with index: index file, journal file and
1.681 + // a temporary file. All files contain the hash of the data, so we can check
1.682 + // whether the content is valid and complete. Index file contains also a dirty
1.683 + // flag in the index header which is unset on a clean shutdown. During opening
1.684 + // and reading of the files we determine the status of the whole index from
1.685 + // the states of the separate files. Following table shows all possible
1.686 + // combinations:
1.687 + //
1.688 + // index, journal, tmpfile
1.689 + // M * * - index is missing -> BUILD
1.690 + // I * * - index is invalid -> BUILD
1.691 + // D * * - index is dirty -> UPDATE
1.692 + // C M * - index is dirty -> UPDATE
1.693 + // C I * - unexpected state -> UPDATE
1.694 + // C V E - unexpected state -> UPDATE
1.695 + // C V M - index is up to date -> READY
1.696 + //
1.697 + // where the letters mean:
1.698 + // * - any state
1.699 + // E - file exists
1.700 + // M - file is missing
1.701 + // I - data is invalid (parsing failed or hash didn't match)
1.702 + // D - dirty (data in index file is correct, but dirty flag is set)
1.703 + // C - clean (index file is clean)
1.704 + // V - valid (data in journal file is correct)
1.705 + //
1.706 + // Note: We accept the data from journal only when the index is up to date as
1.707 + // a whole (i.e. C,V,M state).
1.708 + //
1.709 + // We rename the journal file to the temporary file as soon as possible after
1.710 + // initial test to ensure that we start update process on the next startup if
1.711 + // FF crashes during parsing of the index.
1.712 + //
1.713 + // Initiates reading index from disk.
1.714 + void ReadIndexFromDisk();
1.715 + // Starts reading data from index file.
1.716 + void StartReadingIndex();
1.717 + // Parses data read from index file.
1.718 + void ParseRecords();
1.719 + // Starts reading data from journal file.
1.720 + void StartReadingJournal();
1.721 + // Parses data read from journal file.
1.722 + void ParseJournal();
1.723 + // Merges entries from journal into mIndex.
1.724 + void MergeJournal();
1.725 + // In debug build this method checks that we have no fresh entry in mIndex
1.726 + // after we finish reading index and before we process pending operations.
1.727 + void EnsureNoFreshEntry();
1.728 + // In debug build this method is called after processing pending operations
1.729 + // to make sure mIndexStats contains correct information.
1.730 + void EnsureCorrectStats();
1.731 + static PLDHashOperator SumIndexStats(CacheIndexEntry *aEntry, void* aClosure);
1.732 + // Finalizes reading process.
1.733 + void FinishRead(bool aSucceeded);
1.734 +
1.735 + static PLDHashOperator ProcessJournalEntry(CacheIndexEntry *aEntry,
1.736 + void* aClosure);
1.737 +
1.738 + // Following methods perform updating and building of the index.
1.739 + // Timer callback that starts update or build process.
1.740 + static void DelayedUpdate(nsITimer *aTimer, void *aClosure);
1.741 + // Posts timer event that start update or build process.
1.742 + nsresult ScheduleUpdateTimer(uint32_t aDelay);
1.743 + nsresult SetupDirectoryEnumerator();
1.744 + void InitEntryFromDiskData(CacheIndexEntry *aEntry,
1.745 + CacheFileMetadata *aMetaData,
1.746 + int64_t aFileSize);
1.747 + // Returns true when either a timer is scheduled or event is posted.
1.748 + bool IsUpdatePending();
1.749 + // Iterates through all files in entries directory that we didn't create/open
1.750 + // during this session, parses them and adds the entries to the index.
1.751 + void BuildIndex();
1.752 +
1.753 + bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState = false);
1.754 + // Starts update or build process or fires a timer when it is too early after
1.755 + // startup.
1.756 + void StartUpdatingIndex(bool aRebuild);
1.757 + // Iterates through all files in entries directory that we didn't create/open
1.758 + // during this session and theirs last modified time is newer than timestamp
1.759 + // in the index header. Parses the files and adds the entries to the index.
1.760 + void UpdateIndex();
1.761 + // Finalizes update or build process.
1.762 + void FinishUpdate(bool aSucceeded);
1.763 +
1.764 + static PLDHashOperator RemoveNonFreshEntries(CacheIndexEntry *aEntry,
1.765 + void* aClosure);
1.766 +
1.767 + enum EState {
1.768 + // Initial state in which the index is not usable
1.769 + // Possible transitions:
1.770 + // -> READING
1.771 + INITIAL = 0,
1.772 +
1.773 + // Index is being read from the disk.
1.774 + // Possible transitions:
1.775 + // -> INITIAL - We failed to dispatch a read event.
1.776 + // -> BUILDING - No or corrupted index file was found.
1.777 + // -> UPDATING - No or corrupted journal file was found.
1.778 + // - Dirty flag was set in index header.
1.779 + // -> READY - Index was read successfully or was interrupted by
1.780 + // pre-shutdown.
1.781 + // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1.782 + READING = 1,
1.783 +
1.784 + // Index is being written to the disk.
1.785 + // Possible transitions:
1.786 + // -> READY - Writing of index finished or was interrupted by
1.787 + // pre-shutdown..
1.788 + // -> UPDATING - Writing of index finished, but index was found outdated
1.789 + // during writing.
1.790 + // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1.791 + WRITING = 2,
1.792 +
1.793 + // Index is being build.
1.794 + // Possible transitions:
1.795 + // -> READY - Building of index finished or was interrupted by
1.796 + // pre-shutdown.
1.797 + // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1.798 + BUILDING = 3,
1.799 +
1.800 + // Index is being updated.
1.801 + // Possible transitions:
1.802 + // -> READY - Updating of index finished or was interrupted by
1.803 + // pre-shutdown.
1.804 + // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
1.805 + UPDATING = 4,
1.806 +
1.807 + // Index is ready.
1.808 + // Possible transitions:
1.809 + // -> UPDATING - Index was found outdated.
1.810 + // -> SHUTDOWN - Index is shutting down.
1.811 + READY = 5,
1.812 +
1.813 + // Index is shutting down.
1.814 + SHUTDOWN = 6
1.815 + };
1.816 +
1.817 +#ifdef PR_LOGGING
1.818 + static char const * StateString(EState aState);
1.819 +#endif
1.820 + void ChangeState(EState aNewState);
1.821 +
1.822 + // Allocates and releases buffer used for reading and writing index.
1.823 + void AllocBuffer();
1.824 + void ReleaseBuffer();
1.825 +
1.826 + // Methods used by CacheIndexEntryAutoManage to keep the arrays up to date.
1.827 + void InsertRecordToFrecencyArray(CacheIndexRecord *aRecord);
1.828 + void InsertRecordToExpirationArray(CacheIndexRecord *aRecord);
1.829 + void RemoveRecordFromFrecencyArray(CacheIndexRecord *aRecord);
1.830 + void RemoveRecordFromExpirationArray(CacheIndexRecord *aRecord);
1.831 +
1.832 + // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date.
1.833 + void AddRecordToIterators(CacheIndexRecord *aRecord);
1.834 + void RemoveRecordFromIterators(CacheIndexRecord *aRecord);
1.835 + void ReplaceRecordInIterators(CacheIndexRecord *aOldRecord,
1.836 + CacheIndexRecord *aNewRecord);
1.837 +
1.838 + // Memory reporting (private part)
1.839 + size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
1.840 +
1.841 + static CacheIndex *gInstance;
1.842 +
1.843 + nsCOMPtr<nsIFile> mCacheDirectory;
1.844 +
1.845 + mozilla::Mutex mLock;
1.846 + EState mState;
1.847 + // Timestamp of time when the index was initialized. We use it to delay
1.848 + // initial update or build of index.
1.849 + TimeStamp mStartTime;
1.850 + // Set to true in PreShutdown(), it is checked on variaous places to prevent
1.851 + // starting any process (write, update, etc.) during shutdown.
1.852 + bool mShuttingDown;
1.853 + // When set to true, update process should start as soon as possible. This
1.854 + // flag is set whenever we find some inconsistency which would be fixed by
1.855 + // update process. The flag is checked always when switching to READY state.
1.856 + // To make sure we start the update process as soon as possible, methods that
1.857 + // set this flag should also call StartUpdatingIndexIfNeeded() to cover the
1.858 + // case when we are currently in READY state.
1.859 + bool mIndexNeedsUpdate;
1.860 + // Set at the beginning of RemoveAll() which clears the whole index. When
1.861 + // removing all entries we must stop any pending reading, writing, updating or
1.862 + // building operation. This flag is checked at various places and it prevents
1.863 + // we won't start another operation (e.g. canceling reading of the index would
1.864 + // normally start update or build process)
1.865 + bool mRemovingAll;
1.866 + // Whether the index file on disk exists and is valid.
1.867 + bool mIndexOnDiskIsValid;
1.868 + // When something goes wrong during updating or building process, we don't
1.869 + // mark index clean (and also don't write journal) to ensure that update or
1.870 + // build will be initiated on the next start.
1.871 + bool mDontMarkIndexClean;
1.872 + // Timestamp value from index file. It is used during update process to skip
1.873 + // entries that were last modified before this timestamp.
1.874 + uint32_t mIndexTimeStamp;
1.875 + // Timestamp of last time the index was dumped to disk.
1.876 + // NOTE: The index might not be necessarily dumped at this time. The value
1.877 + // is used to schedule next dump of the index.
1.878 + TimeStamp mLastDumpTime;
1.879 +
1.880 + // Timer of delayed update/build.
1.881 + nsCOMPtr<nsITimer> mUpdateTimer;
1.882 + // True when build or update event is posted
1.883 + bool mUpdateEventPending;
1.884 +
1.885 + // Helper members used when reading/writing index from/to disk.
1.886 + // Contains number of entries that should be skipped:
1.887 + // - in hashtable when writing index because they were already written
1.888 + // - in index file when reading index because they were already read
1.889 + uint32_t mSkipEntries;
1.890 + // Number of entries that should be written to disk. This is number of entries
1.891 + // in hashtable that are initialized and are not marked as removed when writing
1.892 + // begins.
1.893 + uint32_t mProcessEntries;
1.894 + char *mRWBuf;
1.895 + uint32_t mRWBufSize;
1.896 + uint32_t mRWBufPos;
1.897 + nsRefPtr<CacheHash> mRWHash;
1.898 +
1.899 + // Reading of journal succeeded if true.
1.900 + bool mJournalReadSuccessfully;
1.901 +
1.902 + // Handle used for writing and reading index file.
1.903 + nsRefPtr<CacheFileHandle> mIndexHandle;
1.904 + // Handle used for reading journal file.
1.905 + nsRefPtr<CacheFileHandle> mJournalHandle;
1.906 + // Used to check the existence of the file during reading process.
1.907 + nsRefPtr<CacheFileHandle> mTmpHandle;
1.908 +
1.909 + nsRefPtr<FileOpenHelper> mIndexFileOpener;
1.910 + nsRefPtr<FileOpenHelper> mJournalFileOpener;
1.911 + nsRefPtr<FileOpenHelper> mTmpFileOpener;
1.912 +
1.913 + // Directory enumerator used when building and updating index.
1.914 + nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator;
1.915 +
1.916 + // Main index hashtable.
1.917 + nsTHashtable<CacheIndexEntry> mIndex;
1.918 +
1.919 + // We cannot add, remove or change any entry in mIndex in states READING and
1.920 + // WRITING. We track all changes in mPendingUpdates during these states.
1.921 + nsTHashtable<CacheIndexEntry> mPendingUpdates;
1.922 +
1.923 + // Contains information statistics for mIndex + mPendingUpdates.
1.924 + CacheIndexStats mIndexStats;
1.925 +
1.926 + // When reading journal, we must first parse the whole file and apply the
1.927 + // changes iff the journal was read successfully. mTmpJournal is used to store
1.928 + // entries from the journal file. We throw away all these entries if parsing
1.929 + // of the journal fails or the hash does not match.
1.930 + nsTHashtable<CacheIndexEntry> mTmpJournal;
1.931 +
1.932 + // Arrays that keep entry records ordered by eviction preference. When looking
1.933 + // for an entry to evict, we first try to find an expired entry. If there is
1.934 + // no expired entry, we take the entry with lowest valid frecency. Zero
1.935 + // frecency is an initial value and such entries are stored at the end of the
1.936 + // array. Uninitialized entries and entries marked as deleted are not present
1.937 + // in these arrays.
1.938 + nsTArray<CacheIndexRecord *> mFrecencyArray;
1.939 + nsTArray<CacheIndexRecord *> mExpirationArray;
1.940 +
1.941 + nsTArray<CacheIndexIterator *> mIterators;
1.942 +
1.943 + class DiskConsumptionObserver : public nsRunnable
1.944 + {
1.945 + public:
1.946 + static DiskConsumptionObserver* Init(nsICacheStorageConsumptionObserver* aObserver)
1.947 + {
1.948 + nsWeakPtr observer = do_GetWeakReference(aObserver);
1.949 + if (!observer)
1.950 + return nullptr;
1.951 +
1.952 + return new DiskConsumptionObserver(observer);
1.953 + }
1.954 +
1.955 + void OnDiskConsumption(int64_t aSize)
1.956 + {
1.957 + mSize = aSize;
1.958 + NS_DispatchToMainThread(this);
1.959 + }
1.960 +
1.961 + private:
1.962 + DiskConsumptionObserver(nsWeakPtr const &aWeakObserver)
1.963 + : mObserver(aWeakObserver) { }
1.964 + virtual ~DiskConsumptionObserver() { }
1.965 +
1.966 + NS_IMETHODIMP Run()
1.967 + {
1.968 + MOZ_ASSERT(NS_IsMainThread());
1.969 +
1.970 + nsCOMPtr<nsICacheStorageConsumptionObserver> observer =
1.971 + do_QueryReferent(mObserver);
1.972 +
1.973 + if (observer) {
1.974 + observer->OnNetworkCacheDiskConsumption(mSize);
1.975 + }
1.976 +
1.977 + return NS_OK;
1.978 + }
1.979 +
1.980 + nsWeakPtr mObserver;
1.981 + int64_t mSize;
1.982 + };
1.983 +
1.984 + // List of async observers that want to get disk consumption information
1.985 + nsTArray<nsRefPtr<DiskConsumptionObserver> > mDiskConsumptionObservers;
1.986 +};
1.987 +
1.988 +class CacheIndexAutoLock {
1.989 +public:
1.990 + CacheIndexAutoLock(CacheIndex *aIndex)
1.991 + : mIndex(aIndex)
1.992 + , mLocked(true)
1.993 + {
1.994 + mIndex->Lock();
1.995 + }
1.996 + ~CacheIndexAutoLock()
1.997 + {
1.998 + if (mLocked) {
1.999 + mIndex->Unlock();
1.1000 + }
1.1001 + }
1.1002 + void Lock()
1.1003 + {
1.1004 + MOZ_ASSERT(!mLocked);
1.1005 + mIndex->Lock();
1.1006 + mLocked = true;
1.1007 + }
1.1008 + void Unlock()
1.1009 + {
1.1010 + MOZ_ASSERT(mLocked);
1.1011 + mIndex->Unlock();
1.1012 + mLocked = false;
1.1013 + }
1.1014 +
1.1015 +private:
1.1016 + nsRefPtr<CacheIndex> mIndex;
1.1017 + bool mLocked;
1.1018 +};
1.1019 +
1.1020 +class CacheIndexAutoUnlock {
1.1021 +public:
1.1022 + CacheIndexAutoUnlock(CacheIndex *aIndex)
1.1023 + : mIndex(aIndex)
1.1024 + , mLocked(false)
1.1025 + {
1.1026 + mIndex->Unlock();
1.1027 + }
1.1028 + ~CacheIndexAutoUnlock()
1.1029 + {
1.1030 + if (!mLocked) {
1.1031 + mIndex->Lock();
1.1032 + }
1.1033 + }
1.1034 + void Lock()
1.1035 + {
1.1036 + MOZ_ASSERT(!mLocked);
1.1037 + mIndex->Lock();
1.1038 + mLocked = true;
1.1039 + }
1.1040 + void Unlock()
1.1041 + {
1.1042 + MOZ_ASSERT(mLocked);
1.1043 + mIndex->Unlock();
1.1044 + mLocked = false;
1.1045 + }
1.1046 +
1.1047 +private:
1.1048 + nsRefPtr<CacheIndex> mIndex;
1.1049 + bool mLocked;
1.1050 +};
1.1051 +
1.1052 +} // net
1.1053 +} // mozilla
1.1054 +
1.1055 +#endif
