The Tor Browser / file comparison

comparison: netwerk/cache2/CacheIndex.h

netwerk/cache2/CacheIndex.h

branch
TOR_BUG_9701
changeset 15
b8a032363ba2
equal deleted inserted replaced
-1:000000000000 0:1b13dbf1db75
1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 #ifndef CacheIndex__h__
6 #define CacheIndex__h__
7
8 #include "CacheLog.h"
9 #include "CacheFileIOManager.h"
10 #include "nsIRunnable.h"
11 #include "CacheHashUtils.h"
12 #include "nsICacheStorageService.h"
13 #include "nsICacheEntry.h"
14 #include "nsILoadContextInfo.h"
15 #include "nsTHashtable.h"
16 #include "nsThreadUtils.h"
17 #include "nsWeakReference.h"
18 #include "mozilla/SHA1.h"
19 #include "mozilla/Mutex.h"
20 #include "mozilla/Endian.h"
21 #include "mozilla/TimeStamp.h"
22
23 class nsIFile;
24 class nsIDirectoryEnumerator;
25 class nsITimer;
26
27
28 #ifdef DEBUG
29 #define DEBUG_STATS 1
30 #endif
31
32 namespace mozilla {
33 namespace net {
34
35 class CacheFileMetadata;
36 class FileOpenHelper;
37 class CacheIndexIterator;
38
39 typedef struct {
40 // Version of the index. The index must be ignored and deleted when the file
41 // on disk was written with a newer version.
42 uint32_t mVersion;
43
44 // Timestamp of time when the last successful write of the index started.
45 // During update process we use this timestamp for a quick validation of entry
46 // files. If last modified time of the file is lower than this timestamp, we
47 // skip parsing of such file since the information in index should be up to
48 // date.
49 uint32_t mTimeStamp;
50
51 // We set this flag as soon as possible after parsing index during startup
52 // and clean it after we write journal to disk during shutdown. We ignore the
53 // journal and start update process whenever this flag is set during index
54 // parsing.
55 uint32_t mIsDirty;
56 } CacheIndexHeader;
57
58 struct CacheIndexRecord {
59 SHA1Sum::Hash mHash;
60 uint32_t mFrecency;
61 uint32_t mExpirationTime;
62 uint32_t mAppId;
63
64 /*
65 * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized
66 * 0100 0000 0000 0000 0000 0000 0000 0000 : anonymous
67 * 0010 0000 0000 0000 0000 0000 0000 0000 : inBrowser
68 * 0001 0000 0000 0000 0000 0000 0000 0000 : removed
69 * 0000 1000 0000 0000 0000 0000 0000 0000 : dirty
70 * 0000 0100 0000 0000 0000 0000 0000 0000 : fresh
71 * 0000 0011 0000 0000 0000 0000 0000 0000 : reserved
72 * 0000 0000 1111 1111 1111 1111 1111 1111 : file size (in kB)
73 */
74 uint32_t mFlags;
75
76 CacheIndexRecord()
77 : mFrecency(0)
78 , mExpirationTime(nsICacheEntry::NO_EXPIRATION_TIME)
79 , mAppId(nsILoadContextInfo::NO_APP_ID)
80 , mFlags(0)
81 {}
82 };
83
84 class CacheIndexEntry : public PLDHashEntryHdr
85 {
86 public:
87 typedef const SHA1Sum::Hash& KeyType;
88 typedef const SHA1Sum::Hash* KeyTypePointer;
89
90 CacheIndexEntry(KeyTypePointer aKey)
91 {
92 MOZ_COUNT_CTOR(CacheIndexEntry);
93 mRec = new CacheIndexRecord();
94 LOG(("CacheIndexEntry::CacheIndexEntry() - Created record [rec=%p]", mRec.get()));
95 memcpy(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash));
96 }
97 CacheIndexEntry(const CacheIndexEntry& aOther)
98 {
99 NS_NOTREACHED("CacheIndexEntry copy constructor is forbidden!");
100 }
101 ~CacheIndexEntry()
102 {
103 MOZ_COUNT_DTOR(CacheIndexEntry);
104 LOG(("CacheIndexEntry::~CacheIndexEntry() - Deleting record [rec=%p]",
105 mRec.get()));
106 }
107
108 // KeyEquals(): does this entry match this key?
109 bool KeyEquals(KeyTypePointer aKey) const
110 {
111 return memcmp(&mRec->mHash, aKey, sizeof(SHA1Sum::Hash)) == 0;
112 }
113
114 // KeyToPointer(): Convert KeyType to KeyTypePointer
115 static KeyTypePointer KeyToPointer(KeyType aKey) { return &aKey; }
116
117 // HashKey(): calculate the hash number
118 static PLDHashNumber HashKey(KeyTypePointer aKey)
119 {
120 return (reinterpret_cast<const uint32_t *>(aKey))[0];
121 }
122
123 // ALLOW_MEMMOVE can we move this class with memmove(), or do we have
124 // to use the copy constructor?
125 enum { ALLOW_MEMMOVE = true };
126
127 bool operator==(const CacheIndexEntry& aOther) const
128 {
129 return KeyEquals(&aOther.mRec->mHash);
130 }
131
132 CacheIndexEntry& operator=(const CacheIndexEntry& aOther)
133 {
134 MOZ_ASSERT(memcmp(&mRec->mHash, &aOther.mRec->mHash,
135 sizeof(SHA1Sum::Hash)) == 0);
136 mRec->mFrecency = aOther.mRec->mFrecency;
137 mRec->mExpirationTime = aOther.mRec->mExpirationTime;
138 mRec->mAppId = aOther.mRec->mAppId;
139 mRec->mFlags = aOther.mRec->mFlags;
140 return *this;
141 }
142
143 void InitNew()
144 {
145 mRec->mFrecency = 0;
146 mRec->mExpirationTime = nsICacheEntry::NO_EXPIRATION_TIME;
147 mRec->mAppId = nsILoadContextInfo::NO_APP_ID;
148 mRec->mFlags = 0;
149 }
150
151 void Init(uint32_t aAppId, bool aAnonymous, bool aInBrowser)
152 {
153 MOZ_ASSERT(mRec->mFrecency == 0);
154 MOZ_ASSERT(mRec->mExpirationTime == nsICacheEntry::NO_EXPIRATION_TIME);
155 MOZ_ASSERT(mRec->mAppId == nsILoadContextInfo::NO_APP_ID);
156 // When we init the entry it must be fresh and may be dirty
157 MOZ_ASSERT((mRec->mFlags & ~kDirtyMask) == kFreshMask);
158
159 mRec->mAppId = aAppId;
160 mRec->mFlags |= kInitializedMask;
161 if (aAnonymous) {
162 mRec->mFlags |= kAnonymousMask;
163 }
164 if (aInBrowser) {
165 mRec->mFlags |= kInBrowserMask;
166 }
167 }
168
169 const SHA1Sum::Hash * Hash() { return &mRec->mHash; }
170
171 bool IsInitialized() { return !!(mRec->mFlags & kInitializedMask); }
172
173 uint32_t AppId() { return mRec->mAppId; }
174 bool Anonymous() { return !!(mRec->mFlags & kAnonymousMask); }
175 bool InBrowser() { return !!(mRec->mFlags & kInBrowserMask); }
176
177 bool IsRemoved() { return !!(mRec->mFlags & kRemovedMask); }
178 void MarkRemoved() { mRec->mFlags |= kRemovedMask; }
179
180 bool IsDirty() { return !!(mRec->mFlags & kDirtyMask); }
181 void MarkDirty() { mRec->mFlags |= kDirtyMask; }
182 void ClearDirty() { mRec->mFlags &= ~kDirtyMask; }
183
184 bool IsFresh() { return !!(mRec->mFlags & kFreshMask); }
185 void MarkFresh() { mRec->mFlags |= kFreshMask; }
186
187 void SetFrecency(uint32_t aFrecency) { mRec->mFrecency = aFrecency; }
188 uint32_t GetFrecency() { return mRec->mFrecency; }
189
190 void SetExpirationTime(uint32_t aExpirationTime)
191 {
192 mRec->mExpirationTime = aExpirationTime;
193 }
194 uint32_t GetExpirationTime() { return mRec->mExpirationTime; }
195
196 // Sets filesize in kilobytes.
197 void SetFileSize(uint32_t aFileSize)
198 {
199 if (aFileSize > kFileSizeMask) {
200 LOG(("CacheIndexEntry::SetFileSize() - FileSize is too large, "
201 "truncating to %u", kFileSizeMask));
202 aFileSize = kFileSizeMask;
203 }
204 mRec->mFlags &= ~kFileSizeMask;
205 mRec->mFlags |= aFileSize;
206 }
207 // Returns filesize in kilobytes.
208 uint32_t GetFileSize() { return mRec->mFlags & kFileSizeMask; }
209 bool IsFileEmpty() { return GetFileSize() == 0; }
210
211 void WriteToBuf(void *aBuf)
212 {
213 CacheIndexRecord *dst = reinterpret_cast<CacheIndexRecord *>(aBuf);
214
215 // Copy the whole record to the buffer.
216 memcpy(aBuf, mRec, sizeof(CacheIndexRecord));
217
218 // Dirty and fresh flags should never go to disk, since they make sense only
219 // during current session.
220 dst->mFlags &= ~kDirtyMask;
221 dst->mFlags &= ~kFreshMask;
222
223 #if defined(IS_LITTLE_ENDIAN)
224 // Data in the buffer are in machine byte order and we want them in network
225 // byte order.
226 NetworkEndian::writeUint32(&dst->mFrecency, dst->mFrecency);
227 NetworkEndian::writeUint32(&dst->mExpirationTime, dst->mExpirationTime);
228 NetworkEndian::writeUint32(&dst->mAppId, dst->mAppId);
229 NetworkEndian::writeUint32(&dst->mFlags, dst->mFlags);
230 #endif
231 }
232
233 void ReadFromBuf(void *aBuf)
234 {
235 CacheIndexRecord *src= reinterpret_cast<CacheIndexRecord *>(aBuf);
236 MOZ_ASSERT(memcmp(&mRec->mHash, &src->mHash,
237 sizeof(SHA1Sum::Hash)) == 0);
238
239 mRec->mFrecency = NetworkEndian::readUint32(&src->mFrecency);
240 mRec->mExpirationTime = NetworkEndian::readUint32(&src->mExpirationTime);
241 mRec->mAppId = NetworkEndian::readUint32(&src->mAppId);
242 mRec->mFlags = NetworkEndian::readUint32(&src->mFlags);
243 }
244
245 void Log() {
246 LOG(("CacheIndexEntry::Log() [this=%p, hash=%08x%08x%08x%08x%08x, fresh=%u,"
247 " initialized=%u, removed=%u, dirty=%u, anonymous=%u, inBrowser=%u, "
248 "appId=%u, frecency=%u, expirationTime=%u, size=%u]",
249 this, LOGSHA1(mRec->mHash), IsFresh(), IsInitialized(), IsRemoved(),
250 IsDirty(), Anonymous(), InBrowser(), AppId(), GetFrecency(),
251 GetExpirationTime(), GetFileSize()));
252 }
253
254 static bool RecordMatchesLoadContextInfo(CacheIndexRecord *aRec,
255 nsILoadContextInfo *aInfo)
256 {
257 if (!aInfo->IsPrivate() &&
258 aInfo->AppId() == aRec->mAppId &&
259 aInfo->IsAnonymous() == !!(aRec->mFlags & kAnonymousMask) &&
260 aInfo->IsInBrowserElement() == !!(aRec->mFlags & kInBrowserMask)) {
261 return true;
262 }
263
264 return false;
265 }
266
267 // Memory reporting
268 size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const
269 {
270 return mallocSizeOf(mRec.get());
271 }
272
273 size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
274 {
275 return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
276 }
277
278 private:
279 friend class CacheIndex;
280 friend class CacheIndexEntryAutoManage;
281
282 static const uint32_t kInitializedMask = 0x80000000;
283 static const uint32_t kAnonymousMask = 0x40000000;
284 static const uint32_t kInBrowserMask = 0x20000000;
285
286 // This flag is set when the entry was removed. We need to keep this
287 // information in memory until we write the index file.
288 static const uint32_t kRemovedMask = 0x10000000;
289
290 // This flag is set when the information in memory is not in sync with the
291 // information in index file on disk.
292 static const uint32_t kDirtyMask = 0x08000000;
293
294 // This flag is set when the information about the entry is fresh, i.e.
295 // we've created or opened this entry during this session, or we've seen
296 // this entry during update or build process.
297 static const uint32_t kFreshMask = 0x04000000;
298
299 static const uint32_t kReservedMask = 0x03000000;
300
301 // FileSize in kilobytes
302 static const uint32_t kFileSizeMask = 0x00FFFFFF;
303
304 nsAutoPtr<CacheIndexRecord> mRec;
305 };
306
307 class CacheIndexStats
308 {
309 public:
310 CacheIndexStats()
311 : mCount(0)
312 , mNotInitialized(0)
313 , mRemoved(0)
314 , mDirty(0)
315 , mFresh(0)
316 , mEmpty(0)
317 , mSize(0)
318 #ifdef DEBUG
319 , mStateLogged(false)
320 , mDisableLogging(false)
321 #endif
322 {
323 }
324
325 bool operator==(const CacheIndexStats& aOther) const
326 {
327 return
328 #ifdef DEBUG
329 aOther.mStateLogged == mStateLogged &&
330 #endif
331 aOther.mCount == mCount &&
332 aOther.mNotInitialized == mNotInitialized &&
333 aOther.mRemoved == mRemoved &&
334 aOther.mDirty == mDirty &&
335 aOther.mFresh == mFresh &&
336 aOther.mEmpty == mEmpty &&
337 aOther.mSize == mSize;
338 }
339
340 #ifdef DEBUG
341 void DisableLogging() {
342 mDisableLogging = true;
343 }
344 #endif
345
346 void Log() {
347 LOG(("CacheIndexStats::Log() [count=%u, notInitialized=%u, removed=%u, "
348 "dirty=%u, fresh=%u, empty=%u, size=%u]", mCount, mNotInitialized,
349 mRemoved, mDirty, mFresh, mEmpty, mSize));
350 }
351
352 void Clear() {
353 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Clear() - state logged!");
354
355 mCount = 0;
356 mNotInitialized = 0;
357 mRemoved = 0;
358 mDirty = 0;
359 mFresh = 0;
360 mEmpty = 0;
361 mSize = 0;
362 }
363
364 #ifdef DEBUG
365 bool StateLogged() {
366 return mStateLogged;
367 }
368 #endif
369
370 uint32_t Count() {
371 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Count() - state logged!");
372 return mCount;
373 }
374
375 uint32_t Dirty() {
376 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Dirty() - state logged!");
377 return mDirty;
378 }
379
380 uint32_t Fresh() {
381 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Fresh() - state logged!");
382 return mFresh;
383 }
384
385 uint32_t ActiveEntriesCount() {
386 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::ActiveEntriesCount() - state "
387 "logged!");
388 return mCount - mRemoved - mNotInitialized - mEmpty;
389 }
390
391 uint32_t Size() {
392 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::Size() - state logged!");
393 return mSize;
394 }
395
396 void BeforeChange(CacheIndexEntry *aEntry) {
397 #ifdef DEBUG_STATS
398 if (!mDisableLogging) {
399 LOG(("CacheIndexStats::BeforeChange()"));
400 Log();
401 }
402 #endif
403
404 MOZ_ASSERT(!mStateLogged, "CacheIndexStats::BeforeChange() - state "
405 "logged!");
406 #ifdef DEBUG
407 mStateLogged = true;
408 #endif
409 if (aEntry) {
410 MOZ_ASSERT(mCount);
411 mCount--;
412 if (aEntry->IsDirty()) {
413 MOZ_ASSERT(mDirty);
414 mDirty--;
415 }
416 if (aEntry->IsFresh()) {
417 MOZ_ASSERT(mFresh);
418 mFresh--;
419 }
420 if (aEntry->IsRemoved()) {
421 MOZ_ASSERT(mRemoved);
422 mRemoved--;
423 } else {
424 if (!aEntry->IsInitialized()) {
425 MOZ_ASSERT(mNotInitialized);
426 mNotInitialized--;
427 } else {
428 if (aEntry->IsFileEmpty()) {
429 MOZ_ASSERT(mEmpty);
430 mEmpty--;
431 } else {
432 MOZ_ASSERT(mSize >= aEntry->GetFileSize());
433 mSize -= aEntry->GetFileSize();
434 }
435 }
436 }
437 }
438 }
439
440 void AfterChange(CacheIndexEntry *aEntry) {
441 MOZ_ASSERT(mStateLogged, "CacheIndexStats::AfterChange() - state not "
442 "logged!");
443 #ifdef DEBUG
444 mStateLogged = false;
445 #endif
446 if (aEntry) {
447 ++mCount;
448 if (aEntry->IsDirty()) {
449 mDirty++;
450 }
451 if (aEntry->IsFresh()) {
452 mFresh++;
453 }
454 if (aEntry->IsRemoved()) {
455 mRemoved++;
456 } else {
457 if (!aEntry->IsInitialized()) {
458 mNotInitialized++;
459 } else {
460 if (aEntry->IsFileEmpty()) {
461 mEmpty++;
462 } else {
463 mSize += aEntry->GetFileSize();
464 }
465 }
466 }
467 }
468
469 #ifdef DEBUG_STATS
470 if (!mDisableLogging) {
471 LOG(("CacheIndexStats::AfterChange()"));
472 Log();
473 }
474 #endif
475 }
476
477 private:
478 uint32_t mCount;
479 uint32_t mNotInitialized;
480 uint32_t mRemoved;
481 uint32_t mDirty;
482 uint32_t mFresh;
483 uint32_t mEmpty;
484 uint32_t mSize;
485 #ifdef DEBUG
486 // We completely remove the data about an entry from the stats in
487 // BeforeChange() and set this flag to true. The entry is then modified,
488 // deleted or created and the data is again put into the stats and this flag
489 // set to false. Statistics must not be read during this time since the
490 // information is not correct.
491 bool mStateLogged;
492
493 // Disables logging in this instance of CacheIndexStats
494 bool mDisableLogging;
495 #endif
496 };
497
498 class CacheIndex : public CacheFileIOListener
499 , public nsIRunnable
500 {
501 public:
502 NS_DECL_THREADSAFE_ISUPPORTS
503 NS_DECL_NSIRUNNABLE
504
505 CacheIndex();
506
507 static nsresult Init(nsIFile *aCacheDirectory);
508 static nsresult PreShutdown();
509 static nsresult Shutdown();
510
511 // Following methods can be called only on IO thread.
512
513 // Add entry to the index. The entry shouldn't be present in index. This
514 // method is called whenever a new handle for a new entry file is created. The
515 // newly created entry is not initialized and it must be either initialized
516 // with InitEntry() or removed with RemoveEntry().
517 static nsresult AddEntry(const SHA1Sum::Hash *aHash);
518
519 // Inform index about an existing entry that should be present in index. This
520 // method is called whenever a new handle for an existing entry file is
521 // created. Like in case of AddEntry(), either InitEntry() or RemoveEntry()
522 // must be called on the entry, since the entry is not initizlized if the
523 // index is outdated.
524 static nsresult EnsureEntryExists(const SHA1Sum::Hash *aHash);
525
526 // Initialize the entry. It MUST be present in index. Call to AddEntry() or
527 // EnsureEntryExists() must precede the call to this method.
528 static nsresult InitEntry(const SHA1Sum::Hash *aHash,
529 uint32_t aAppId,
530 bool aAnonymous,
531 bool aInBrowser);
532
533 // Remove entry from index. The entry should be present in index.
534 static nsresult RemoveEntry(const SHA1Sum::Hash *aHash);
535
536 // Update some information in entry. The entry MUST be present in index and
537 // MUST be initialized. Call to AddEntry() or EnsureEntryExists() and to
538 // InitEntry() must precede the call to this method.
539 // Pass nullptr if the value didn't change.
540 static nsresult UpdateEntry(const SHA1Sum::Hash *aHash,
541 const uint32_t *aFrecency,
542 const uint32_t *aExpirationTime,
543 const uint32_t *aSize);
544
545 // Remove all entries from the index. Called when clearing the whole cache.
546 static nsresult RemoveAll();
547
548 enum EntryStatus {
549 EXISTS = 0,
550 DOES_NOT_EXIST = 1,
551 DO_NOT_KNOW = 2
552 };
553
554 // Returns status of the entry in index for the given key. It can be called
555 // on any thread.
556 static nsresult HasEntry(const nsACString &aKey, EntryStatus *_retval);
557
558 // Returns a hash of the least important entry that should be evicted if the
559 // cache size is over limit and also returns a total number of all entries in
560 // the index.
561 static nsresult GetEntryForEviction(SHA1Sum::Hash *aHash, uint32_t *aCnt);
562
563 // Returns cache size in kB.
564 static nsresult GetCacheSize(uint32_t *_retval);
565
566 // Asynchronously gets the disk cache size, used for display in the UI.
567 static nsresult AsyncGetDiskConsumption(nsICacheStorageConsumptionObserver* aObserver);
568
569 // Returns an iterator that returns entries matching a given context that were
570 // present in the index at the time this method was called. If aAddNew is true
571 // then the iterator will also return entries created after this call.
572 // NOTE: When some entry is removed from index it is removed also from the
573 // iterator regardless what aAddNew was passed.
574 static nsresult GetIterator(nsILoadContextInfo *aInfo, bool aAddNew,
575 CacheIndexIterator **_retval);
576
577 // Returns true if we _think_ that the index is up to date. I.e. the state is
578 // READY or WRITING and mIndexNeedsUpdate as well as mShuttingDown is false.
579 static nsresult IsUpToDate(bool *_retval);
580
581 // Memory reporting
582 static size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
583 static size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf);
584
585 private:
586 friend class CacheIndexEntryAutoManage;
587 friend class CacheIndexAutoLock;
588 friend class CacheIndexAutoUnlock;
589 friend class FileOpenHelper;
590 friend class CacheIndexIterator;
591
592 virtual ~CacheIndex();
593
594 NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult);
595 nsresult OnFileOpenedInternal(FileOpenHelper *aOpener,
596 CacheFileHandle *aHandle, nsresult aResult);
597 NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf,
598 nsresult aResult);
599 NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult);
600 NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult);
601 NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult);
602 NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult);
603
604 void Lock();
605 void Unlock();
606 void AssertOwnsLock();
607
608 nsresult InitInternal(nsIFile *aCacheDirectory);
609 void PreShutdownInternal();
610
611 // This method returns false when index is not initialized or is shut down.
612 bool IsIndexUsable();
613
614 // This method checks whether the entry has the same values of appId,
615 // isAnonymous and isInBrowser. We don't expect to find a collision since
616 // these values are part of the key that we hash and we use a strong hash
617 // function.
618 static bool IsCollision(CacheIndexEntry *aEntry,
619 uint32_t aAppId,
620 bool aAnonymous,
621 bool aInBrowser);
622
623 // Checks whether any of the information about the entry has changed.
624 static bool HasEntryChanged(CacheIndexEntry *aEntry,
625 const uint32_t *aFrecency,
626 const uint32_t *aExpirationTime,
627 const uint32_t *aSize);
628
629 // Merge all pending operations from mPendingUpdates into mIndex.
630 void ProcessPendingOperations();
631 static PLDHashOperator UpdateEntryInIndex(CacheIndexEntry *aEntry,
632 void* aClosure);
633
634 // Following methods perform writing of the index file.
635 //
636 // The index is written periodically, but not earlier than once in
637 // kMinDumpInterval and there must be at least kMinUnwrittenChanges
638 // differences between index on disk and in memory. Index is always first
639 // written to a temporary file and the old index file is replaced when the
640 // writing process succeeds.
641 //
642 // Starts writing of index when both limits (minimal delay between writes and
643 // minimum number of changes in index) were exceeded.
644 bool WriteIndexToDiskIfNeeded();
645 // Starts writing of index file.
646 void WriteIndexToDisk();
647 // Serializes part of mIndex hashtable to the write buffer a writes the buffer
648 // to the file.
649 void WriteRecords();
650 // Finalizes writing process.
651 void FinishWrite(bool aSucceeded);
652
653 static PLDHashOperator CopyRecordsToRWBuf(CacheIndexEntry *aEntry,
654 void* aClosure);
655 static PLDHashOperator ApplyIndexChanges(CacheIndexEntry *aEntry,
656 void* aClosure);
657
658 // Following methods perform writing of the journal during shutdown. All these
659 // methods must be called only during shutdown since they write/delete files
660 // directly on the main thread instead of using CacheFileIOManager that does
661 // it asynchronously on IO thread. Journal contains only entries that are
662 // dirty, i.e. changes that are not present in the index file on the disk.
663 // When the log is written successfully, the dirty flag in index file is
664 // cleared.
665 nsresult GetFile(const nsACString &aName, nsIFile **_retval);
666 nsresult RemoveFile(const nsACString &aName);
667 void RemoveIndexFromDisk();
668 // Writes journal to the disk and clears dirty flag in index header.
669 nsresult WriteLogToDisk();
670
671 static PLDHashOperator WriteEntryToLog(CacheIndexEntry *aEntry,
672 void* aClosure);
673
674 // Following methods perform reading of the index from the disk.
675 //
676 // Index is read at startup just after initializing the CacheIndex. There are
677 // 3 files used when manipulating with index: index file, journal file and
678 // a temporary file. All files contain the hash of the data, so we can check
679 // whether the content is valid and complete. Index file contains also a dirty
680 // flag in the index header which is unset on a clean shutdown. During opening
681 // and reading of the files we determine the status of the whole index from
682 // the states of the separate files. Following table shows all possible
683 // combinations:
684 //
685 // index, journal, tmpfile
686 // M * * - index is missing -> BUILD
687 // I * * - index is invalid -> BUILD
688 // D * * - index is dirty -> UPDATE
689 // C M * - index is dirty -> UPDATE
690 // C I * - unexpected state -> UPDATE
691 // C V E - unexpected state -> UPDATE
692 // C V M - index is up to date -> READY
693 //
694 // where the letters mean:
695 // * - any state
696 // E - file exists
697 // M - file is missing
698 // I - data is invalid (parsing failed or hash didn't match)
699 // D - dirty (data in index file is correct, but dirty flag is set)
700 // C - clean (index file is clean)
701 // V - valid (data in journal file is correct)
702 //
703 // Note: We accept the data from journal only when the index is up to date as
704 // a whole (i.e. C,V,M state).
705 //
706 // We rename the journal file to the temporary file as soon as possible after
707 // initial test to ensure that we start update process on the next startup if
708 // FF crashes during parsing of the index.
709 //
710 // Initiates reading index from disk.
711 void ReadIndexFromDisk();
712 // Starts reading data from index file.
713 void StartReadingIndex();
714 // Parses data read from index file.
715 void ParseRecords();
716 // Starts reading data from journal file.
717 void StartReadingJournal();
718 // Parses data read from journal file.
719 void ParseJournal();
720 // Merges entries from journal into mIndex.
721 void MergeJournal();
722 // In debug build this method checks that we have no fresh entry in mIndex
723 // after we finish reading index and before we process pending operations.
724 void EnsureNoFreshEntry();
725 // In debug build this method is called after processing pending operations
726 // to make sure mIndexStats contains correct information.
727 void EnsureCorrectStats();
728 static PLDHashOperator SumIndexStats(CacheIndexEntry *aEntry, void* aClosure);
729 // Finalizes reading process.
730 void FinishRead(bool aSucceeded);
731
732 static PLDHashOperator ProcessJournalEntry(CacheIndexEntry *aEntry,
733 void* aClosure);
734
735 // Following methods perform updating and building of the index.
736 // Timer callback that starts update or build process.
737 static void DelayedUpdate(nsITimer *aTimer, void *aClosure);
738 // Posts timer event that start update or build process.
739 nsresult ScheduleUpdateTimer(uint32_t aDelay);
740 nsresult SetupDirectoryEnumerator();
741 void InitEntryFromDiskData(CacheIndexEntry *aEntry,
742 CacheFileMetadata *aMetaData,
743 int64_t aFileSize);
744 // Returns true when either a timer is scheduled or event is posted.
745 bool IsUpdatePending();
746 // Iterates through all files in entries directory that we didn't create/open
747 // during this session, parses them and adds the entries to the index.
748 void BuildIndex();
749
750 bool StartUpdatingIndexIfNeeded(bool aSwitchingToReadyState = false);
751 // Starts update or build process or fires a timer when it is too early after
752 // startup.
753 void StartUpdatingIndex(bool aRebuild);
754 // Iterates through all files in entries directory that we didn't create/open
755 // during this session and theirs last modified time is newer than timestamp
756 // in the index header. Parses the files and adds the entries to the index.
757 void UpdateIndex();
758 // Finalizes update or build process.
759 void FinishUpdate(bool aSucceeded);
760
761 static PLDHashOperator RemoveNonFreshEntries(CacheIndexEntry *aEntry,
762 void* aClosure);
763
764 enum EState {
765 // Initial state in which the index is not usable
766 // Possible transitions:
767 // -> READING
768 INITIAL = 0,
769
770 // Index is being read from the disk.
771 // Possible transitions:
772 // -> INITIAL - We failed to dispatch a read event.
773 // -> BUILDING - No or corrupted index file was found.
774 // -> UPDATING - No or corrupted journal file was found.
775 // - Dirty flag was set in index header.
776 // -> READY - Index was read successfully or was interrupted by
777 // pre-shutdown.
778 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
779 READING = 1,
780
781 // Index is being written to the disk.
782 // Possible transitions:
783 // -> READY - Writing of index finished or was interrupted by
784 // pre-shutdown..
785 // -> UPDATING - Writing of index finished, but index was found outdated
786 // during writing.
787 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
788 WRITING = 2,
789
790 // Index is being build.
791 // Possible transitions:
792 // -> READY - Building of index finished or was interrupted by
793 // pre-shutdown.
794 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
795 BUILDING = 3,
796
797 // Index is being updated.
798 // Possible transitions:
799 // -> READY - Updating of index finished or was interrupted by
800 // pre-shutdown.
801 // -> SHUTDOWN - This could happen only in case of pre-shutdown failure.
802 UPDATING = 4,
803
804 // Index is ready.
805 // Possible transitions:
806 // -> UPDATING - Index was found outdated.
807 // -> SHUTDOWN - Index is shutting down.
808 READY = 5,
809
810 // Index is shutting down.
811 SHUTDOWN = 6
812 };
813
814 #ifdef PR_LOGGING
815 static char const * StateString(EState aState);
816 #endif
817 void ChangeState(EState aNewState);
818
819 // Allocates and releases buffer used for reading and writing index.
820 void AllocBuffer();
821 void ReleaseBuffer();
822
823 // Methods used by CacheIndexEntryAutoManage to keep the arrays up to date.
824 void InsertRecordToFrecencyArray(CacheIndexRecord *aRecord);
825 void InsertRecordToExpirationArray(CacheIndexRecord *aRecord);
826 void RemoveRecordFromFrecencyArray(CacheIndexRecord *aRecord);
827 void RemoveRecordFromExpirationArray(CacheIndexRecord *aRecord);
828
829 // Methods used by CacheIndexEntryAutoManage to keep the iterators up to date.
830 void AddRecordToIterators(CacheIndexRecord *aRecord);
831 void RemoveRecordFromIterators(CacheIndexRecord *aRecord);
832 void ReplaceRecordInIterators(CacheIndexRecord *aOldRecord,
833 CacheIndexRecord *aNewRecord);
834
835 // Memory reporting (private part)
836 size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
837
838 static CacheIndex *gInstance;
839
840 nsCOMPtr<nsIFile> mCacheDirectory;
841
842 mozilla::Mutex mLock;
843 EState mState;
844 // Timestamp of time when the index was initialized. We use it to delay
845 // initial update or build of index.
846 TimeStamp mStartTime;
847 // Set to true in PreShutdown(), it is checked on variaous places to prevent
848 // starting any process (write, update, etc.) during shutdown.
849 bool mShuttingDown;
850 // When set to true, update process should start as soon as possible. This
851 // flag is set whenever we find some inconsistency which would be fixed by
852 // update process. The flag is checked always when switching to READY state.
853 // To make sure we start the update process as soon as possible, methods that
854 // set this flag should also call StartUpdatingIndexIfNeeded() to cover the
855 // case when we are currently in READY state.
856 bool mIndexNeedsUpdate;
857 // Set at the beginning of RemoveAll() which clears the whole index. When
858 // removing all entries we must stop any pending reading, writing, updating or
859 // building operation. This flag is checked at various places and it prevents
860 // we won't start another operation (e.g. canceling reading of the index would
861 // normally start update or build process)
862 bool mRemovingAll;
863 // Whether the index file on disk exists and is valid.
864 bool mIndexOnDiskIsValid;
865 // When something goes wrong during updating or building process, we don't
866 // mark index clean (and also don't write journal) to ensure that update or
867 // build will be initiated on the next start.
868 bool mDontMarkIndexClean;
869 // Timestamp value from index file. It is used during update process to skip
870 // entries that were last modified before this timestamp.
871 uint32_t mIndexTimeStamp;
872 // Timestamp of last time the index was dumped to disk.
873 // NOTE: The index might not be necessarily dumped at this time. The value
874 // is used to schedule next dump of the index.
875 TimeStamp mLastDumpTime;
876
877 // Timer of delayed update/build.
878 nsCOMPtr<nsITimer> mUpdateTimer;
879 // True when build or update event is posted
880 bool mUpdateEventPending;
881
882 // Helper members used when reading/writing index from/to disk.
883 // Contains number of entries that should be skipped:
884 // - in hashtable when writing index because they were already written
885 // - in index file when reading index because they were already read
886 uint32_t mSkipEntries;
887 // Number of entries that should be written to disk. This is number of entries
888 // in hashtable that are initialized and are not marked as removed when writing
889 // begins.
890 uint32_t mProcessEntries;
891 char *mRWBuf;
892 uint32_t mRWBufSize;
893 uint32_t mRWBufPos;
894 nsRefPtr<CacheHash> mRWHash;
895
896 // Reading of journal succeeded if true.
897 bool mJournalReadSuccessfully;
898
899 // Handle used for writing and reading index file.
900 nsRefPtr<CacheFileHandle> mIndexHandle;
901 // Handle used for reading journal file.
902 nsRefPtr<CacheFileHandle> mJournalHandle;
903 // Used to check the existence of the file during reading process.
904 nsRefPtr<CacheFileHandle> mTmpHandle;
905
906 nsRefPtr<FileOpenHelper> mIndexFileOpener;
907 nsRefPtr<FileOpenHelper> mJournalFileOpener;
908 nsRefPtr<FileOpenHelper> mTmpFileOpener;
909
910 // Directory enumerator used when building and updating index.
911 nsCOMPtr<nsIDirectoryEnumerator> mDirEnumerator;
912
913 // Main index hashtable.
914 nsTHashtable<CacheIndexEntry> mIndex;
915
916 // We cannot add, remove or change any entry in mIndex in states READING and
917 // WRITING. We track all changes in mPendingUpdates during these states.
918 nsTHashtable<CacheIndexEntry> mPendingUpdates;
919
920 // Contains information statistics for mIndex + mPendingUpdates.
921 CacheIndexStats mIndexStats;
922
923 // When reading journal, we must first parse the whole file and apply the
924 // changes iff the journal was read successfully. mTmpJournal is used to store
925 // entries from the journal file. We throw away all these entries if parsing
926 // of the journal fails or the hash does not match.
927 nsTHashtable<CacheIndexEntry> mTmpJournal;
928
929 // Arrays that keep entry records ordered by eviction preference. When looking
930 // for an entry to evict, we first try to find an expired entry. If there is
931 // no expired entry, we take the entry with lowest valid frecency. Zero
932 // frecency is an initial value and such entries are stored at the end of the
933 // array. Uninitialized entries and entries marked as deleted are not present
934 // in these arrays.
935 nsTArray<CacheIndexRecord *> mFrecencyArray;
936 nsTArray<CacheIndexRecord *> mExpirationArray;
937
938 nsTArray<CacheIndexIterator *> mIterators;
939
940 class DiskConsumptionObserver : public nsRunnable
941 {
942 public:
943 static DiskConsumptionObserver* Init(nsICacheStorageConsumptionObserver* aObserver)
944 {
945 nsWeakPtr observer = do_GetWeakReference(aObserver);
946 if (!observer)
947 return nullptr;
948
949 return new DiskConsumptionObserver(observer);
950 }
951
952 void OnDiskConsumption(int64_t aSize)
953 {
954 mSize = aSize;
955 NS_DispatchToMainThread(this);
956 }
957
958 private:
959 DiskConsumptionObserver(nsWeakPtr const &aWeakObserver)
960 : mObserver(aWeakObserver) { }
961 virtual ~DiskConsumptionObserver() { }
962
963 NS_IMETHODIMP Run()
964 {
965 MOZ_ASSERT(NS_IsMainThread());
966
967 nsCOMPtr<nsICacheStorageConsumptionObserver> observer =
968 do_QueryReferent(mObserver);
969
970 if (observer) {
971 observer->OnNetworkCacheDiskConsumption(mSize);
972 }
973
974 return NS_OK;
975 }
976
977 nsWeakPtr mObserver;
978 int64_t mSize;
979 };
980
981 // List of async observers that want to get disk consumption information
982 nsTArray<nsRefPtr<DiskConsumptionObserver> > mDiskConsumptionObservers;
983 };
984
985 class CacheIndexAutoLock {
986 public:
987 CacheIndexAutoLock(CacheIndex *aIndex)
988 : mIndex(aIndex)
989 , mLocked(true)
990 {
991 mIndex->Lock();
992 }
993 ~CacheIndexAutoLock()
994 {
995 if (mLocked) {
996 mIndex->Unlock();
997 }
998 }
999 void Lock()
1000 {
1001 MOZ_ASSERT(!mLocked);
1002 mIndex->Lock();
1003 mLocked = true;
1004 }
1005 void Unlock()
1006 {
1007 MOZ_ASSERT(mLocked);
1008 mIndex->Unlock();
1009 mLocked = false;
1010 }
1011
1012 private:
1013 nsRefPtr<CacheIndex> mIndex;
1014 bool mLocked;
1015 };
1016
1017 class CacheIndexAutoUnlock {
1018 public:
1019 CacheIndexAutoUnlock(CacheIndex *aIndex)
1020 : mIndex(aIndex)
1021 , mLocked(false)
1022 {
1023 mIndex->Unlock();
1024 }
1025 ~CacheIndexAutoUnlock()
1026 {
1027 if (!mLocked) {
1028 mIndex->Lock();
1029 }
1030 }
1031 void Lock()
1032 {
1033 MOZ_ASSERT(!mLocked);
1034 mIndex->Lock();
1035 mLocked = true;
1036 }
1037 void Unlock()
1038 {
1039 MOZ_ASSERT(mLocked);
1040 mIndex->Unlock();
1041 mLocked = false;
1042 }
1043
1044 private:
1045 nsRefPtr<CacheIndex> mIndex;
1046 bool mLocked;
1047 };
1048
1049 } // net
1050 } // mozilla
1051
1052 #endif

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial