netwerk/cache/nsDiskCacheMap.h@6474c204b198
netwerk/cache/nsDiskCacheMap.h
Wed, 31 Dec 2014 06:09:35 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:09:35 +0100
- changeset 0
- 6474c204b198
- permissions
- -rw-r--r--
Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.
1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /* vim:set ts=4 sw=4 sts=4 cin et: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef _nsDiskCacheMap_h_
8 #define _nsDiskCacheMap_h_
10 #include "mozilla/MemoryReporting.h"
11 #include <limits.h>
13 #include "prnetdb.h"
14 #include "nsDebug.h"
15 #include "nsError.h"
16 #include "nsIFile.h"
17 #include "nsITimer.h"
19 #include "nsDiskCache.h"
20 #include "nsDiskCacheBlockFile.h"
23 class nsDiskCacheBinding;
24 struct nsDiskCacheEntry;
26 /******************************************************************************
27 * nsDiskCacheRecord
28 *
29 * Cache Location Format
30 *
31 * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit
32 *
33 * 0011 0000 0000 0000 0000 0000 0000 0000 : File Selector (0 = separate file)
34 * 0000 0011 0000 0000 0000 0000 0000 0000 : number of extra contiguous blocks 1-4
35 * 0100 1100 0000 0000 0000 0000 0000 0000 : reserved bits
36 * 0000 0000 1111 1111 1111 1111 1111 1111 : block# 0-16777216 (2^24)
37 *
38 * 0000 0000 1111 1111 1111 1111 0000 0000 : eFileSizeMask (size of file in k: see note)
39 * 0000 0000 0000 0000 0000 0000 1111 1111 : eFileGenerationMask
40 *
41 * File Selector:
42 * 0 = separate file on disk
43 * 1 = 256 byte block file
44 * 2 = 1k block file
45 * 3 = 4k block file
46 *
47 * eFileSizeMask note: Files larger than 65535 KiB have this limit stored in
48 * the location. The file itself must be examined to
49 * determine its actual size if necessary.
50 *
51 *****************************************************************************/
53 /*
54 We have 3 block files with roughly the same max size (32MB)
55 1 - block size 256B, number of blocks 131072
56 2 - block size 1kB, number of blocks 32768
57 3 - block size 4kB, number of blocks 8192
58 */
59 #define kNumBlockFiles 3
60 #define SIZE_SHIFT(idx) (2 * ((idx) - 1))
61 #define BLOCK_SIZE_FOR_INDEX(idx) ((idx) ? (256 << SIZE_SHIFT(idx)) : 0)
62 #define BITMAP_SIZE_FOR_INDEX(idx) ((idx) ? (131072 >> SIZE_SHIFT(idx)) : 0)
64 // Min and max values for the number of records in the DiskCachemap
65 #define kMinRecordCount 512
67 #define kSeparateFile 0
68 #define kBuckets (1 << 5) // must be a power of 2!
70 // Maximum size in K which can be stored in the location (see eFileSizeMask).
71 // Both data and metadata can be larger, but only up to kMaxDataSizeK can be
72 // counted into total cache size. I.e. if there are entries where either data or
73 // metadata is larger than kMaxDataSizeK, the total cache size will be
74 // inaccurate (smaller) than the actual cache size. The alternative is to stat
75 // the files to find the real size, which was decided against for performance
76 // reasons. See bug #651100 comment #21.
77 #define kMaxDataSizeK 0xFFFF
79 // preallocate up to 1MB of separate cache file
80 #define kPreallocateLimit 1 * 1024 * 1024
82 // The minimum amount of milliseconds to wait before re-attempting to
83 // revalidate the cache.
84 #define kRevalidateCacheTimeout 3000
85 #define kRevalidateCacheTimeoutTolerance 10
86 #define kRevalidateCacheErrorTimeout 1000
88 class nsDiskCacheRecord {
90 private:
91 uint32_t mHashNumber;
92 uint32_t mEvictionRank;
93 uint32_t mDataLocation;
94 uint32_t mMetaLocation;
96 enum {
97 eLocationInitializedMask = 0x80000000,
99 eLocationSelectorMask = 0x30000000,
100 eLocationSelectorOffset = 28,
102 eExtraBlocksMask = 0x03000000,
103 eExtraBlocksOffset = 24,
105 eReservedMask = 0x4C000000,
107 eBlockNumberMask = 0x00FFFFFF,
109 eFileSizeMask = 0x00FFFF00,
110 eFileSizeOffset = 8,
111 eFileGenerationMask = 0x000000FF,
112 eFileReservedMask = 0x4F000000
114 };
116 public:
117 nsDiskCacheRecord()
118 : mHashNumber(0), mEvictionRank(0), mDataLocation(0), mMetaLocation(0)
119 {
120 }
122 bool ValidRecord()
123 {
124 if ((mDataLocation & eReservedMask) || (mMetaLocation & eReservedMask))
125 return false;
126 return true;
127 }
129 // HashNumber accessors
130 uint32_t HashNumber() const { return mHashNumber; }
131 void SetHashNumber( uint32_t hashNumber) { mHashNumber = hashNumber; }
133 // EvictionRank accessors
134 uint32_t EvictionRank() const { return mEvictionRank; }
135 void SetEvictionRank( uint32_t rank) { mEvictionRank = rank ? rank : 1; }
137 // DataLocation accessors
138 bool DataLocationInitialized() const { return 0 != (mDataLocation & eLocationInitializedMask); }
139 void ClearDataLocation() { mDataLocation = 0; }
141 uint32_t DataFile() const
142 {
143 return (uint32_t)(mDataLocation & eLocationSelectorMask) >> eLocationSelectorOffset;
144 }
146 void SetDataBlocks( uint32_t index, uint32_t startBlock, uint32_t blockCount)
147 {
148 // clear everything
149 mDataLocation = 0;
151 // set file index
152 NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index");
153 NS_ASSERTION( index > 0,"invalid location index");
154 mDataLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask;
156 // set startBlock
157 NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number");
158 mDataLocation |= startBlock & eBlockNumberMask;
160 // set blockCount
161 NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count");
162 --blockCount;
163 mDataLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask;
165 mDataLocation |= eLocationInitializedMask;
166 }
168 uint32_t DataBlockCount() const
169 {
170 return (uint32_t)((mDataLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1;
171 }
173 uint32_t DataStartBlock() const
174 {
175 return (mDataLocation & eBlockNumberMask);
176 }
178 uint32_t DataBlockSize() const
179 {
180 return BLOCK_SIZE_FOR_INDEX(DataFile());
181 }
183 uint32_t DataFileSize() const { return (mDataLocation & eFileSizeMask) >> eFileSizeOffset; }
184 void SetDataFileSize(uint32_t size)
185 {
186 NS_ASSERTION((mDataLocation & eFileReservedMask) == 0, "bad location");
187 mDataLocation &= ~eFileSizeMask; // clear eFileSizeMask
188 mDataLocation |= (size << eFileSizeOffset) & eFileSizeMask;
189 }
191 uint8_t DataFileGeneration() const
192 {
193 return (mDataLocation & eFileGenerationMask);
194 }
196 void SetDataFileGeneration( uint8_t generation)
197 {
198 // clear everything, (separate file index = 0)
199 mDataLocation = 0;
200 mDataLocation |= generation & eFileGenerationMask;
201 mDataLocation |= eLocationInitializedMask;
202 }
204 // MetaLocation accessors
205 bool MetaLocationInitialized() const { return 0 != (mMetaLocation & eLocationInitializedMask); }
206 void ClearMetaLocation() { mMetaLocation = 0; }
207 uint32_t MetaLocation() const { return mMetaLocation; }
209 uint32_t MetaFile() const
210 {
211 return (uint32_t)(mMetaLocation & eLocationSelectorMask) >> eLocationSelectorOffset;
212 }
214 void SetMetaBlocks( uint32_t index, uint32_t startBlock, uint32_t blockCount)
215 {
216 // clear everything
217 mMetaLocation = 0;
219 // set file index
220 NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index");
221 NS_ASSERTION( index > 0, "invalid location index");
222 mMetaLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask;
224 // set startBlock
225 NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number");
226 mMetaLocation |= startBlock & eBlockNumberMask;
228 // set blockCount
229 NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count");
230 --blockCount;
231 mMetaLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask;
233 mMetaLocation |= eLocationInitializedMask;
234 }
236 uint32_t MetaBlockCount() const
237 {
238 return (uint32_t)((mMetaLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1;
239 }
241 uint32_t MetaStartBlock() const
242 {
243 return (mMetaLocation & eBlockNumberMask);
244 }
246 uint32_t MetaBlockSize() const
247 {
248 return BLOCK_SIZE_FOR_INDEX(MetaFile());
249 }
251 uint32_t MetaFileSize() const { return (mMetaLocation & eFileSizeMask) >> eFileSizeOffset; }
252 void SetMetaFileSize(uint32_t size)
253 {
254 mMetaLocation &= ~eFileSizeMask; // clear eFileSizeMask
255 mMetaLocation |= (size << eFileSizeOffset) & eFileSizeMask;
256 }
258 uint8_t MetaFileGeneration() const
259 {
260 return (mMetaLocation & eFileGenerationMask);
261 }
263 void SetMetaFileGeneration( uint8_t generation)
264 {
265 // clear everything, (separate file index = 0)
266 mMetaLocation = 0;
267 mMetaLocation |= generation & eFileGenerationMask;
268 mMetaLocation |= eLocationInitializedMask;
269 }
271 uint8_t Generation() const
272 {
273 if ((mDataLocation & eLocationInitializedMask) &&
274 (DataFile() == 0))
275 return DataFileGeneration();
277 if ((mMetaLocation & eLocationInitializedMask) &&
278 (MetaFile() == 0))
279 return MetaFileGeneration();
281 return 0; // no generation
282 }
284 #if defined(IS_LITTLE_ENDIAN)
285 void Swap()
286 {
287 mHashNumber = htonl(mHashNumber);
288 mEvictionRank = htonl(mEvictionRank);
289 mDataLocation = htonl(mDataLocation);
290 mMetaLocation = htonl(mMetaLocation);
291 }
292 #endif
294 #if defined(IS_LITTLE_ENDIAN)
295 void Unswap()
296 {
297 mHashNumber = ntohl(mHashNumber);
298 mEvictionRank = ntohl(mEvictionRank);
299 mDataLocation = ntohl(mDataLocation);
300 mMetaLocation = ntohl(mMetaLocation);
301 }
302 #endif
304 };
307 /******************************************************************************
308 * nsDiskCacheRecordVisitor
309 *****************************************************************************/
311 enum { kDeleteRecordAndContinue = -1,
312 kStopVisitingRecords = 0,
313 kVisitNextRecord = 1
314 };
316 class nsDiskCacheRecordVisitor {
317 public:
319 virtual int32_t VisitRecord( nsDiskCacheRecord * mapRecord) = 0;
320 };
323 /******************************************************************************
324 * nsDiskCacheHeader
325 *****************************************************************************/
327 struct nsDiskCacheHeader {
328 uint32_t mVersion; // cache version.
329 uint32_t mDataSize; // size of cache in units of 1024bytes.
330 int32_t mEntryCount; // number of entries stored in cache.
331 uint32_t mIsDirty; // dirty flag.
332 int32_t mRecordCount; // Number of records
333 uint32_t mEvictionRank[kBuckets]; // Highest EvictionRank of the bucket
334 uint32_t mBucketUsage[kBuckets]; // Number of used entries in the bucket
336 nsDiskCacheHeader()
337 : mVersion(nsDiskCache::kCurrentVersion)
338 , mDataSize(0)
339 , mEntryCount(0)
340 , mIsDirty(true)
341 , mRecordCount(0)
342 {}
344 void Swap()
345 {
346 #if defined(IS_LITTLE_ENDIAN)
347 mVersion = htonl(mVersion);
348 mDataSize = htonl(mDataSize);
349 mEntryCount = htonl(mEntryCount);
350 mIsDirty = htonl(mIsDirty);
351 mRecordCount = htonl(mRecordCount);
353 for (uint32_t i = 0; i < kBuckets ; i++) {
354 mEvictionRank[i] = htonl(mEvictionRank[i]);
355 mBucketUsage[i] = htonl(mBucketUsage[i]);
356 }
357 #endif
358 }
360 void Unswap()
361 {
362 #if defined(IS_LITTLE_ENDIAN)
363 mVersion = ntohl(mVersion);
364 mDataSize = ntohl(mDataSize);
365 mEntryCount = ntohl(mEntryCount);
366 mIsDirty = ntohl(mIsDirty);
367 mRecordCount = ntohl(mRecordCount);
369 for (uint32_t i = 0; i < kBuckets ; i++) {
370 mEvictionRank[i] = ntohl(mEvictionRank[i]);
371 mBucketUsage[i] = ntohl(mBucketUsage[i]);
372 }
373 #endif
374 }
375 };
378 /******************************************************************************
379 * nsDiskCacheMap
380 *****************************************************************************/
382 class nsDiskCacheMap {
383 public:
385 nsDiskCacheMap() :
386 mCacheDirectory(nullptr),
387 mMapFD(nullptr),
388 mCleanFD(nullptr),
389 mRecordArray(nullptr),
390 mBufferSize(0),
391 mBuffer(nullptr),
392 mMaxRecordCount(16384), // this default value won't matter
393 mIsDirtyCacheFlushed(false),
394 mLastInvalidateTime(0)
395 { }
397 ~nsDiskCacheMap()
398 {
399 (void) Close(true);
400 }
402 /**
403 * File Operations
404 *
405 * Open
406 *
407 * Creates a new cache map file if one doesn't exist.
408 * Returns error if it detects change in format or cache wasn't closed.
409 */
410 nsresult Open( nsIFile * cacheDirectory,
411 nsDiskCache::CorruptCacheInfo * corruptInfo,
412 bool reportCacheCleanTelemetryData);
413 nsresult Close(bool flush);
414 nsresult Trim();
416 nsresult FlushHeader();
417 nsresult FlushRecords( bool unswap);
419 void NotifyCapacityChange(uint32_t capacity);
421 /**
422 * Record operations
423 */
424 nsresult AddRecord( nsDiskCacheRecord * mapRecord, nsDiskCacheRecord * oldRecord);
425 nsresult UpdateRecord( nsDiskCacheRecord * mapRecord);
426 nsresult FindRecord( uint32_t hashNumber, nsDiskCacheRecord * mapRecord);
427 nsresult DeleteRecord( nsDiskCacheRecord * mapRecord);
428 nsresult VisitRecords( nsDiskCacheRecordVisitor * visitor);
429 nsresult EvictRecords( nsDiskCacheRecordVisitor * visitor);
431 /**
432 * Disk Entry operations
433 */
434 nsresult DeleteStorage( nsDiskCacheRecord * record);
436 nsresult GetFileForDiskCacheRecord( nsDiskCacheRecord * record,
437 bool meta,
438 bool createPath,
439 nsIFile ** result);
441 nsresult GetLocalFileForDiskCacheRecord( nsDiskCacheRecord * record,
442 bool meta,
443 bool createPath,
444 nsIFile ** result);
446 // On success, this returns the buffer owned by nsDiskCacheMap,
447 // so it must not be deleted by the caller.
448 nsDiskCacheEntry * ReadDiskCacheEntry( nsDiskCacheRecord * record);
450 nsresult WriteDiskCacheEntry( nsDiskCacheBinding * binding);
452 nsresult ReadDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size);
453 nsresult WriteDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size);
454 nsresult DeleteStorage( nsDiskCacheRecord * record, bool metaData);
456 /**
457 * Statistical Operations
458 */
459 void IncrementTotalSize( uint32_t delta)
460 {
461 mHeader.mDataSize += delta;
462 mHeader.mIsDirty = true;
463 }
465 void DecrementTotalSize( uint32_t delta)
466 {
467 NS_ASSERTION(mHeader.mDataSize >= delta, "disk cache size negative?");
468 mHeader.mDataSize = mHeader.mDataSize > delta ? mHeader.mDataSize - delta : 0;
469 mHeader.mIsDirty = true;
470 }
472 inline void IncrementTotalSize( uint32_t blocks, uint32_t blockSize)
473 {
474 // Round up to nearest K
475 IncrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10);
476 }
478 inline void DecrementTotalSize( uint32_t blocks, uint32_t blockSize)
479 {
480 // Round up to nearest K
481 DecrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10);
482 }
484 uint32_t TotalSize() { return mHeader.mDataSize; }
486 int32_t EntryCount() { return mHeader.mEntryCount; }
488 size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
491 private:
493 /**
494 * Private methods
495 */
496 nsresult OpenBlockFiles(nsDiskCache::CorruptCacheInfo * corruptInfo);
497 nsresult CloseBlockFiles(bool flush);
498 bool CacheFilesExist();
500 nsresult CreateCacheSubDirectories();
502 uint32_t CalculateFileIndex(uint32_t size);
504 nsresult GetBlockFileForIndex( uint32_t index, nsIFile ** result);
505 uint32_t GetBlockSizeForIndex( uint32_t index) const {
506 return BLOCK_SIZE_FOR_INDEX(index);
507 }
508 uint32_t GetBitMapSizeForIndex( uint32_t index) const {
509 return BITMAP_SIZE_FOR_INDEX(index);
510 }
512 // returns the bucket number
513 uint32_t GetBucketIndex( uint32_t hashNumber) const {
514 return (hashNumber & (kBuckets - 1));
515 }
517 // Gets the size of the bucket (in number of records)
518 uint32_t GetRecordsPerBucket() const {
519 return mHeader.mRecordCount / kBuckets;
520 }
522 // Gets the first record in the bucket
523 nsDiskCacheRecord *GetFirstRecordInBucket(uint32_t bucket) const {
524 return mRecordArray + bucket * GetRecordsPerBucket();
525 }
527 uint32_t GetBucketRank(uint32_t bucketIndex, uint32_t targetRank);
529 int32_t VisitEachRecord(uint32_t bucketIndex,
530 nsDiskCacheRecordVisitor * visitor,
531 uint32_t evictionRank);
533 nsresult GrowRecords();
534 nsresult ShrinkRecords();
536 nsresult EnsureBuffer(uint32_t bufSize);
538 // The returned structure will point to the buffer owned by nsDiskCacheMap,
539 // so it must not be deleted by the caller.
540 nsDiskCacheEntry * CreateDiskCacheEntry(nsDiskCacheBinding * binding,
541 uint32_t * size);
543 // Initializes the _CACHE_CLEAN_ related functionality
544 nsresult InitCacheClean(nsIFile * cacheDirectory,
545 nsDiskCache::CorruptCacheInfo * corruptInfo,
546 bool reportCacheCleanTelemetryData);
547 // Writes out a value of '0' or '1' in the _CACHE_CLEAN_ file
548 nsresult WriteCacheClean(bool clean);
549 // Resets the timout for revalidating the cache
550 nsresult ResetCacheTimer(int32_t timeout = kRevalidateCacheTimeout);
551 // Invalidates the cache, calls WriteCacheClean and ResetCacheTimer
552 nsresult InvalidateCache();
553 // Determines if the cache is in a safe state
554 bool IsCacheInSafeState();
555 // Revalidates the cache by writting out the header, records, and finally
556 // by calling WriteCacheClean(true).
557 nsresult RevalidateCache();
558 // Timer which revalidates the cache
559 static void RevalidateTimerCallback(nsITimer *aTimer, void *arg);
561 /**
562 * data members
563 */
564 private:
565 nsCOMPtr<nsITimer> mCleanCacheTimer;
566 nsCOMPtr<nsIFile> mCacheDirectory;
567 PRFileDesc * mMapFD;
568 PRFileDesc * mCleanFD;
569 nsDiskCacheRecord * mRecordArray;
570 nsDiskCacheBlockFile mBlockFile[kNumBlockFiles];
571 uint32_t mBufferSize;
572 char * mBuffer;
573 nsDiskCacheHeader mHeader;
574 int32_t mMaxRecordCount;
575 bool mIsDirtyCacheFlushed;
576 PRIntervalTime mLastInvalidateTime;
577 };
579 #endif // _nsDiskCacheMap_h_
