1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/toolkit/components/url-classifier/HashStore.h Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,200 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +#ifndef HashStore_h__
1.9 +#define HashStore_h__
1.10 +
1.11 +#include "Entries.h"
1.12 +#include "ChunkSet.h"
1.13 +
1.14 +#include "nsString.h"
1.15 +#include "nsTArray.h"
1.16 +#include "nsIFile.h"
1.17 +#include "nsIFileStreams.h"
1.18 +#include "nsCOMPtr.h"
1.19 +
1.20 +namespace mozilla {
1.21 +namespace safebrowsing {
1.22 +
1.23 +// A table update is built from a single update chunk from the server. As the
1.24 +// protocol parser processes each chunk, it constructs a table update with the
1.25 +// new hashes.
1.26 +class TableUpdate {
1.27 +public:
1.28 + TableUpdate(const nsACString& aTable)
1.29 + : mTable(aTable), mLocalUpdate(false) {}
1.30 + const nsCString& TableName() const { return mTable; }
1.31 +
1.32 + bool Empty() const {
1.33 + return mAddChunks.Length() == 0 &&
1.34 + mSubChunks.Length() == 0 &&
1.35 + mAddExpirations.Length() == 0 &&
1.36 + mSubExpirations.Length() == 0 &&
1.37 + mAddPrefixes.Length() == 0 &&
1.38 + mSubPrefixes.Length() == 0 &&
1.39 + mAddCompletes.Length() == 0 &&
1.40 + mSubCompletes.Length() == 0;
1.41 + }
1.42 +
1.43 + // Throughout, uint32_t aChunk refers only to the chunk number. Chunk data is
1.44 + // stored in the Prefix structures.
1.45 + void NewAddChunk(uint32_t aChunk) { mAddChunks.Set(aChunk); }
1.46 + void NewSubChunk(uint32_t aChunk) { mSubChunks.Set(aChunk); }
1.47 +
1.48 + void NewAddExpiration(uint32_t aChunk) { mAddExpirations.Set(aChunk); }
1.49 + void NewSubExpiration(uint32_t aChunk) { mSubExpirations.Set(aChunk); }
1.50 +
1.51 + void NewAddPrefix(uint32_t aAddChunk, const Prefix& aPrefix);
1.52 + void NewSubPrefix(uint32_t aAddChunk, const Prefix& aPrefix, uint32_t aSubChunk);
1.53 +
1.54 + void NewAddComplete(uint32_t aChunk, const Completion& aCompletion);
1.55 + void NewSubComplete(uint32_t aAddChunk, const Completion& aCompletion,
1.56 + uint32_t aSubChunk);
1.57 + void SetLocalUpdate(void) { mLocalUpdate = true; }
1.58 + bool IsLocalUpdate(void) { return mLocalUpdate; }
1.59 +
1.60 + ChunkSet& AddChunks() { return mAddChunks; }
1.61 + ChunkSet& SubChunks() { return mSubChunks; }
1.62 +
1.63 + // Expirations for chunks.
1.64 + ChunkSet& AddExpirations() { return mAddExpirations; }
1.65 + ChunkSet& SubExpirations() { return mSubExpirations; }
1.66 +
1.67 + // Hashes associated with this chunk.
1.68 + AddPrefixArray& AddPrefixes() { return mAddPrefixes; }
1.69 + SubPrefixArray& SubPrefixes() { return mSubPrefixes; }
1.70 + AddCompleteArray& AddCompletes() { return mAddCompletes; }
1.71 + SubCompleteArray& SubCompletes() { return mSubCompletes; }
1.72 +
1.73 +private:
1.74 + nsCString mTable;
1.75 + // Update not from the remote server (no freshness)
1.76 + bool mLocalUpdate;
1.77 +
1.78 + // The list of chunk numbers that we have for each of the type of chunks.
1.79 + ChunkSet mAddChunks;
1.80 + ChunkSet mSubChunks;
1.81 + ChunkSet mAddExpirations;
1.82 + ChunkSet mSubExpirations;
1.83 +
1.84 + // 4-byte sha256 prefixes.
1.85 + AddPrefixArray mAddPrefixes;
1.86 + SubPrefixArray mSubPrefixes;
1.87 +
1.88 + // 32-byte hashes.
1.89 + AddCompleteArray mAddCompletes;
1.90 + SubCompleteArray mSubCompletes;
1.91 +};
1.92 +
1.93 +// There is one hash store per table.
1.94 +class HashStore {
1.95 +public:
1.96 + HashStore(const nsACString& aTableName, nsIFile* aStoreFile);
1.97 + ~HashStore();
1.98 +
1.99 + const nsCString& TableName() const { return mTableName; }
1.100 +
1.101 + nsresult Open();
1.102 + // Add Prefixes are stored partly in the PrefixSet (contains the
1.103 + // Prefix data organized for fast lookup/low RAM usage) and partly in the
1.104 + // HashStore (Add Chunk numbers - only used for updates, slow retrieval).
1.105 + // AugmentAdds function joins the separate datasets into one complete
1.106 + // prefixes+chunknumbers dataset.
1.107 + nsresult AugmentAdds(const nsTArray<uint32_t>& aPrefixes);
1.108 +
1.109 + ChunkSet& AddChunks() { return mAddChunks; }
1.110 + ChunkSet& SubChunks() { return mSubChunks; }
1.111 + AddPrefixArray& AddPrefixes() { return mAddPrefixes; }
1.112 + AddCompleteArray& AddCompletes() { return mAddCompletes; }
1.113 + SubPrefixArray& SubPrefixes() { return mSubPrefixes; }
1.114 + SubCompleteArray& SubCompletes() { return mSubCompletes; }
1.115 +
1.116 + // =======
1.117 + // Updates
1.118 + // =======
1.119 + // Begin the update process. Reads the store into memory.
1.120 + nsresult BeginUpdate();
1.121 +
1.122 + // Imports the data from a TableUpdate.
1.123 + nsresult ApplyUpdate(TableUpdate &aUpdate);
1.124 +
1.125 + // Process expired chunks
1.126 + nsresult Expire();
1.127 +
1.128 + // Rebuild the store, Incorporating all the applied updates.
1.129 + nsresult Rebuild();
1.130 +
1.131 + // Write the current state of the store to disk.
1.132 + // If you call between ApplyUpdate() and Rebuild(), you'll
1.133 + // have a mess on your hands.
1.134 + nsresult WriteFile();
1.135 +
1.136 + // Wipe out all Completes.
1.137 + void ClearCompletes();
1.138 +
1.139 +private:
1.140 + nsresult Reset();
1.141 +
1.142 + nsresult ReadHeader();
1.143 + nsresult SanityCheck();
1.144 + nsresult CalculateChecksum(nsAutoCString& aChecksum, uint32_t aFileSize,
1.145 + bool aChecksumPresent);
1.146 + nsresult CheckChecksum(nsIFile* aStoreFile, uint32_t aFileSize);
1.147 + void UpdateHeader();
1.148 +
1.149 + nsresult ReadChunkNumbers();
1.150 + nsresult ReadHashes();
1.151 +
1.152 + nsresult ReadAddPrefixes();
1.153 + nsresult ReadSubPrefixes();
1.154 +
1.155 + nsresult WriteAddPrefixes(nsIOutputStream* aOut);
1.156 + nsresult WriteSubPrefixes(nsIOutputStream* aOut);
1.157 +
1.158 + nsresult ProcessSubs();
1.159 +
1.160 + // This is used for checking that the database is correct and for figuring out
1.161 + // the number of chunks, etc. to read from disk on restart.
1.162 + struct Header {
1.163 + uint32_t magic;
1.164 + uint32_t version;
1.165 + uint32_t numAddChunks;
1.166 + uint32_t numSubChunks;
1.167 + uint32_t numAddPrefixes;
1.168 + uint32_t numSubPrefixes;
1.169 + uint32_t numAddCompletes;
1.170 + uint32_t numSubCompletes;
1.171 + };
1.172 +
1.173 + Header mHeader;
1.174 +
1.175 + // The name of the table (must end in -shavar or -digest256, or evidently
1.176 + // -simple for unittesting.
1.177 + nsCString mTableName;
1.178 + nsCOMPtr<nsIFile> mStoreDirectory;
1.179 +
1.180 + bool mInUpdate;
1.181 +
1.182 + nsCOMPtr<nsIInputStream> mInputStream;
1.183 +
1.184 + // Chunk numbers, stored as uint32_t arrays.
1.185 + ChunkSet mAddChunks;
1.186 + ChunkSet mSubChunks;
1.187 +
1.188 + ChunkSet mAddExpirations;
1.189 + ChunkSet mSubExpirations;
1.190 +
1.191 + // Chunk data for shavar tables. See Entries.h for format.
1.192 + AddPrefixArray mAddPrefixes;
1.193 + SubPrefixArray mSubPrefixes;
1.194 +
1.195 + // See bug 806422 for background. We must be able to distinguish between
1.196 + // updates from the completion server and updates from the regular server.
1.197 + AddCompleteArray mAddCompletes;
1.198 + SubCompleteArray mSubCompletes;
1.199 +};
1.200 +
1.201 +}
1.202 +}
1.203 +#endif
