• file
• revisions
• annotate
• diff
• comparison
• raw

toolkit/components/url-classifier/HashStore.h@b8a032363ba2 (annotated)

toolkit/components/url-classifier/HashStore.h

Thu, 22 Jan 2015 13:21:57 +0100

author

Michael Schloh von Bennewitz <michael@schloh.com>

date

Thu, 22 Jan 2015 13:21:57 +0100

branch

TOR_BUG_9701

changeset 15

b8a032363ba2

permissions

-rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

michael@0	1	/* This Source Code Form is subject to the terms of the Mozilla Public
michael@0	2	* License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0	3	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0	4
michael@0	5	#ifndef HashStore_h__
michael@0	6	#define HashStore_h__
michael@0	7
michael@0	8	#include "Entries.h"
michael@0	9	#include "ChunkSet.h"
michael@0	10
michael@0	11	#include "nsString.h"
michael@0	12	#include "nsTArray.h"
michael@0	13	#include "nsIFile.h"
michael@0	14	#include "nsIFileStreams.h"
michael@0	15	#include "nsCOMPtr.h"
michael@0	16
michael@0	17	namespace mozilla {
michael@0	18	namespace safebrowsing {
michael@0	19
michael@0	20	// A table update is built from a single update chunk from the server. As the
michael@0	21	// protocol parser processes each chunk, it constructs a table update with the
michael@0	22	// new hashes.
michael@0	23	class TableUpdate {
michael@0	24	public:
michael@0	25	TableUpdate(const nsACString& aTable)
michael@0	26	: mTable(aTable), mLocalUpdate(false) {}
michael@0	27	const nsCString& TableName() const { return mTable; }
michael@0	28
michael@0	29	bool Empty() const {
michael@0	30	return mAddChunks.Length() == 0 &&
michael@0	31	mSubChunks.Length() == 0 &&
michael@0	32	mAddExpirations.Length() == 0 &&
michael@0	33	mSubExpirations.Length() == 0 &&
michael@0	34	mAddPrefixes.Length() == 0 &&
michael@0	35	mSubPrefixes.Length() == 0 &&
michael@0	36	mAddCompletes.Length() == 0 &&
michael@0	37	mSubCompletes.Length() == 0;
michael@0	38	}
michael@0	39
michael@0	40	// Throughout, uint32_t aChunk refers only to the chunk number. Chunk data is
michael@0	41	// stored in the Prefix structures.
michael@0	42	void NewAddChunk(uint32_t aChunk) { mAddChunks.Set(aChunk); }
michael@0	43	void NewSubChunk(uint32_t aChunk) { mSubChunks.Set(aChunk); }
michael@0	44
michael@0	45	void NewAddExpiration(uint32_t aChunk) { mAddExpirations.Set(aChunk); }
michael@0	46	void NewSubExpiration(uint32_t aChunk) { mSubExpirations.Set(aChunk); }
michael@0	47
michael@0	48	void NewAddPrefix(uint32_t aAddChunk, const Prefix& aPrefix);
michael@0	49	void NewSubPrefix(uint32_t aAddChunk, const Prefix& aPrefix, uint32_t aSubChunk);
michael@0	50
michael@0	51	void NewAddComplete(uint32_t aChunk, const Completion& aCompletion);
michael@0	52	void NewSubComplete(uint32_t aAddChunk, const Completion& aCompletion,
michael@0	53	uint32_t aSubChunk);
michael@0	54	void SetLocalUpdate(void) { mLocalUpdate = true; }
michael@0	55	bool IsLocalUpdate(void) { return mLocalUpdate; }
michael@0	56
michael@0	57	ChunkSet& AddChunks() { return mAddChunks; }
michael@0	58	ChunkSet& SubChunks() { return mSubChunks; }
michael@0	59
michael@0	60	// Expirations for chunks.
michael@0	61	ChunkSet& AddExpirations() { return mAddExpirations; }
michael@0	62	ChunkSet& SubExpirations() { return mSubExpirations; }
michael@0	63
michael@0	64	// Hashes associated with this chunk.
michael@0	65	AddPrefixArray& AddPrefixes() { return mAddPrefixes; }
michael@0	66	SubPrefixArray& SubPrefixes() { return mSubPrefixes; }
michael@0	67	AddCompleteArray& AddCompletes() { return mAddCompletes; }
michael@0	68	SubCompleteArray& SubCompletes() { return mSubCompletes; }
michael@0	69
michael@0	70	private:
michael@0	71	nsCString mTable;
michael@0	72	// Update not from the remote server (no freshness)
michael@0	73	bool mLocalUpdate;
michael@0	74
michael@0	75	// The list of chunk numbers that we have for each of the type of chunks.
michael@0	76	ChunkSet mAddChunks;
michael@0	77	ChunkSet mSubChunks;
michael@0	78	ChunkSet mAddExpirations;
michael@0	79	ChunkSet mSubExpirations;
michael@0	80
michael@0	81	// 4-byte sha256 prefixes.
michael@0	82	AddPrefixArray mAddPrefixes;
michael@0	83	SubPrefixArray mSubPrefixes;
michael@0	84
michael@0	85	// 32-byte hashes.
michael@0	86	AddCompleteArray mAddCompletes;
michael@0	87	SubCompleteArray mSubCompletes;
michael@0	88	};
michael@0	89
michael@0	90	// There is one hash store per table.
michael@0	91	class HashStore {
michael@0	92	public:
michael@0	93	HashStore(const nsACString& aTableName, nsIFile* aStoreFile);
michael@0	94	~HashStore();
michael@0	95
michael@0	96	const nsCString& TableName() const { return mTableName; }
michael@0	97
michael@0	98	nsresult Open();
michael@0	99	// Add Prefixes are stored partly in the PrefixSet (contains the
michael@0	100	// Prefix data organized for fast lookup/low RAM usage) and partly in the
michael@0	101	// HashStore (Add Chunk numbers - only used for updates, slow retrieval).
michael@0	102	// AugmentAdds function joins the separate datasets into one complete
michael@0	103	// prefixes+chunknumbers dataset.
michael@0	104	nsresult AugmentAdds(const nsTArray<uint32_t>& aPrefixes);
michael@0	105
michael@0	106	ChunkSet& AddChunks() { return mAddChunks; }
michael@0	107	ChunkSet& SubChunks() { return mSubChunks; }
michael@0	108	AddPrefixArray& AddPrefixes() { return mAddPrefixes; }
michael@0	109	AddCompleteArray& AddCompletes() { return mAddCompletes; }
michael@0	110	SubPrefixArray& SubPrefixes() { return mSubPrefixes; }
michael@0	111	SubCompleteArray& SubCompletes() { return mSubCompletes; }
michael@0	112
michael@0	113	// =======
michael@0	114	// Updates
michael@0	115	// =======
michael@0	116	// Begin the update process. Reads the store into memory.
michael@0	117	nsresult BeginUpdate();
michael@0	118
michael@0	119	// Imports the data from a TableUpdate.
michael@0	120	nsresult ApplyUpdate(TableUpdate &aUpdate);
michael@0	121
michael@0	122	// Process expired chunks
michael@0	123	nsresult Expire();
michael@0	124
michael@0	125	// Rebuild the store, Incorporating all the applied updates.
michael@0	126	nsresult Rebuild();
michael@0	127
michael@0	128	// Write the current state of the store to disk.
michael@0	129	// If you call between ApplyUpdate() and Rebuild(), you'll
michael@0	130	// have a mess on your hands.
michael@0	131	nsresult WriteFile();
michael@0	132
michael@0	133	// Wipe out all Completes.
michael@0	134	void ClearCompletes();
michael@0	135
michael@0	136	private:
michael@0	137	nsresult Reset();
michael@0	138
michael@0	139	nsresult ReadHeader();
michael@0	140	nsresult SanityCheck();
michael@0	141	nsresult CalculateChecksum(nsAutoCString& aChecksum, uint32_t aFileSize,
michael@0	142	bool aChecksumPresent);
michael@0	143	nsresult CheckChecksum(nsIFile* aStoreFile, uint32_t aFileSize);
michael@0	144	void UpdateHeader();
michael@0	145
michael@0	146	nsresult ReadChunkNumbers();
michael@0	147	nsresult ReadHashes();
michael@0	148
michael@0	149	nsresult ReadAddPrefixes();
michael@0	150	nsresult ReadSubPrefixes();
michael@0	151
michael@0	152	nsresult WriteAddPrefixes(nsIOutputStream* aOut);
michael@0	153	nsresult WriteSubPrefixes(nsIOutputStream* aOut);
michael@0	154
michael@0	155	nsresult ProcessSubs();
michael@0	156
michael@0	157	// This is used for checking that the database is correct and for figuring out
michael@0	158	// the number of chunks, etc. to read from disk on restart.
michael@0	159	struct Header {
michael@0	160	uint32_t magic;
michael@0	161	uint32_t version;
michael@0	162	uint32_t numAddChunks;
michael@0	163	uint32_t numSubChunks;
michael@0	164	uint32_t numAddPrefixes;
michael@0	165	uint32_t numSubPrefixes;
michael@0	166	uint32_t numAddCompletes;
michael@0	167	uint32_t numSubCompletes;
michael@0	168	};
michael@0	169
michael@0	170	Header mHeader;
michael@0	171
michael@0	172	// The name of the table (must end in -shavar or -digest256, or evidently
michael@0	173	// -simple for unittesting.
michael@0	174	nsCString mTableName;
michael@0	175	nsCOMPtr<nsIFile> mStoreDirectory;
michael@0	176
michael@0	177	bool mInUpdate;
michael@0	178
michael@0	179	nsCOMPtr<nsIInputStream> mInputStream;
michael@0	180
michael@0	181	// Chunk numbers, stored as uint32_t arrays.
michael@0	182	ChunkSet mAddChunks;
michael@0	183	ChunkSet mSubChunks;
michael@0	184
michael@0	185	ChunkSet mAddExpirations;
michael@0	186	ChunkSet mSubExpirations;
michael@0	187
michael@0	188	// Chunk data for shavar tables. See Entries.h for format.
michael@0	189	AddPrefixArray mAddPrefixes;
michael@0	190	SubPrefixArray mSubPrefixes;
michael@0	191
michael@0	192	// See bug 806422 for background. We must be able to distinguish between
michael@0	193	// updates from the completion server and updates from the regular server.
michael@0	194	AddCompleteArray mAddCompletes;
michael@0	195	SubCompleteArray mSubCompletes;
michael@0	196	};
michael@0	197
michael@0	198	}
michael@0	199	}
michael@0	200	#endif