Fri, 16 Jan 2015 18:13:44 +0100
Integrate suggestion from review to improve consistency with existing code.
michael@0 | 1 | //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ |
michael@0 | 2 | /* This Source Code Form is subject to the terms of the Mozilla Public |
michael@0 | 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this |
michael@0 | 4 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
michael@0 | 5 | |
michael@0 | 6 | #include "Classifier.h" |
michael@0 | 7 | #include "nsIPrefBranch.h" |
michael@0 | 8 | #include "nsIPrefService.h" |
michael@0 | 9 | #include "nsISimpleEnumerator.h" |
michael@0 | 10 | #include "nsIRandomGenerator.h" |
michael@0 | 11 | #include "nsIInputStream.h" |
michael@0 | 12 | #include "nsISeekableStream.h" |
michael@0 | 13 | #include "nsIFile.h" |
michael@0 | 14 | #include "nsAutoPtr.h" |
michael@0 | 15 | #include "mozilla/Telemetry.h" |
michael@0 | 16 | #include "prlog.h" |
michael@0 | 17 | |
michael@0 | 18 | // NSPR_LOG_MODULES=UrlClassifierDbService:5 |
michael@0 | 19 | extern PRLogModuleInfo *gUrlClassifierDbServiceLog; |
michael@0 | 20 | #if defined(PR_LOGGING) |
michael@0 | 21 | #define LOG(args) PR_LOG(gUrlClassifierDbServiceLog, PR_LOG_DEBUG, args) |
michael@0 | 22 | #define LOG_ENABLED() PR_LOG_TEST(gUrlClassifierDbServiceLog, 4) |
michael@0 | 23 | #else |
michael@0 | 24 | #define LOG(args) |
michael@0 | 25 | #define LOG_ENABLED() (false) |
michael@0 | 26 | #endif |
michael@0 | 27 | |
michael@0 | 28 | #define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing") |
michael@0 | 29 | #define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete") |
michael@0 | 30 | #define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup") |
michael@0 | 31 | |
michael@0 | 32 | namespace mozilla { |
michael@0 | 33 | namespace safebrowsing { |
michael@0 | 34 | |
michael@0 | 35 | void |
michael@0 | 36 | Classifier::SplitTables(const nsACString& str, nsTArray<nsCString>& tables) |
michael@0 | 37 | { |
michael@0 | 38 | tables.Clear(); |
michael@0 | 39 | |
michael@0 | 40 | nsACString::const_iterator begin, iter, end; |
michael@0 | 41 | str.BeginReading(begin); |
michael@0 | 42 | str.EndReading(end); |
michael@0 | 43 | while (begin != end) { |
michael@0 | 44 | iter = begin; |
michael@0 | 45 | FindCharInReadable(',', iter, end); |
michael@0 | 46 | nsDependentCSubstring table = Substring(begin,iter); |
michael@0 | 47 | if (!table.IsEmpty()) { |
michael@0 | 48 | tables.AppendElement(Substring(begin, iter)); |
michael@0 | 49 | } |
michael@0 | 50 | begin = iter; |
michael@0 | 51 | if (begin != end) { |
michael@0 | 52 | begin++; |
michael@0 | 53 | } |
michael@0 | 54 | } |
michael@0 | 55 | } |
michael@0 | 56 | |
michael@0 | 57 | Classifier::Classifier() |
michael@0 | 58 | : mFreshTime(45 * 60) |
michael@0 | 59 | { |
michael@0 | 60 | } |
michael@0 | 61 | |
michael@0 | 62 | Classifier::~Classifier() |
michael@0 | 63 | { |
michael@0 | 64 | Close(); |
michael@0 | 65 | } |
michael@0 | 66 | |
michael@0 | 67 | nsresult |
michael@0 | 68 | Classifier::SetupPathNames() |
michael@0 | 69 | { |
michael@0 | 70 | // Get the root directory where to store all the databases. |
michael@0 | 71 | nsresult rv = mCacheDirectory->Clone(getter_AddRefs(mStoreDirectory)); |
michael@0 | 72 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 73 | |
michael@0 | 74 | rv = mStoreDirectory->AppendNative(STORE_DIRECTORY); |
michael@0 | 75 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 76 | |
michael@0 | 77 | // Make sure LookupCaches (which are persistent and survive updates) |
michael@0 | 78 | // are reading/writing in the right place. We will be moving their |
michael@0 | 79 | // files "underneath" them during backup/restore. |
michael@0 | 80 | for (uint32_t i = 0; i < mLookupCaches.Length(); i++) { |
michael@0 | 81 | mLookupCaches[i]->UpdateDirHandle(mStoreDirectory); |
michael@0 | 82 | } |
michael@0 | 83 | |
michael@0 | 84 | // Directory where to move a backup before an update. |
michael@0 | 85 | rv = mCacheDirectory->Clone(getter_AddRefs(mBackupDirectory)); |
michael@0 | 86 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 87 | |
michael@0 | 88 | rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX); |
michael@0 | 89 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 90 | |
michael@0 | 91 | // Directory where to move the backup so we can atomically |
michael@0 | 92 | // delete (really move) it. |
michael@0 | 93 | rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory)); |
michael@0 | 94 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 95 | |
michael@0 | 96 | rv = mToDeleteDirectory->AppendNative(STORE_DIRECTORY + TO_DELETE_DIR_SUFFIX); |
michael@0 | 97 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 98 | |
michael@0 | 99 | return NS_OK; |
michael@0 | 100 | } |
michael@0 | 101 | |
michael@0 | 102 | nsresult |
michael@0 | 103 | Classifier::CreateStoreDirectory() |
michael@0 | 104 | { |
michael@0 | 105 | // Ensure the safebrowsing directory exists. |
michael@0 | 106 | bool storeExists; |
michael@0 | 107 | nsresult rv = mStoreDirectory->Exists(&storeExists); |
michael@0 | 108 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 109 | |
michael@0 | 110 | if (!storeExists) { |
michael@0 | 111 | rv = mStoreDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755); |
michael@0 | 112 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 113 | } else { |
michael@0 | 114 | bool storeIsDir; |
michael@0 | 115 | rv = mStoreDirectory->IsDirectory(&storeIsDir); |
michael@0 | 116 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 117 | if (!storeIsDir) |
michael@0 | 118 | return NS_ERROR_FILE_DESTINATION_NOT_DIR; |
michael@0 | 119 | } |
michael@0 | 120 | |
michael@0 | 121 | return NS_OK; |
michael@0 | 122 | } |
michael@0 | 123 | |
michael@0 | 124 | nsresult |
michael@0 | 125 | Classifier::Open(nsIFile& aCacheDirectory) |
michael@0 | 126 | { |
michael@0 | 127 | // Remember the Local profile directory. |
michael@0 | 128 | nsresult rv = aCacheDirectory.Clone(getter_AddRefs(mCacheDirectory)); |
michael@0 | 129 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 130 | |
michael@0 | 131 | // Create the handles to the update and backup directories. |
michael@0 | 132 | rv = SetupPathNames(); |
michael@0 | 133 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 134 | |
michael@0 | 135 | // Clean up any to-delete directories that haven't been deleted yet. |
michael@0 | 136 | rv = CleanToDelete(); |
michael@0 | 137 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 138 | |
michael@0 | 139 | // Check whether we have an incomplete update and recover from the |
michael@0 | 140 | // backup if so. |
michael@0 | 141 | rv = RecoverBackups(); |
michael@0 | 142 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 143 | |
michael@0 | 144 | // Make sure the main store directory exists. |
michael@0 | 145 | rv = CreateStoreDirectory(); |
michael@0 | 146 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 147 | |
michael@0 | 148 | mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); |
michael@0 | 149 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 150 | |
michael@0 | 151 | // Build the list of know urlclassifier lists |
michael@0 | 152 | // XXX: Disk IO potentially on the main thread during startup |
michael@0 | 153 | RegenActiveTables(); |
michael@0 | 154 | |
michael@0 | 155 | return NS_OK; |
michael@0 | 156 | } |
michael@0 | 157 | |
michael@0 | 158 | void |
michael@0 | 159 | Classifier::Close() |
michael@0 | 160 | { |
michael@0 | 161 | DropStores(); |
michael@0 | 162 | } |
michael@0 | 163 | |
michael@0 | 164 | void |
michael@0 | 165 | Classifier::Reset() |
michael@0 | 166 | { |
michael@0 | 167 | DropStores(); |
michael@0 | 168 | |
michael@0 | 169 | mStoreDirectory->Remove(true); |
michael@0 | 170 | mBackupDirectory->Remove(true); |
michael@0 | 171 | mToDeleteDirectory->Remove(true); |
michael@0 | 172 | |
michael@0 | 173 | CreateStoreDirectory(); |
michael@0 | 174 | |
michael@0 | 175 | mTableFreshness.Clear(); |
michael@0 | 176 | RegenActiveTables(); |
michael@0 | 177 | } |
michael@0 | 178 | |
michael@0 | 179 | void |
michael@0 | 180 | Classifier::TableRequest(nsACString& aResult) |
michael@0 | 181 | { |
michael@0 | 182 | nsTArray<nsCString> tables; |
michael@0 | 183 | ActiveTables(tables); |
michael@0 | 184 | for (uint32_t i = 0; i < tables.Length(); i++) { |
michael@0 | 185 | nsAutoPtr<HashStore> store(new HashStore(tables[i], mStoreDirectory)); |
michael@0 | 186 | if (!store) |
michael@0 | 187 | continue; |
michael@0 | 188 | |
michael@0 | 189 | nsresult rv = store->Open(); |
michael@0 | 190 | if (NS_FAILED(rv)) |
michael@0 | 191 | continue; |
michael@0 | 192 | |
michael@0 | 193 | aResult.Append(store->TableName()); |
michael@0 | 194 | aResult.Append(";"); |
michael@0 | 195 | |
michael@0 | 196 | ChunkSet &adds = store->AddChunks(); |
michael@0 | 197 | ChunkSet &subs = store->SubChunks(); |
michael@0 | 198 | |
michael@0 | 199 | if (adds.Length() > 0) { |
michael@0 | 200 | aResult.Append("a:"); |
michael@0 | 201 | nsAutoCString addList; |
michael@0 | 202 | adds.Serialize(addList); |
michael@0 | 203 | aResult.Append(addList); |
michael@0 | 204 | } |
michael@0 | 205 | |
michael@0 | 206 | if (subs.Length() > 0) { |
michael@0 | 207 | if (adds.Length() > 0) |
michael@0 | 208 | aResult.Append(':'); |
michael@0 | 209 | aResult.Append("s:"); |
michael@0 | 210 | nsAutoCString subList; |
michael@0 | 211 | subs.Serialize(subList); |
michael@0 | 212 | aResult.Append(subList); |
michael@0 | 213 | } |
michael@0 | 214 | |
michael@0 | 215 | aResult.Append('\n'); |
michael@0 | 216 | } |
michael@0 | 217 | } |
michael@0 | 218 | |
michael@0 | 219 | nsresult |
michael@0 | 220 | Classifier::Check(const nsACString& aSpec, |
michael@0 | 221 | const nsACString& aTables, |
michael@0 | 222 | LookupResultArray& aResults) |
michael@0 | 223 | { |
michael@0 | 224 | Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_CHECK_TIME> timer; |
michael@0 | 225 | |
michael@0 | 226 | // Get the set of fragments based on the url. This is necessary because we |
michael@0 | 227 | // only look up at most 5 URLs per aSpec, even if aSpec has more than 5 |
michael@0 | 228 | // components. |
michael@0 | 229 | nsTArray<nsCString> fragments; |
michael@0 | 230 | nsresult rv = LookupCache::GetLookupFragments(aSpec, &fragments); |
michael@0 | 231 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 232 | |
michael@0 | 233 | nsTArray<nsCString> activeTables; |
michael@0 | 234 | SplitTables(aTables, activeTables); |
michael@0 | 235 | |
michael@0 | 236 | nsTArray<LookupCache*> cacheArray; |
michael@0 | 237 | for (uint32_t i = 0; i < activeTables.Length(); i++) { |
michael@0 | 238 | LOG(("Checking table %s", activeTables[i].get())); |
michael@0 | 239 | LookupCache *cache = GetLookupCache(activeTables[i]); |
michael@0 | 240 | if (cache) { |
michael@0 | 241 | cacheArray.AppendElement(cache); |
michael@0 | 242 | } else { |
michael@0 | 243 | return NS_ERROR_FAILURE; |
michael@0 | 244 | } |
michael@0 | 245 | } |
michael@0 | 246 | |
michael@0 | 247 | // Now check each lookup fragment against the entries in the DB. |
michael@0 | 248 | for (uint32_t i = 0; i < fragments.Length(); i++) { |
michael@0 | 249 | Completion lookupHash; |
michael@0 | 250 | lookupHash.FromPlaintext(fragments[i], mCryptoHash); |
michael@0 | 251 | |
michael@0 | 252 | // Get list of host keys to look up |
michael@0 | 253 | Completion hostKey; |
michael@0 | 254 | rv = LookupCache::GetKey(fragments[i], &hostKey, mCryptoHash); |
michael@0 | 255 | if (NS_FAILED(rv)) { |
michael@0 | 256 | // Local host on the network. |
michael@0 | 257 | continue; |
michael@0 | 258 | } |
michael@0 | 259 | |
michael@0 | 260 | #if DEBUG && defined(PR_LOGGING) |
michael@0 | 261 | if (LOG_ENABLED()) { |
michael@0 | 262 | nsAutoCString checking; |
michael@0 | 263 | lookupHash.ToHexString(checking); |
michael@0 | 264 | LOG(("Checking fragment %s, hash %s (%X)", fragments[i].get(), |
michael@0 | 265 | checking.get(), lookupHash.ToUint32())); |
michael@0 | 266 | } |
michael@0 | 267 | #endif |
michael@0 | 268 | for (uint32_t i = 0; i < cacheArray.Length(); i++) { |
michael@0 | 269 | LookupCache *cache = cacheArray[i]; |
michael@0 | 270 | bool has, complete; |
michael@0 | 271 | rv = cache->Has(lookupHash, &has, &complete); |
michael@0 | 272 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 273 | if (has) { |
michael@0 | 274 | LookupResult *result = aResults.AppendElement(); |
michael@0 | 275 | if (!result) |
michael@0 | 276 | return NS_ERROR_OUT_OF_MEMORY; |
michael@0 | 277 | |
michael@0 | 278 | int64_t age; |
michael@0 | 279 | bool found = mTableFreshness.Get(cache->TableName(), &age); |
michael@0 | 280 | if (!found) { |
michael@0 | 281 | age = 24 * 60 * 60; // just a large number |
michael@0 | 282 | } else { |
michael@0 | 283 | int64_t now = (PR_Now() / PR_USEC_PER_SEC); |
michael@0 | 284 | age = now - age; |
michael@0 | 285 | } |
michael@0 | 286 | |
michael@0 | 287 | LOG(("Found a result in %s: %s (Age: %Lds)", |
michael@0 | 288 | cache->TableName().get(), |
michael@0 | 289 | complete ? "complete." : "Not complete.", |
michael@0 | 290 | age)); |
michael@0 | 291 | |
michael@0 | 292 | result->hash.complete = lookupHash; |
michael@0 | 293 | result->mComplete = complete; |
michael@0 | 294 | result->mFresh = (age < mFreshTime); |
michael@0 | 295 | result->mTableName.Assign(cache->TableName()); |
michael@0 | 296 | } |
michael@0 | 297 | } |
michael@0 | 298 | |
michael@0 | 299 | } |
michael@0 | 300 | |
michael@0 | 301 | return NS_OK; |
michael@0 | 302 | } |
michael@0 | 303 | |
michael@0 | 304 | nsresult |
michael@0 | 305 | Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates) |
michael@0 | 306 | { |
michael@0 | 307 | Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_UPDATE_TIME> timer; |
michael@0 | 308 | |
michael@0 | 309 | #if defined(PR_LOGGING) |
michael@0 | 310 | PRIntervalTime clockStart = 0; |
michael@0 | 311 | if (LOG_ENABLED() || true) { |
michael@0 | 312 | clockStart = PR_IntervalNow(); |
michael@0 | 313 | } |
michael@0 | 314 | #endif |
michael@0 | 315 | |
michael@0 | 316 | LOG(("Backup before update.")); |
michael@0 | 317 | |
michael@0 | 318 | nsresult rv = BackupTables(); |
michael@0 | 319 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 320 | |
michael@0 | 321 | LOG(("Applying table updates.")); |
michael@0 | 322 | |
michael@0 | 323 | for (uint32_t i = 0; i < aUpdates->Length(); i++) { |
michael@0 | 324 | // Previous ApplyTableUpdates() may have consumed this update.. |
michael@0 | 325 | if ((*aUpdates)[i]) { |
michael@0 | 326 | // Run all updates for one table |
michael@0 | 327 | nsCString updateTable(aUpdates->ElementAt(i)->TableName()); |
michael@0 | 328 | rv = ApplyTableUpdates(aUpdates, updateTable); |
michael@0 | 329 | if (NS_FAILED(rv)) { |
michael@0 | 330 | if (rv != NS_ERROR_OUT_OF_MEMORY) { |
michael@0 | 331 | Reset(); |
michael@0 | 332 | } |
michael@0 | 333 | return rv; |
michael@0 | 334 | } |
michael@0 | 335 | } |
michael@0 | 336 | } |
michael@0 | 337 | aUpdates->Clear(); |
michael@0 | 338 | |
michael@0 | 339 | rv = RegenActiveTables(); |
michael@0 | 340 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 341 | |
michael@0 | 342 | LOG(("Cleaning up backups.")); |
michael@0 | 343 | |
michael@0 | 344 | // Move the backup directory away (signaling the transaction finished |
michael@0 | 345 | // successfully). This is atomic. |
michael@0 | 346 | rv = RemoveBackupTables(); |
michael@0 | 347 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 348 | |
michael@0 | 349 | // Do the actual deletion of the backup files. |
michael@0 | 350 | rv = CleanToDelete(); |
michael@0 | 351 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 352 | |
michael@0 | 353 | LOG(("Done applying updates.")); |
michael@0 | 354 | |
michael@0 | 355 | #if defined(PR_LOGGING) |
michael@0 | 356 | if (LOG_ENABLED() || true) { |
michael@0 | 357 | PRIntervalTime clockEnd = PR_IntervalNow(); |
michael@0 | 358 | LOG(("update took %dms\n", |
michael@0 | 359 | PR_IntervalToMilliseconds(clockEnd - clockStart))); |
michael@0 | 360 | } |
michael@0 | 361 | #endif |
michael@0 | 362 | |
michael@0 | 363 | return NS_OK; |
michael@0 | 364 | } |
michael@0 | 365 | |
michael@0 | 366 | nsresult |
michael@0 | 367 | Classifier::MarkSpoiled(nsTArray<nsCString>& aTables) |
michael@0 | 368 | { |
michael@0 | 369 | for (uint32_t i = 0; i < aTables.Length(); i++) { |
michael@0 | 370 | LOG(("Spoiling table: %s", aTables[i].get())); |
michael@0 | 371 | // Spoil this table by marking it as no known freshness |
michael@0 | 372 | mTableFreshness.Remove(aTables[i]); |
michael@0 | 373 | // Remove any cached Completes for this table |
michael@0 | 374 | LookupCache *cache = GetLookupCache(aTables[i]); |
michael@0 | 375 | if (cache) { |
michael@0 | 376 | cache->ClearCompleteCache(); |
michael@0 | 377 | } |
michael@0 | 378 | } |
michael@0 | 379 | return NS_OK; |
michael@0 | 380 | } |
michael@0 | 381 | |
michael@0 | 382 | void |
michael@0 | 383 | Classifier::DropStores() |
michael@0 | 384 | { |
michael@0 | 385 | for (uint32_t i = 0; i < mHashStores.Length(); i++) { |
michael@0 | 386 | delete mHashStores[i]; |
michael@0 | 387 | } |
michael@0 | 388 | mHashStores.Clear(); |
michael@0 | 389 | for (uint32_t i = 0; i < mLookupCaches.Length(); i++) { |
michael@0 | 390 | delete mLookupCaches[i]; |
michael@0 | 391 | } |
michael@0 | 392 | mLookupCaches.Clear(); |
michael@0 | 393 | } |
michael@0 | 394 | |
michael@0 | 395 | nsresult |
michael@0 | 396 | Classifier::RegenActiveTables() |
michael@0 | 397 | { |
michael@0 | 398 | mActiveTablesCache.Clear(); |
michael@0 | 399 | |
michael@0 | 400 | nsTArray<nsCString> foundTables; |
michael@0 | 401 | ScanStoreDir(foundTables); |
michael@0 | 402 | |
michael@0 | 403 | for (uint32_t i = 0; i < foundTables.Length(); i++) { |
michael@0 | 404 | nsAutoPtr<HashStore> store(new HashStore(nsCString(foundTables[i]), mStoreDirectory)); |
michael@0 | 405 | if (!store) |
michael@0 | 406 | return NS_ERROR_OUT_OF_MEMORY; |
michael@0 | 407 | |
michael@0 | 408 | nsresult rv = store->Open(); |
michael@0 | 409 | if (NS_FAILED(rv)) |
michael@0 | 410 | continue; |
michael@0 | 411 | |
michael@0 | 412 | LookupCache *lookupCache = GetLookupCache(store->TableName()); |
michael@0 | 413 | if (!lookupCache) { |
michael@0 | 414 | continue; |
michael@0 | 415 | } |
michael@0 | 416 | |
michael@0 | 417 | if (!lookupCache->IsPrimed()) |
michael@0 | 418 | continue; |
michael@0 | 419 | |
michael@0 | 420 | const ChunkSet &adds = store->AddChunks(); |
michael@0 | 421 | const ChunkSet &subs = store->SubChunks(); |
michael@0 | 422 | |
michael@0 | 423 | if (adds.Length() == 0 && subs.Length() == 0) |
michael@0 | 424 | continue; |
michael@0 | 425 | |
michael@0 | 426 | LOG(("Active table: %s", store->TableName().get())); |
michael@0 | 427 | mActiveTablesCache.AppendElement(store->TableName()); |
michael@0 | 428 | } |
michael@0 | 429 | |
michael@0 | 430 | return NS_OK; |
michael@0 | 431 | } |
michael@0 | 432 | |
michael@0 | 433 | nsresult |
michael@0 | 434 | Classifier::ScanStoreDir(nsTArray<nsCString>& aTables) |
michael@0 | 435 | { |
michael@0 | 436 | nsCOMPtr<nsISimpleEnumerator> entries; |
michael@0 | 437 | nsresult rv = mStoreDirectory->GetDirectoryEntries(getter_AddRefs(entries)); |
michael@0 | 438 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 439 | |
michael@0 | 440 | bool hasMore; |
michael@0 | 441 | while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) { |
michael@0 | 442 | nsCOMPtr<nsISupports> supports; |
michael@0 | 443 | rv = entries->GetNext(getter_AddRefs(supports)); |
michael@0 | 444 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 445 | |
michael@0 | 446 | nsCOMPtr<nsIFile> file = do_QueryInterface(supports); |
michael@0 | 447 | |
michael@0 | 448 | nsCString leafName; |
michael@0 | 449 | rv = file->GetNativeLeafName(leafName); |
michael@0 | 450 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 451 | |
michael@0 | 452 | nsCString suffix(NS_LITERAL_CSTRING(".sbstore")); |
michael@0 | 453 | |
michael@0 | 454 | int32_t dot = leafName.RFind(suffix, 0); |
michael@0 | 455 | if (dot != -1) { |
michael@0 | 456 | leafName.Cut(dot, suffix.Length()); |
michael@0 | 457 | aTables.AppendElement(leafName); |
michael@0 | 458 | } |
michael@0 | 459 | } |
michael@0 | 460 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 461 | |
michael@0 | 462 | return NS_OK; |
michael@0 | 463 | } |
michael@0 | 464 | |
michael@0 | 465 | nsresult |
michael@0 | 466 | Classifier::ActiveTables(nsTArray<nsCString>& aTables) |
michael@0 | 467 | { |
michael@0 | 468 | aTables = mActiveTablesCache; |
michael@0 | 469 | return NS_OK; |
michael@0 | 470 | } |
michael@0 | 471 | |
michael@0 | 472 | nsresult |
michael@0 | 473 | Classifier::CleanToDelete() |
michael@0 | 474 | { |
michael@0 | 475 | bool exists; |
michael@0 | 476 | nsresult rv = mToDeleteDirectory->Exists(&exists); |
michael@0 | 477 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 478 | |
michael@0 | 479 | if (exists) { |
michael@0 | 480 | rv = mToDeleteDirectory->Remove(true); |
michael@0 | 481 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 482 | } |
michael@0 | 483 | |
michael@0 | 484 | return NS_OK; |
michael@0 | 485 | } |
michael@0 | 486 | |
michael@0 | 487 | nsresult |
michael@0 | 488 | Classifier::BackupTables() |
michael@0 | 489 | { |
michael@0 | 490 | // We have to work in reverse here: first move the normal directory |
michael@0 | 491 | // away to be the backup directory, then copy the files over |
michael@0 | 492 | // to the normal directory. This ensures that if we crash the backup |
michael@0 | 493 | // dir always has a valid, complete copy, instead of a partial one, |
michael@0 | 494 | // because that's the one we will copy over the normal store dir. |
michael@0 | 495 | |
michael@0 | 496 | nsCString backupDirName; |
michael@0 | 497 | nsresult rv = mBackupDirectory->GetNativeLeafName(backupDirName); |
michael@0 | 498 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 499 | |
michael@0 | 500 | nsCString storeDirName; |
michael@0 | 501 | rv = mStoreDirectory->GetNativeLeafName(storeDirName); |
michael@0 | 502 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 503 | |
michael@0 | 504 | rv = mStoreDirectory->MoveToNative(nullptr, backupDirName); |
michael@0 | 505 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 506 | |
michael@0 | 507 | rv = mStoreDirectory->CopyToNative(nullptr, storeDirName); |
michael@0 | 508 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 509 | |
michael@0 | 510 | // We moved some things to new places, so move the handles around, too. |
michael@0 | 511 | rv = SetupPathNames(); |
michael@0 | 512 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 513 | |
michael@0 | 514 | return NS_OK; |
michael@0 | 515 | } |
michael@0 | 516 | |
michael@0 | 517 | nsresult |
michael@0 | 518 | Classifier::RemoveBackupTables() |
michael@0 | 519 | { |
michael@0 | 520 | nsCString toDeleteName; |
michael@0 | 521 | nsresult rv = mToDeleteDirectory->GetNativeLeafName(toDeleteName); |
michael@0 | 522 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 523 | |
michael@0 | 524 | rv = mBackupDirectory->MoveToNative(nullptr, toDeleteName); |
michael@0 | 525 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 526 | |
michael@0 | 527 | // mBackupDirectory now points to toDelete, fix that up. |
michael@0 | 528 | rv = SetupPathNames(); |
michael@0 | 529 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 530 | |
michael@0 | 531 | return NS_OK; |
michael@0 | 532 | } |
michael@0 | 533 | |
michael@0 | 534 | nsresult |
michael@0 | 535 | Classifier::RecoverBackups() |
michael@0 | 536 | { |
michael@0 | 537 | bool backupExists; |
michael@0 | 538 | nsresult rv = mBackupDirectory->Exists(&backupExists); |
michael@0 | 539 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 540 | |
michael@0 | 541 | if (backupExists) { |
michael@0 | 542 | // Remove the safebrowsing dir if it exists |
michael@0 | 543 | nsCString storeDirName; |
michael@0 | 544 | rv = mStoreDirectory->GetNativeLeafName(storeDirName); |
michael@0 | 545 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 546 | |
michael@0 | 547 | bool storeExists; |
michael@0 | 548 | rv = mStoreDirectory->Exists(&storeExists); |
michael@0 | 549 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 550 | |
michael@0 | 551 | if (storeExists) { |
michael@0 | 552 | rv = mStoreDirectory->Remove(true); |
michael@0 | 553 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 554 | } |
michael@0 | 555 | |
michael@0 | 556 | // Move the backup to the store location |
michael@0 | 557 | rv = mBackupDirectory->MoveToNative(nullptr, storeDirName); |
michael@0 | 558 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 559 | |
michael@0 | 560 | // mBackupDirectory now points to storeDir, fix up. |
michael@0 | 561 | rv = SetupPathNames(); |
michael@0 | 562 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 563 | } |
michael@0 | 564 | |
michael@0 | 565 | return NS_OK; |
michael@0 | 566 | } |
michael@0 | 567 | |
michael@0 | 568 | /* |
michael@0 | 569 | * This will consume+delete updates from the passed nsTArray. |
michael@0 | 570 | */ |
michael@0 | 571 | nsresult |
michael@0 | 572 | Classifier::ApplyTableUpdates(nsTArray<TableUpdate*>* aUpdates, |
michael@0 | 573 | const nsACString& aTable) |
michael@0 | 574 | { |
michael@0 | 575 | LOG(("Classifier::ApplyTableUpdates(%s)", PromiseFlatCString(aTable).get())); |
michael@0 | 576 | |
michael@0 | 577 | nsAutoPtr<HashStore> store(new HashStore(aTable, mStoreDirectory)); |
michael@0 | 578 | |
michael@0 | 579 | if (!store) |
michael@0 | 580 | return NS_ERROR_FAILURE; |
michael@0 | 581 | |
michael@0 | 582 | // take the quick exit if there is no valid update for us |
michael@0 | 583 | // (common case) |
michael@0 | 584 | uint32_t validupdates = 0; |
michael@0 | 585 | |
michael@0 | 586 | for (uint32_t i = 0; i < aUpdates->Length(); i++) { |
michael@0 | 587 | TableUpdate *update = aUpdates->ElementAt(i); |
michael@0 | 588 | if (!update || !update->TableName().Equals(store->TableName())) |
michael@0 | 589 | continue; |
michael@0 | 590 | if (update->Empty()) { |
michael@0 | 591 | aUpdates->ElementAt(i) = nullptr; |
michael@0 | 592 | delete update; |
michael@0 | 593 | continue; |
michael@0 | 594 | } |
michael@0 | 595 | validupdates++; |
michael@0 | 596 | } |
michael@0 | 597 | |
michael@0 | 598 | if (!validupdates) { |
michael@0 | 599 | // This can happen if the update was only valid for one table. |
michael@0 | 600 | return NS_OK; |
michael@0 | 601 | } |
michael@0 | 602 | |
michael@0 | 603 | nsresult rv = store->Open(); |
michael@0 | 604 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 605 | rv = store->BeginUpdate(); |
michael@0 | 606 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 607 | |
michael@0 | 608 | // Read the part of the store that is (only) in the cache |
michael@0 | 609 | LookupCache *prefixSet = GetLookupCache(store->TableName()); |
michael@0 | 610 | if (!prefixSet) { |
michael@0 | 611 | return NS_ERROR_FAILURE; |
michael@0 | 612 | } |
michael@0 | 613 | nsTArray<uint32_t> AddPrefixHashes; |
michael@0 | 614 | rv = prefixSet->GetPrefixes(&AddPrefixHashes); |
michael@0 | 615 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 616 | rv = store->AugmentAdds(AddPrefixHashes); |
michael@0 | 617 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 618 | AddPrefixHashes.Clear(); |
michael@0 | 619 | |
michael@0 | 620 | uint32_t applied = 0; |
michael@0 | 621 | bool updateFreshness = false; |
michael@0 | 622 | bool hasCompletes = false; |
michael@0 | 623 | |
michael@0 | 624 | for (uint32_t i = 0; i < aUpdates->Length(); i++) { |
michael@0 | 625 | TableUpdate *update = aUpdates->ElementAt(i); |
michael@0 | 626 | if (!update || !update->TableName().Equals(store->TableName())) |
michael@0 | 627 | continue; |
michael@0 | 628 | |
michael@0 | 629 | rv = store->ApplyUpdate(*update); |
michael@0 | 630 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 631 | |
michael@0 | 632 | applied++; |
michael@0 | 633 | |
michael@0 | 634 | LOG(("Applied update to table %s:", store->TableName().get())); |
michael@0 | 635 | LOG((" %d add chunks", update->AddChunks().Length())); |
michael@0 | 636 | LOG((" %d add prefixes", update->AddPrefixes().Length())); |
michael@0 | 637 | LOG((" %d add completions", update->AddCompletes().Length())); |
michael@0 | 638 | LOG((" %d sub chunks", update->SubChunks().Length())); |
michael@0 | 639 | LOG((" %d sub prefixes", update->SubPrefixes().Length())); |
michael@0 | 640 | LOG((" %d sub completions", update->SubCompletes().Length())); |
michael@0 | 641 | LOG((" %d add expirations", update->AddExpirations().Length())); |
michael@0 | 642 | LOG((" %d sub expirations", update->SubExpirations().Length())); |
michael@0 | 643 | |
michael@0 | 644 | if (!update->IsLocalUpdate()) { |
michael@0 | 645 | updateFreshness = true; |
michael@0 | 646 | LOG(("Remote update, updating freshness")); |
michael@0 | 647 | } |
michael@0 | 648 | |
michael@0 | 649 | if (update->AddCompletes().Length() > 0 |
michael@0 | 650 | || update->SubCompletes().Length() > 0) { |
michael@0 | 651 | hasCompletes = true; |
michael@0 | 652 | LOG(("Contains Completes, keeping cache.")); |
michael@0 | 653 | } |
michael@0 | 654 | |
michael@0 | 655 | aUpdates->ElementAt(i) = nullptr; |
michael@0 | 656 | delete update; |
michael@0 | 657 | } |
michael@0 | 658 | |
michael@0 | 659 | LOG(("Applied %d update(s) to %s.", applied, store->TableName().get())); |
michael@0 | 660 | |
michael@0 | 661 | rv = store->Rebuild(); |
michael@0 | 662 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 663 | |
michael@0 | 664 | // Not an update with Completes, clear all completes data. |
michael@0 | 665 | if (!hasCompletes) { |
michael@0 | 666 | store->ClearCompletes(); |
michael@0 | 667 | } |
michael@0 | 668 | |
michael@0 | 669 | LOG(("Table %s now has:", store->TableName().get())); |
michael@0 | 670 | LOG((" %d add chunks", store->AddChunks().Length())); |
michael@0 | 671 | LOG((" %d add prefixes", store->AddPrefixes().Length())); |
michael@0 | 672 | LOG((" %d add completions", store->AddCompletes().Length())); |
michael@0 | 673 | LOG((" %d sub chunks", store->SubChunks().Length())); |
michael@0 | 674 | LOG((" %d sub prefixes", store->SubPrefixes().Length())); |
michael@0 | 675 | LOG((" %d sub completions", store->SubCompletes().Length())); |
michael@0 | 676 | |
michael@0 | 677 | rv = store->WriteFile(); |
michael@0 | 678 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 679 | |
michael@0 | 680 | // At this point the store is updated and written out to disk, but |
michael@0 | 681 | // the data is still in memory. Build our quick-lookup table here. |
michael@0 | 682 | rv = prefixSet->Build(store->AddPrefixes(), store->AddCompletes()); |
michael@0 | 683 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 684 | |
michael@0 | 685 | #if defined(DEBUG) && defined(PR_LOGGING) |
michael@0 | 686 | prefixSet->Dump(); |
michael@0 | 687 | #endif |
michael@0 | 688 | rv = prefixSet->WriteFile(); |
michael@0 | 689 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 690 | |
michael@0 | 691 | if (updateFreshness) { |
michael@0 | 692 | int64_t now = (PR_Now() / PR_USEC_PER_SEC); |
michael@0 | 693 | LOG(("Successfully updated %s", store->TableName().get())); |
michael@0 | 694 | mTableFreshness.Put(store->TableName(), now); |
michael@0 | 695 | } |
michael@0 | 696 | |
michael@0 | 697 | return NS_OK; |
michael@0 | 698 | } |
michael@0 | 699 | |
michael@0 | 700 | LookupCache * |
michael@0 | 701 | Classifier::GetLookupCache(const nsACString& aTable) |
michael@0 | 702 | { |
michael@0 | 703 | for (uint32_t i = 0; i < mLookupCaches.Length(); i++) { |
michael@0 | 704 | if (mLookupCaches[i]->TableName().Equals(aTable)) { |
michael@0 | 705 | return mLookupCaches[i]; |
michael@0 | 706 | } |
michael@0 | 707 | } |
michael@0 | 708 | |
michael@0 | 709 | LookupCache *cache = new LookupCache(aTable, mStoreDirectory); |
michael@0 | 710 | nsresult rv = cache->Init(); |
michael@0 | 711 | if (NS_FAILED(rv)) { |
michael@0 | 712 | return nullptr; |
michael@0 | 713 | } |
michael@0 | 714 | rv = cache->Open(); |
michael@0 | 715 | if (NS_FAILED(rv)) { |
michael@0 | 716 | if (rv == NS_ERROR_FILE_CORRUPTED) { |
michael@0 | 717 | Reset(); |
michael@0 | 718 | } |
michael@0 | 719 | return nullptr; |
michael@0 | 720 | } |
michael@0 | 721 | mLookupCaches.AppendElement(cache); |
michael@0 | 722 | return cache; |
michael@0 | 723 | } |
michael@0 | 724 | |
michael@0 | 725 | nsresult |
michael@0 | 726 | Classifier::ReadNoiseEntries(const Prefix& aPrefix, |
michael@0 | 727 | const nsACString& aTableName, |
michael@0 | 728 | uint32_t aCount, |
michael@0 | 729 | PrefixArray* aNoiseEntries) |
michael@0 | 730 | { |
michael@0 | 731 | LookupCache *cache = GetLookupCache(aTableName); |
michael@0 | 732 | if (!cache) { |
michael@0 | 733 | return NS_ERROR_FAILURE; |
michael@0 | 734 | } |
michael@0 | 735 | |
michael@0 | 736 | nsTArray<uint32_t> prefixes; |
michael@0 | 737 | nsresult rv = cache->GetPrefixes(&prefixes); |
michael@0 | 738 | NS_ENSURE_SUCCESS(rv, rv); |
michael@0 | 739 | |
michael@0 | 740 | uint32_t idx = prefixes.BinaryIndexOf(aPrefix.ToUint32()); |
michael@0 | 741 | |
michael@0 | 742 | if (idx == nsTArray<uint32_t>::NoIndex) { |
michael@0 | 743 | NS_WARNING("Could not find prefix in PrefixSet during noise lookup"); |
michael@0 | 744 | return NS_ERROR_FAILURE; |
michael@0 | 745 | } |
michael@0 | 746 | |
michael@0 | 747 | idx -= idx % aCount; |
michael@0 | 748 | |
michael@0 | 749 | for (uint32_t i = 0; (i < aCount) && ((idx+i) < prefixes.Length()); i++) { |
michael@0 | 750 | Prefix newPref; |
michael@0 | 751 | newPref.FromUint32(prefixes[idx+i]); |
michael@0 | 752 | if (newPref != aPrefix) { |
michael@0 | 753 | aNoiseEntries->AppendElement(newPref); |
michael@0 | 754 | } |
michael@0 | 755 | } |
michael@0 | 756 | |
michael@0 | 757 | return NS_OK; |
michael@0 | 758 | } |
michael@0 | 759 | |
michael@0 | 760 | } // namespace safebrowsing |
michael@0 | 761 | } // namespace mozilla |