michael@0: //* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "Classifier.h"
michael@0: #include "nsIPrefBranch.h"
michael@0: #include "nsIPrefService.h"
michael@0: #include "nsISimpleEnumerator.h"
michael@0: #include "nsIRandomGenerator.h"
michael@0: #include "nsIInputStream.h"
michael@0: #include "nsISeekableStream.h"
michael@0: #include "nsIFile.h"
michael@0: #include "nsAutoPtr.h"
michael@0: #include "mozilla/Telemetry.h"
michael@0: #include "prlog.h"
michael@0: 
michael@0: // NSPR_LOG_MODULES=UrlClassifierDbService:5
michael@0: extern PRLogModuleInfo *gUrlClassifierDbServiceLog;
michael@0: #if defined(PR_LOGGING)
michael@0: #define LOG(args) PR_LOG(gUrlClassifierDbServiceLog, PR_LOG_DEBUG, args)
michael@0: #define LOG_ENABLED() PR_LOG_TEST(gUrlClassifierDbServiceLog, 4)
michael@0: #else
michael@0: #define LOG(args)
michael@0: #define LOG_ENABLED() (false)
michael@0: #endif
michael@0: 
michael@0: #define STORE_DIRECTORY      NS_LITERAL_CSTRING("safebrowsing")
michael@0: #define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete")
michael@0: #define BACKUP_DIR_SUFFIX    NS_LITERAL_CSTRING("-backup")
michael@0: 
michael@0: namespace mozilla {
michael@0: namespace safebrowsing {
michael@0: 
michael@0: void
michael@0: Classifier::SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
michael@0: {
michael@0:   tables.Clear();
michael@0: 
michael@0:   nsACString::const_iterator begin, iter, end;
michael@0:   str.BeginReading(begin);
michael@0:   str.EndReading(end);
michael@0:   while (begin != end) {
michael@0:     iter = begin;
michael@0:     FindCharInReadable(',', iter, end);
michael@0:     nsDependentCSubstring table = Substring(begin,iter);
michael@0:     if (!table.IsEmpty()) {
michael@0:       tables.AppendElement(Substring(begin, iter));
michael@0:     }
michael@0:     begin = iter;
michael@0:     if (begin != end) {
michael@0:       begin++;
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: Classifier::Classifier()
michael@0:   : mFreshTime(45 * 60)
michael@0: {
michael@0: }
michael@0: 
michael@0: Classifier::~Classifier()
michael@0: {
michael@0:   Close();
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::SetupPathNames()
michael@0: {
michael@0:   // Get the root directory where to store all the databases.
michael@0:   nsresult rv = mCacheDirectory->Clone(getter_AddRefs(mStoreDirectory));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mStoreDirectory->AppendNative(STORE_DIRECTORY);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Make sure LookupCaches (which are persistent and survive updates)
michael@0:   // are reading/writing in the right place. We will be moving their
michael@0:   // files "underneath" them during backup/restore.
michael@0:   for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
michael@0:     mLookupCaches[i]->UpdateDirHandle(mStoreDirectory);
michael@0:   }
michael@0: 
michael@0:   // Directory where to move a backup before an update.
michael@0:   rv = mCacheDirectory->Clone(getter_AddRefs(mBackupDirectory));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Directory where to move the backup so we can atomically
michael@0:   // delete (really move) it.
michael@0:   rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mToDeleteDirectory->AppendNative(STORE_DIRECTORY + TO_DELETE_DIR_SUFFIX);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::CreateStoreDirectory()
michael@0: {
michael@0:   // Ensure the safebrowsing directory exists.
michael@0:   bool storeExists;
michael@0:   nsresult rv = mStoreDirectory->Exists(&storeExists);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   if (!storeExists) {
michael@0:     rv = mStoreDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   } else {
michael@0:     bool storeIsDir;
michael@0:     rv = mStoreDirectory->IsDirectory(&storeIsDir);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:     if (!storeIsDir)
michael@0:       return NS_ERROR_FILE_DESTINATION_NOT_DIR;
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::Open(nsIFile& aCacheDirectory)
michael@0: {
michael@0:   // Remember the Local profile directory.
michael@0:   nsresult rv = aCacheDirectory.Clone(getter_AddRefs(mCacheDirectory));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Create the handles to the update and backup directories.
michael@0:   rv = SetupPathNames();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Clean up any to-delete directories that haven't been deleted yet.
michael@0:   rv = CleanToDelete();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Check whether we have an incomplete update and recover from the
michael@0:   // backup if so.
michael@0:   rv = RecoverBackups();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Make sure the main store directory exists.
michael@0:   rv = CreateStoreDirectory();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Build the list of know urlclassifier lists
michael@0:   // XXX: Disk IO potentially on the main thread during startup
michael@0:   RegenActiveTables();
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: void
michael@0: Classifier::Close()
michael@0: {
michael@0:   DropStores();
michael@0: }
michael@0: 
michael@0: void
michael@0: Classifier::Reset()
michael@0: {
michael@0:   DropStores();
michael@0: 
michael@0:   mStoreDirectory->Remove(true);
michael@0:   mBackupDirectory->Remove(true);
michael@0:   mToDeleteDirectory->Remove(true);
michael@0: 
michael@0:   CreateStoreDirectory();
michael@0: 
michael@0:   mTableFreshness.Clear();
michael@0:   RegenActiveTables();
michael@0: }
michael@0: 
michael@0: void
michael@0: Classifier::TableRequest(nsACString& aResult)
michael@0: {
michael@0:   nsTArray<nsCString> tables;
michael@0:   ActiveTables(tables);
michael@0:   for (uint32_t i = 0; i < tables.Length(); i++) {
michael@0:     nsAutoPtr<HashStore> store(new HashStore(tables[i], mStoreDirectory));
michael@0:     if (!store)
michael@0:       continue;
michael@0: 
michael@0:     nsresult rv = store->Open();
michael@0:     if (NS_FAILED(rv))
michael@0:       continue;
michael@0: 
michael@0:     aResult.Append(store->TableName());
michael@0:     aResult.Append(";");
michael@0: 
michael@0:     ChunkSet &adds = store->AddChunks();
michael@0:     ChunkSet &subs = store->SubChunks();
michael@0: 
michael@0:     if (adds.Length() > 0) {
michael@0:       aResult.Append("a:");
michael@0:       nsAutoCString addList;
michael@0:       adds.Serialize(addList);
michael@0:       aResult.Append(addList);
michael@0:     }
michael@0: 
michael@0:     if (subs.Length() > 0) {
michael@0:       if (adds.Length() > 0)
michael@0:         aResult.Append(':');
michael@0:       aResult.Append("s:");
michael@0:       nsAutoCString subList;
michael@0:       subs.Serialize(subList);
michael@0:       aResult.Append(subList);
michael@0:     }
michael@0: 
michael@0:     aResult.Append('\n');
michael@0:   }
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::Check(const nsACString& aSpec,
michael@0:                   const nsACString& aTables,
michael@0:                   LookupResultArray& aResults)
michael@0: {
michael@0:   Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_CHECK_TIME> timer;
michael@0: 
michael@0:   // Get the set of fragments based on the url. This is necessary because we
michael@0:   // only look up at most 5 URLs per aSpec, even if aSpec has more than 5
michael@0:   // components.
michael@0:   nsTArray<nsCString> fragments;
michael@0:   nsresult rv = LookupCache::GetLookupFragments(aSpec, &fragments);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   nsTArray<nsCString> activeTables;
michael@0:   SplitTables(aTables, activeTables);
michael@0: 
michael@0:   nsTArray<LookupCache*> cacheArray;
michael@0:   for (uint32_t i = 0; i < activeTables.Length(); i++) {
michael@0:     LOG(("Checking table %s", activeTables[i].get()));
michael@0:     LookupCache *cache = GetLookupCache(activeTables[i]);
michael@0:     if (cache) {
michael@0:       cacheArray.AppendElement(cache);
michael@0:     } else {
michael@0:       return NS_ERROR_FAILURE;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Now check each lookup fragment against the entries in the DB.
michael@0:   for (uint32_t i = 0; i < fragments.Length(); i++) {
michael@0:     Completion lookupHash;
michael@0:     lookupHash.FromPlaintext(fragments[i], mCryptoHash);
michael@0: 
michael@0:     // Get list of host keys to look up
michael@0:     Completion hostKey;
michael@0:     rv = LookupCache::GetKey(fragments[i], &hostKey, mCryptoHash);
michael@0:     if (NS_FAILED(rv)) {
michael@0:       // Local host on the network.
michael@0:       continue;
michael@0:     }
michael@0: 
michael@0: #if DEBUG && defined(PR_LOGGING)
michael@0:     if (LOG_ENABLED()) {
michael@0:       nsAutoCString checking;
michael@0:       lookupHash.ToHexString(checking);
michael@0:       LOG(("Checking fragment %s, hash %s (%X)", fragments[i].get(),
michael@0:            checking.get(), lookupHash.ToUint32()));
michael@0:     }
michael@0: #endif
michael@0:     for (uint32_t i = 0; i < cacheArray.Length(); i++) {
michael@0:       LookupCache *cache = cacheArray[i];
michael@0:       bool has, complete;
michael@0:       rv = cache->Has(lookupHash, &has, &complete);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0:       if (has) {
michael@0:         LookupResult *result = aResults.AppendElement();
michael@0:         if (!result)
michael@0:           return NS_ERROR_OUT_OF_MEMORY;
michael@0: 
michael@0:         int64_t age;
michael@0:         bool found = mTableFreshness.Get(cache->TableName(), &age);
michael@0:         if (!found) {
michael@0:           age = 24 * 60 * 60; // just a large number
michael@0:         } else {
michael@0:           int64_t now = (PR_Now() / PR_USEC_PER_SEC);
michael@0:           age = now - age;
michael@0:         }
michael@0: 
michael@0:         LOG(("Found a result in %s: %s (Age: %Lds)",
michael@0:              cache->TableName().get(),
michael@0:              complete ? "complete." : "Not complete.",
michael@0:              age));
michael@0: 
michael@0:         result->hash.complete = lookupHash;
michael@0:         result->mComplete = complete;
michael@0:         result->mFresh = (age < mFreshTime);
michael@0:         result->mTableName.Assign(cache->TableName());
michael@0:       }
michael@0:     }
michael@0: 
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
michael@0: {
michael@0:   Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_CL_UPDATE_TIME> timer;
michael@0: 
michael@0: #if defined(PR_LOGGING)
michael@0:   PRIntervalTime clockStart = 0;
michael@0:   if (LOG_ENABLED() || true) {
michael@0:     clockStart = PR_IntervalNow();
michael@0:   }
michael@0: #endif
michael@0: 
michael@0:   LOG(("Backup before update."));
michael@0: 
michael@0:   nsresult rv = BackupTables();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   LOG(("Applying table updates."));
michael@0: 
michael@0:   for (uint32_t i = 0; i < aUpdates->Length(); i++) {
michael@0:     // Previous ApplyTableUpdates() may have consumed this update..
michael@0:     if ((*aUpdates)[i]) {
michael@0:       // Run all updates for one table
michael@0:       nsCString updateTable(aUpdates->ElementAt(i)->TableName());
michael@0:       rv = ApplyTableUpdates(aUpdates, updateTable);
michael@0:       if (NS_FAILED(rv)) {
michael@0:         if (rv != NS_ERROR_OUT_OF_MEMORY) {
michael@0:           Reset();
michael@0:         }
michael@0:         return rv;
michael@0:       }
michael@0:     }
michael@0:   }
michael@0:   aUpdates->Clear();
michael@0: 
michael@0:   rv = RegenActiveTables();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   LOG(("Cleaning up backups."));
michael@0: 
michael@0:   // Move the backup directory away (signaling the transaction finished
michael@0:   // successfully). This is atomic.
michael@0:   rv = RemoveBackupTables();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Do the actual deletion of the backup files.
michael@0:   rv = CleanToDelete();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   LOG(("Done applying updates."));
michael@0: 
michael@0: #if defined(PR_LOGGING)
michael@0:   if (LOG_ENABLED() || true) {
michael@0:     PRIntervalTime clockEnd = PR_IntervalNow();
michael@0:     LOG(("update took %dms\n",
michael@0:          PR_IntervalToMilliseconds(clockEnd - clockStart)));
michael@0:   }
michael@0: #endif
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::MarkSpoiled(nsTArray<nsCString>& aTables)
michael@0: {
michael@0:   for (uint32_t i = 0; i < aTables.Length(); i++) {
michael@0:     LOG(("Spoiling table: %s", aTables[i].get()));
michael@0:     // Spoil this table by marking it as no known freshness
michael@0:     mTableFreshness.Remove(aTables[i]);
michael@0:     // Remove any cached Completes for this table
michael@0:     LookupCache *cache = GetLookupCache(aTables[i]);
michael@0:     if (cache) {
michael@0:       cache->ClearCompleteCache();
michael@0:     }
michael@0:   }
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: void
michael@0: Classifier::DropStores()
michael@0: {
michael@0:   for (uint32_t i = 0; i < mHashStores.Length(); i++) {
michael@0:     delete mHashStores[i];
michael@0:   }
michael@0:   mHashStores.Clear();
michael@0:   for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
michael@0:     delete mLookupCaches[i];
michael@0:   }
michael@0:   mLookupCaches.Clear();
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::RegenActiveTables()
michael@0: {
michael@0:   mActiveTablesCache.Clear();
michael@0: 
michael@0:   nsTArray<nsCString> foundTables;
michael@0:   ScanStoreDir(foundTables);
michael@0: 
michael@0:   for (uint32_t i = 0; i < foundTables.Length(); i++) {
michael@0:     nsAutoPtr<HashStore> store(new HashStore(nsCString(foundTables[i]), mStoreDirectory));
michael@0:     if (!store)
michael@0:       return NS_ERROR_OUT_OF_MEMORY;
michael@0: 
michael@0:     nsresult rv = store->Open();
michael@0:     if (NS_FAILED(rv))
michael@0:       continue;
michael@0: 
michael@0:     LookupCache *lookupCache = GetLookupCache(store->TableName());
michael@0:     if (!lookupCache) {
michael@0:       continue;
michael@0:     }
michael@0: 
michael@0:     if (!lookupCache->IsPrimed())
michael@0:       continue;
michael@0: 
michael@0:     const ChunkSet &adds = store->AddChunks();
michael@0:     const ChunkSet &subs = store->SubChunks();
michael@0: 
michael@0:     if (adds.Length() == 0 && subs.Length() == 0)
michael@0:       continue;
michael@0: 
michael@0:     LOG(("Active table: %s", store->TableName().get()));
michael@0:     mActiveTablesCache.AppendElement(store->TableName());
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::ScanStoreDir(nsTArray<nsCString>& aTables)
michael@0: {
michael@0:   nsCOMPtr<nsISimpleEnumerator> entries;
michael@0:   nsresult rv = mStoreDirectory->GetDirectoryEntries(getter_AddRefs(entries));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   bool hasMore;
michael@0:   while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
michael@0:     nsCOMPtr<nsISupports> supports;
michael@0:     rv = entries->GetNext(getter_AddRefs(supports));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     nsCOMPtr<nsIFile> file = do_QueryInterface(supports);
michael@0: 
michael@0:     nsCString leafName;
michael@0:     rv = file->GetNativeLeafName(leafName);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     nsCString suffix(NS_LITERAL_CSTRING(".sbstore"));
michael@0: 
michael@0:     int32_t dot = leafName.RFind(suffix, 0);
michael@0:     if (dot != -1) {
michael@0:       leafName.Cut(dot, suffix.Length());
michael@0:       aTables.AppendElement(leafName);
michael@0:     }
michael@0:   }
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::ActiveTables(nsTArray<nsCString>& aTables)
michael@0: {
michael@0:   aTables = mActiveTablesCache;
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::CleanToDelete()
michael@0: {
michael@0:   bool exists;
michael@0:   nsresult rv = mToDeleteDirectory->Exists(&exists);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   if (exists) {
michael@0:     rv = mToDeleteDirectory->Remove(true);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::BackupTables()
michael@0: {
michael@0:   // We have to work in reverse here: first move the normal directory
michael@0:   // away to be the backup directory, then copy the files over
michael@0:   // to the normal directory. This ensures that if we crash the backup
michael@0:   // dir always has a valid, complete copy, instead of a partial one,
michael@0:   // because that's the one we will copy over the normal store dir.
michael@0: 
michael@0:   nsCString backupDirName;
michael@0:   nsresult rv = mBackupDirectory->GetNativeLeafName(backupDirName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   nsCString storeDirName;
michael@0:   rv = mStoreDirectory->GetNativeLeafName(storeDirName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mStoreDirectory->MoveToNative(nullptr, backupDirName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mStoreDirectory->CopyToNative(nullptr, storeDirName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // We moved some things to new places, so move the handles around, too.
michael@0:   rv = SetupPathNames();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::RemoveBackupTables()
michael@0: {
michael@0:   nsCString toDeleteName;
michael@0:   nsresult rv = mToDeleteDirectory->GetNativeLeafName(toDeleteName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mBackupDirectory->MoveToNative(nullptr, toDeleteName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // mBackupDirectory now points to toDelete, fix that up.
michael@0:   rv = SetupPathNames();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::RecoverBackups()
michael@0: {
michael@0:   bool backupExists;
michael@0:   nsresult rv = mBackupDirectory->Exists(&backupExists);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   if (backupExists) {
michael@0:     // Remove the safebrowsing dir if it exists
michael@0:     nsCString storeDirName;
michael@0:     rv = mStoreDirectory->GetNativeLeafName(storeDirName);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     bool storeExists;
michael@0:     rv = mStoreDirectory->Exists(&storeExists);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     if (storeExists) {
michael@0:       rv = mStoreDirectory->Remove(true);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0:     }
michael@0: 
michael@0:     // Move the backup to the store location
michael@0:     rv = mBackupDirectory->MoveToNative(nullptr, storeDirName);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     // mBackupDirectory now points to storeDir, fix up.
michael@0:     rv = SetupPathNames();
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: /*
michael@0:  * This will consume+delete updates from the passed nsTArray.
michael@0: */
michael@0: nsresult
michael@0: Classifier::ApplyTableUpdates(nsTArray<TableUpdate*>* aUpdates,
michael@0:                               const nsACString& aTable)
michael@0: {
michael@0:   LOG(("Classifier::ApplyTableUpdates(%s)", PromiseFlatCString(aTable).get()));
michael@0: 
michael@0:   nsAutoPtr<HashStore> store(new HashStore(aTable, mStoreDirectory));
michael@0: 
michael@0:   if (!store)
michael@0:     return NS_ERROR_FAILURE;
michael@0: 
michael@0:   // take the quick exit if there is no valid update for us
michael@0:   // (common case)
michael@0:   uint32_t validupdates = 0;
michael@0: 
michael@0:   for (uint32_t i = 0; i < aUpdates->Length(); i++) {
michael@0:     TableUpdate *update = aUpdates->ElementAt(i);
michael@0:     if (!update || !update->TableName().Equals(store->TableName()))
michael@0:       continue;
michael@0:     if (update->Empty()) {
michael@0:       aUpdates->ElementAt(i) = nullptr;
michael@0:       delete update;
michael@0:       continue;
michael@0:     }
michael@0:     validupdates++;
michael@0:   }
michael@0: 
michael@0:   if (!validupdates) {
michael@0:     // This can happen if the update was only valid for one table.
michael@0:     return NS_OK;
michael@0:   }
michael@0: 
michael@0:   nsresult rv = store->Open();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   rv = store->BeginUpdate();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Read the part of the store that is (only) in the cache
michael@0:   LookupCache *prefixSet = GetLookupCache(store->TableName());
michael@0:   if (!prefixSet) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0:   nsTArray<uint32_t> AddPrefixHashes;
michael@0:   rv = prefixSet->GetPrefixes(&AddPrefixHashes);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   rv = store->AugmentAdds(AddPrefixHashes);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   AddPrefixHashes.Clear();
michael@0: 
michael@0:   uint32_t applied = 0;
michael@0:   bool updateFreshness = false;
michael@0:   bool hasCompletes = false;
michael@0: 
michael@0:   for (uint32_t i = 0; i < aUpdates->Length(); i++) {
michael@0:     TableUpdate *update = aUpdates->ElementAt(i);
michael@0:     if (!update || !update->TableName().Equals(store->TableName()))
michael@0:       continue;
michael@0: 
michael@0:     rv = store->ApplyUpdate(*update);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     applied++;
michael@0: 
michael@0:     LOG(("Applied update to table %s:", store->TableName().get()));
michael@0:     LOG(("  %d add chunks", update->AddChunks().Length()));
michael@0:     LOG(("  %d add prefixes", update->AddPrefixes().Length()));
michael@0:     LOG(("  %d add completions", update->AddCompletes().Length()));
michael@0:     LOG(("  %d sub chunks", update->SubChunks().Length()));
michael@0:     LOG(("  %d sub prefixes", update->SubPrefixes().Length()));
michael@0:     LOG(("  %d sub completions", update->SubCompletes().Length()));
michael@0:     LOG(("  %d add expirations", update->AddExpirations().Length()));
michael@0:     LOG(("  %d sub expirations", update->SubExpirations().Length()));
michael@0: 
michael@0:     if (!update->IsLocalUpdate()) {
michael@0:       updateFreshness = true;
michael@0:       LOG(("Remote update, updating freshness"));
michael@0:     }
michael@0: 
michael@0:     if (update->AddCompletes().Length() > 0
michael@0:         || update->SubCompletes().Length() > 0) {
michael@0:       hasCompletes = true;
michael@0:       LOG(("Contains Completes, keeping cache."));
michael@0:     }
michael@0: 
michael@0:     aUpdates->ElementAt(i) = nullptr;
michael@0:     delete update;
michael@0:   }
michael@0: 
michael@0:   LOG(("Applied %d update(s) to %s.", applied, store->TableName().get()));
michael@0: 
michael@0:   rv = store->Rebuild();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Not an update with Completes, clear all completes data.
michael@0:   if (!hasCompletes) {
michael@0:     store->ClearCompletes();
michael@0:   }
michael@0: 
michael@0:   LOG(("Table %s now has:", store->TableName().get()));
michael@0:   LOG(("  %d add chunks", store->AddChunks().Length()));
michael@0:   LOG(("  %d add prefixes", store->AddPrefixes().Length()));
michael@0:   LOG(("  %d add completions", store->AddCompletes().Length()));
michael@0:   LOG(("  %d sub chunks", store->SubChunks().Length()));
michael@0:   LOG(("  %d sub prefixes", store->SubPrefixes().Length()));
michael@0:   LOG(("  %d sub completions", store->SubCompletes().Length()));
michael@0: 
michael@0:   rv = store->WriteFile();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // At this point the store is updated and written out to disk, but
michael@0:   // the data is still in memory.  Build our quick-lookup table here.
michael@0:   rv = prefixSet->Build(store->AddPrefixes(), store->AddCompletes());
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0: #if defined(DEBUG) && defined(PR_LOGGING)
michael@0:   prefixSet->Dump();
michael@0: #endif
michael@0:   rv = prefixSet->WriteFile();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   if (updateFreshness) {
michael@0:     int64_t now = (PR_Now() / PR_USEC_PER_SEC);
michael@0:     LOG(("Successfully updated %s", store->TableName().get()));
michael@0:     mTableFreshness.Put(store->TableName(), now);
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: LookupCache *
michael@0: Classifier::GetLookupCache(const nsACString& aTable)
michael@0: {
michael@0:   for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
michael@0:     if (mLookupCaches[i]->TableName().Equals(aTable)) {
michael@0:       return mLookupCaches[i];
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   LookupCache *cache = new LookupCache(aTable, mStoreDirectory);
michael@0:   nsresult rv = cache->Init();
michael@0:   if (NS_FAILED(rv)) {
michael@0:     return nullptr;
michael@0:   }
michael@0:   rv = cache->Open();
michael@0:   if (NS_FAILED(rv)) {
michael@0:     if (rv == NS_ERROR_FILE_CORRUPTED) {
michael@0:       Reset();
michael@0:     }
michael@0:     return nullptr;
michael@0:   }
michael@0:   mLookupCaches.AppendElement(cache);
michael@0:   return cache;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: Classifier::ReadNoiseEntries(const Prefix& aPrefix,
michael@0:                              const nsACString& aTableName,
michael@0:                              uint32_t aCount,
michael@0:                              PrefixArray* aNoiseEntries)
michael@0: {
michael@0:   LookupCache *cache = GetLookupCache(aTableName);
michael@0:   if (!cache) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   nsTArray<uint32_t> prefixes;
michael@0:   nsresult rv = cache->GetPrefixes(&prefixes);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   uint32_t idx = prefixes.BinaryIndexOf(aPrefix.ToUint32());
michael@0: 
michael@0:   if (idx == nsTArray<uint32_t>::NoIndex) {
michael@0:     NS_WARNING("Could not find prefix in PrefixSet during noise lookup");
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   idx -= idx % aCount;
michael@0: 
michael@0:   for (uint32_t i = 0; (i < aCount) && ((idx+i) < prefixes.Length()); i++) {
michael@0:     Prefix newPref;
michael@0:     newPref.FromUint32(prefixes[idx+i]);
michael@0:     if (newPref != aPrefix) {
michael@0:       aNoiseEntries->AppendElement(newPref);
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: } // namespace safebrowsing
michael@0: } // namespace mozilla