michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "DOMStorageDBThread.h"
michael@0: #include "DOMStorageCache.h"
michael@0: 
michael@0: #include "nsIEffectiveTLDService.h"
michael@0: #include "nsDirectoryServiceUtils.h"
michael@0: #include "nsAppDirectoryServiceDefs.h"
michael@0: #include "nsThreadUtils.h"
michael@0: #include "nsProxyRelease.h"
michael@0: #include "mozStorageCID.h"
michael@0: #include "mozStorageHelper.h"
michael@0: #include "mozIStorageService.h"
michael@0: #include "mozIStorageBindingParamsArray.h"
michael@0: #include "mozIStorageBindingParams.h"
michael@0: #include "mozIStorageValueArray.h"
michael@0: #include "mozIStorageFunction.h"
michael@0: #include "nsIObserverService.h"
michael@0: #include "nsIVariant.h"
michael@0: #include "mozilla/IOInterposer.h"
michael@0: #include "mozilla/Services.h"
michael@0: 
michael@0: // How long we collect write oprerations
michael@0: // before they are flushed to the database
michael@0: // In milliseconds.
michael@0: #define FLUSHING_INTERVAL_MS 5000
michael@0: 
michael@0: // Write Ahead Log's maximum size is 512KB
michael@0: #define MAX_WAL_SIZE_BYTES 512 * 1024
michael@0: 
michael@0: namespace mozilla {
michael@0: namespace dom {
michael@0: 
michael@0: DOMStorageDBBridge::DOMStorageDBBridge()
michael@0: {
michael@0: }
michael@0: 
michael@0: 
michael@0: DOMStorageDBThread::DOMStorageDBThread()
michael@0: : mThread(nullptr)
michael@0: , mMonitor("DOMStorageThreadMonitor")
michael@0: , mStopIOThread(false)
michael@0: , mWALModeEnabled(false)
michael@0: , mDBReady(false)
michael@0: , mStatus(NS_OK)
michael@0: , mWorkerStatements(mWorkerConnection)
michael@0: , mReaderStatements(mReaderConnection)
michael@0: , mDirtyEpoch(0)
michael@0: , mFlushImmediately(false)
michael@0: , mPriorityCounter(0)
michael@0: {
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::Init()
michael@0: {
michael@0:   nsresult rv;
michael@0: 
michael@0:   // Need to determine location on the main thread, since
michael@0:   // NS_GetSpecialDirectory access the atom table that can
michael@0:   // be accessed only on the main thread.
michael@0:   rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
michael@0:                               getter_AddRefs(mDatabaseFile));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mDatabaseFile->Append(NS_LITERAL_STRING("webappsstore.sqlite"));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Ensure mozIStorageService init on the main thread first.
michael@0:   nsCOMPtr<mozIStorageService> service =
michael@0:     do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Need to keep the lock to avoid setting mThread later then
michael@0:   // the thread body executes.
michael@0:   MonitorAutoLock monitor(mMonitor);
michael@0: 
michael@0:   mThread = PR_CreateThread(PR_USER_THREAD, &DOMStorageDBThread::ThreadFunc, this,
michael@0:                             PR_PRIORITY_LOW, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD,
michael@0:                             262144);
michael@0:   if (!mThread) {
michael@0:     return NS_ERROR_OUT_OF_MEMORY;
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::Shutdown()
michael@0: {
michael@0:   if (!mThread) {
michael@0:     return NS_ERROR_NOT_INITIALIZED;
michael@0:   }
michael@0: 
michael@0:   Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_SHUTDOWN_DATABASE_MS> timer;
michael@0: 
michael@0:   {
michael@0:     MonitorAutoLock monitor(mMonitor);
michael@0: 
michael@0:     // After we stop, no other operations can be accepted
michael@0:     mFlushImmediately = true;
michael@0:     mStopIOThread = true;
michael@0:     monitor.Notify();
michael@0:   }
michael@0: 
michael@0:   PR_JoinThread(mThread);
michael@0:   mThread = nullptr;
michael@0: 
michael@0:   return mStatus;
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::SyncPreload(DOMStorageCacheBridge* aCache, bool aForceSync)
michael@0: {
michael@0:   if (!aForceSync && aCache->LoadedCount()) {
michael@0:     // Preload already started for this cache, just wait for it to finish.
michael@0:     // LoadWait will exit after LoadDone on the cache has been called.
michael@0:     SetHigherPriority();
michael@0:     aCache->LoadWait();
michael@0:     SetDefaultPriority();
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   // Bypass sync load when an update is pending in the queue to write, we would
michael@0:   // get incosistent data in the cache.  Also don't allow sync main-thread preload
michael@0:   // when DB open and init is still pending on the background thread.
michael@0:   if (mDBReady && mWALModeEnabled) {
michael@0:     bool pendingTasks;
michael@0:     {
michael@0:       MonitorAutoLock monitor(mMonitor);
michael@0:       pendingTasks = mPendingTasks.IsScopeUpdatePending(aCache->Scope()) ||
michael@0:                      mPendingTasks.IsScopeClearPending(aCache->Scope());
michael@0:     }
michael@0: 
michael@0:     if (!pendingTasks) {
michael@0:       // WAL is enabled, thus do the load synchronously on the main thread.
michael@0:       DBOperation preload(DBOperation::opPreload, aCache);
michael@0:       preload.PerformAndFinalize(this);
michael@0:       return;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Need to go asynchronously since WAL is not allowed or scheduled updates
michael@0:   // need to be flushed first.
michael@0:   // Schedule preload for this cache as the first operation.
michael@0:   nsresult rv = InsertDBOp(new DBOperation(DBOperation::opPreloadUrgent, aCache));
michael@0: 
michael@0:   // LoadWait exits after LoadDone of the cache has been called.
michael@0:   if (NS_SUCCEEDED(rv)) {
michael@0:     aCache->LoadWait();
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::AsyncFlush()
michael@0: {
michael@0:   MonitorAutoLock monitor(mMonitor);
michael@0:   mFlushImmediately = true;
michael@0:   monitor.Notify();
michael@0: }
michael@0: 
michael@0: bool
michael@0: DOMStorageDBThread::ShouldPreloadScope(const nsACString& aScope)
michael@0: {
michael@0:   MonitorAutoLock monitor(mMonitor);
michael@0:   return mScopesHavingData.Contains(aScope);
michael@0: }
michael@0: 
michael@0: namespace { // anon
michael@0: 
michael@0: PLDHashOperator
michael@0: GetScopesHavingDataEnum(nsCStringHashKey* aKey, void* aArg)
michael@0: {
michael@0:   InfallibleTArray<nsCString>* scopes =
michael@0:       static_cast<InfallibleTArray<nsCString>*>(aArg);
michael@0:   scopes->AppendElement(aKey->GetKey());
michael@0:   return PL_DHASH_NEXT;
michael@0: }
michael@0: 
michael@0: } // anon
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::GetScopesHavingData(InfallibleTArray<nsCString>* aScopes)
michael@0: {
michael@0:   MonitorAutoLock monitor(mMonitor);
michael@0:   mScopesHavingData.EnumerateEntries(GetScopesHavingDataEnum, aScopes);
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::InsertDBOp(DOMStorageDBThread::DBOperation* aOperation)
michael@0: {
michael@0:   MonitorAutoLock monitor(mMonitor);
michael@0: 
michael@0:   // Sentinel to don't forget to delete the operation when we exit early.
michael@0:   nsAutoPtr<DOMStorageDBThread::DBOperation> opScope(aOperation);
michael@0: 
michael@0:   if (mStopIOThread) {
michael@0:     // Thread use after shutdown demanded.
michael@0:     MOZ_ASSERT(false);
michael@0:     return NS_ERROR_NOT_INITIALIZED;
michael@0:   }
michael@0: 
michael@0:   if (NS_FAILED(mStatus)) {
michael@0:     MonitorAutoUnlock unlock(mMonitor);
michael@0:     aOperation->Finalize(mStatus);
michael@0:     return mStatus;
michael@0:   }
michael@0: 
michael@0:   switch (aOperation->Type()) {
michael@0:   case DBOperation::opPreload:
michael@0:   case DBOperation::opPreloadUrgent:
michael@0:     if (mPendingTasks.IsScopeUpdatePending(aOperation->Scope())) {
michael@0:       // If there is a pending update operation for the scope first do the flush
michael@0:       // before we preload the cache.  This may happen in an extremely rare case
michael@0:       // when a child process throws away its cache before flush on the parent
michael@0:       // has finished.  If we would preloaded the cache as a priority operation 
michael@0:       // before the pending flush, we would have got an inconsistent cache content.
michael@0:       mFlushImmediately = true;
michael@0:     } else if (mPendingTasks.IsScopeClearPending(aOperation->Scope())) {
michael@0:       // The scope is scheduled to be cleared, so just quickly load as empty.
michael@0:       // We need to do this to prevent load of the DB data before the scope has
michael@0:       // actually been cleared from the database.  Preloads are processed
michael@0:       // immediately before update and clear operations on the database that
michael@0:       // are flushed periodically in batches.
michael@0:       MonitorAutoUnlock unlock(mMonitor);
michael@0:       aOperation->Finalize(NS_OK);
michael@0:       return NS_OK;
michael@0:     }
michael@0:     // NO BREAK
michael@0: 
michael@0:   case DBOperation::opGetUsage:
michael@0:     if (aOperation->Type() == DBOperation::opPreloadUrgent) {
michael@0:       SetHigherPriority(); // Dropped back after urgent preload execution
michael@0:       mPreloads.InsertElementAt(0, aOperation);
michael@0:     } else {
michael@0:       mPreloads.AppendElement(aOperation);
michael@0:     }
michael@0: 
michael@0:     // DB operation adopted, don't delete it.
michael@0:     opScope.forget();
michael@0: 
michael@0:     // Immediately start executing this.
michael@0:     monitor.Notify();
michael@0:     break;
michael@0: 
michael@0:   default:
michael@0:     // Update operations are first collected, coalesced and then flushed
michael@0:     // after a short time.
michael@0:     mPendingTasks.Add(aOperation);
michael@0: 
michael@0:     // DB operation adopted, don't delete it.
michael@0:     opScope.forget();
michael@0: 
michael@0:     ScheduleFlush();
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::SetHigherPriority()
michael@0: {
michael@0:   ++mPriorityCounter;
michael@0:   PR_SetThreadPriority(mThread, PR_PRIORITY_URGENT);
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::SetDefaultPriority()
michael@0: {
michael@0:   if (--mPriorityCounter <= 0) {
michael@0:     PR_SetThreadPriority(mThread, PR_PRIORITY_LOW);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::ThreadFunc(void* aArg)
michael@0: {
michael@0:   PR_SetCurrentThreadName("localStorage DB");
michael@0:   mozilla::IOInterposer::RegisterCurrentThread();
michael@0: 
michael@0:   DOMStorageDBThread* thread = static_cast<DOMStorageDBThread*>(aArg);
michael@0:   thread->ThreadFunc();
michael@0:   mozilla::IOInterposer::UnregisterCurrentThread();
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::ThreadFunc()
michael@0: {
michael@0:   nsresult rv = InitDatabase();
michael@0: 
michael@0:   MonitorAutoLock lockMonitor(mMonitor);
michael@0: 
michael@0:   if (NS_FAILED(rv)) {
michael@0:     mStatus = rv;
michael@0:     mStopIOThread = true;
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   while (MOZ_LIKELY(!mStopIOThread || mPreloads.Length() || mPendingTasks.HasTasks())) {
michael@0:     if (MOZ_UNLIKELY(TimeUntilFlush() == 0)) {
michael@0:       // Flush time is up or flush has been forced, do it now.
michael@0:       UnscheduleFlush();
michael@0:       if (mPendingTasks.Prepare()) {
michael@0:         {
michael@0:           MonitorAutoUnlock unlockMonitor(mMonitor);
michael@0:           rv = mPendingTasks.Execute(this);
michael@0:         }
michael@0: 
michael@0:         if (!mPendingTasks.Finalize(rv)) {
michael@0:           mStatus = rv;
michael@0:           NS_WARNING("localStorage DB access broken");
michael@0:         }
michael@0:       }
michael@0:       NotifyFlushCompletion();
michael@0:     } else if (MOZ_LIKELY(mPreloads.Length())) {
michael@0:       nsAutoPtr<DBOperation> op(mPreloads[0]);
michael@0:       mPreloads.RemoveElementAt(0);
michael@0:       {
michael@0:         MonitorAutoUnlock unlockMonitor(mMonitor);
michael@0:         op->PerformAndFinalize(this);
michael@0:       }
michael@0: 
michael@0:       if (op->Type() == DBOperation::opPreloadUrgent) {
michael@0:         SetDefaultPriority(); // urgent preload unscheduled
michael@0:       }
michael@0:     } else if (MOZ_UNLIKELY(!mStopIOThread)) {
michael@0:       lockMonitor.Wait(TimeUntilFlush());
michael@0:     }
michael@0:   } // thread loop
michael@0: 
michael@0:   mStatus = ShutdownDatabase();
michael@0: }
michael@0: 
michael@0: extern void
michael@0: ReverseString(const nsCSubstring& aSource, nsCSubstring& aResult);
michael@0: 
michael@0: namespace { // anon
michael@0: 
michael@0: class nsReverseStringSQLFunction MOZ_FINAL : public mozIStorageFunction
michael@0: {
michael@0:   NS_DECL_ISUPPORTS
michael@0:   NS_DECL_MOZISTORAGEFUNCTION
michael@0: };
michael@0: 
michael@0: NS_IMPL_ISUPPORTS(nsReverseStringSQLFunction, mozIStorageFunction)
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: nsReverseStringSQLFunction::OnFunctionCall(
michael@0:     mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult)
michael@0: {
michael@0:   nsresult rv;
michael@0: 
michael@0:   nsAutoCString stringToReverse;
michael@0:   rv = aFunctionArguments->GetUTF8String(0, stringToReverse);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   nsAutoCString result;
michael@0:   ReverseString(stringToReverse, result);
michael@0: 
michael@0:   nsCOMPtr<nsIWritableVariant> outVar(do_CreateInstance(
michael@0:       NS_VARIANT_CONTRACTID, &rv));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = outVar->SetAsAUTF8String(result);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   *aResult = outVar.get();
michael@0:   outVar.forget();
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: } // anon
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::OpenDatabaseConnection()
michael@0: {
michael@0:   nsresult rv;
michael@0: 
michael@0:   MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:   nsCOMPtr<mozIStorageService> service
michael@0:       = do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   nsCOMPtr<mozIStorageConnection> connection;
michael@0:   rv = service->OpenUnsharedDatabase(mDatabaseFile, getter_AddRefs(mWorkerConnection));
michael@0:   if (rv == NS_ERROR_FILE_CORRUPTED) {
michael@0:     // delete the db and try opening again
michael@0:     rv = mDatabaseFile->Remove(false);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:     rv = service->OpenUnsharedDatabase(mDatabaseFile, getter_AddRefs(mWorkerConnection));
michael@0:   }
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::InitDatabase()
michael@0: {
michael@0:   Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_INIT_DATABASE_MS> timer;
michael@0: 
michael@0:   nsresult rv;
michael@0: 
michael@0:   // Here we are on the worker thread. This opens the worker connection.
michael@0:   MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:   rv = OpenDatabaseConnection();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = TryJournalMode();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Create a read-only clone
michael@0:   (void)mWorkerConnection->Clone(true, getter_AddRefs(mReaderConnection));
michael@0:   NS_ENSURE_TRUE(mReaderConnection, NS_ERROR_FAILURE);
michael@0: 
michael@0:   mozStorageTransaction transaction(mWorkerConnection, false);
michael@0: 
michael@0:   // Ensure Gecko 1.9.1 storage table
michael@0:   rv = mWorkerConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
michael@0:          "CREATE TABLE IF NOT EXISTS webappsstore2 ("
michael@0:          "scope TEXT, "
michael@0:          "key TEXT, "
michael@0:          "value TEXT, "
michael@0:          "secure INTEGER, "
michael@0:          "owner TEXT)"));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = mWorkerConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
michael@0:         "CREATE UNIQUE INDEX IF NOT EXISTS scope_key_index"
michael@0:         " ON webappsstore2(scope, key)"));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   nsCOMPtr<mozIStorageFunction> function1(new nsReverseStringSQLFunction());
michael@0:   NS_ENSURE_TRUE(function1, NS_ERROR_OUT_OF_MEMORY);
michael@0: 
michael@0:   rv = mWorkerConnection->CreateFunction(NS_LITERAL_CSTRING("REVERSESTRING"), 1, function1);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   bool exists;
michael@0: 
michael@0:   // Check if there is storage of Gecko 1.9.0 and if so, upgrade that storage
michael@0:   // to actual webappsstore2 table and drop the obsolete table. First process
michael@0:   // this newer table upgrade to priority potential duplicates from older
michael@0:   // storage table.
michael@0:   rv = mWorkerConnection->TableExists(NS_LITERAL_CSTRING("webappsstore"),
michael@0:                                 &exists);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   if (exists) {
michael@0:     rv = mWorkerConnection->ExecuteSimpleSQL(
michael@0:       NS_LITERAL_CSTRING("INSERT OR IGNORE INTO "
michael@0:                          "webappsstore2(scope, key, value, secure, owner) "
michael@0:                          "SELECT REVERSESTRING(domain) || '.:', key, value, secure, owner "
michael@0:                          "FROM webappsstore"));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     rv = mWorkerConnection->ExecuteSimpleSQL(
michael@0:       NS_LITERAL_CSTRING("DROP TABLE webappsstore"));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   }
michael@0: 
michael@0:   // Check if there is storage of Gecko 1.8 and if so, upgrade that storage
michael@0:   // to actual webappsstore2 table and drop the obsolete table. Potential
michael@0:   // duplicates will be ignored.
michael@0:   rv = mWorkerConnection->TableExists(NS_LITERAL_CSTRING("moz_webappsstore"),
michael@0:                                 &exists);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   if (exists) {
michael@0:     rv = mWorkerConnection->ExecuteSimpleSQL(
michael@0:       NS_LITERAL_CSTRING("INSERT OR IGNORE INTO "
michael@0:                          "webappsstore2(scope, key, value, secure, owner) "
michael@0:                          "SELECT REVERSESTRING(domain) || '.:', key, value, secure, domain "
michael@0:                          "FROM moz_webappsstore"));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     rv = mWorkerConnection->ExecuteSimpleSQL(
michael@0:       NS_LITERAL_CSTRING("DROP TABLE moz_webappsstore"));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   }
michael@0: 
michael@0:   rv = transaction.Commit();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Database open and all initiation operation are done.  Switching this flag
michael@0:   // to true allow main thread to read directly from the database.
michael@0:   // If we would allow this sooner, we would have opened a window where main thread
michael@0:   // read might operate on a totaly broken and incosistent database.
michael@0:   mDBReady = true;
michael@0: 
michael@0:   // List scopes having any stored data
michael@0:   nsCOMPtr<mozIStorageStatement> stmt;
michael@0:   rv = mWorkerConnection->CreateStatement(NS_LITERAL_CSTRING("SELECT DISTINCT scope FROM webappsstore2"),
michael@0:                                     getter_AddRefs(stmt));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:   while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
michael@0:     nsAutoCString foundScope;
michael@0:     rv = stmt->GetUTF8String(0, foundScope);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     MonitorAutoLock monitor(mMonitor);
michael@0:     mScopesHavingData.PutEntry(foundScope);
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::SetJournalMode(bool aIsWal)
michael@0: {
michael@0:   nsresult rv;
michael@0: 
michael@0:   nsAutoCString stmtString(
michael@0:     MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA journal_mode = ");
michael@0:   if (aIsWal) {
michael@0:     stmtString.AppendLiteral("wal");
michael@0:   } else {
michael@0:     stmtString.AppendLiteral("truncate");
michael@0:   }
michael@0: 
michael@0:   nsCOMPtr<mozIStorageStatement> stmt;
michael@0:   rv = mWorkerConnection->CreateStatement(stmtString, getter_AddRefs(stmt));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:   bool hasResult = false;
michael@0:   rv = stmt->ExecuteStep(&hasResult);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   if (!hasResult) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   nsAutoCString journalMode;
michael@0:   rv = stmt->GetUTF8String(0, journalMode);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   if ((aIsWal && !journalMode.EqualsLiteral("wal")) ||
michael@0:       (!aIsWal && !journalMode.EqualsLiteral("truncate"))) {
michael@0:     return NS_ERROR_FAILURE;
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::TryJournalMode()
michael@0: {
michael@0:   nsresult rv;
michael@0: 
michael@0:   rv = SetJournalMode(true);
michael@0:   if (NS_FAILED(rv)) {
michael@0:     mWALModeEnabled = false;
michael@0: 
michael@0:     rv = SetJournalMode(false);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   } else {
michael@0:     mWALModeEnabled = true;
michael@0: 
michael@0:     rv = ConfigureWALBehavior();
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::ConfigureWALBehavior()
michael@0: {
michael@0:   // Get the DB's page size
michael@0:   nsCOMPtr<mozIStorageStatement> stmt;
michael@0:   nsresult rv = mWorkerConnection->CreateStatement(NS_LITERAL_CSTRING(
michael@0:     MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"
michael@0:   ), getter_AddRefs(stmt));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   bool hasResult = false;
michael@0:   rv = stmt->ExecuteStep(&hasResult);
michael@0:   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
michael@0: 
michael@0:   int32_t pageSize = 0;
michael@0:   rv = stmt->GetInt32(0, &pageSize);
michael@0:   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && pageSize > 0, NS_ERROR_UNEXPECTED);
michael@0: 
michael@0:   // Set the threshold for auto-checkpointing the WAL.
michael@0:   // We don't want giant logs slowing down reads & shutdown.
michael@0:   int32_t thresholdInPages = static_cast<int32_t>(MAX_WAL_SIZE_BYTES / pageSize);
michael@0:   nsAutoCString thresholdPragma("PRAGMA wal_autocheckpoint = ");
michael@0:   thresholdPragma.AppendInt(thresholdInPages);
michael@0:   rv = mWorkerConnection->ExecuteSimpleSQL(thresholdPragma);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   // Set the maximum WAL log size to reduce footprint on mobile (large empty
michael@0:   // WAL files will be truncated)
michael@0:   nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
michael@0:   // bug 600307: mak recommends setting this to 3 times the auto-checkpoint threshold
michael@0:   journalSizePragma.AppendInt(MAX_WAL_SIZE_BYTES * 3);
michael@0:   rv = mWorkerConnection->ExecuteSimpleSQL(journalSizePragma);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::ShutdownDatabase()
michael@0: {
michael@0:   // Has to be called on the worker thread.
michael@0:   MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:   nsresult rv = mStatus;
michael@0: 
michael@0:   mDBReady = false;
michael@0: 
michael@0:   // Finalize the cached statements.
michael@0:   mReaderStatements.FinalizeStatements();
michael@0:   mWorkerStatements.FinalizeStatements();
michael@0: 
michael@0:   if (mReaderConnection) {
michael@0:     // No need to sync access to mReaderConnection since the main thread
michael@0:     // is right now joining this thread, unable to execute any events.
michael@0:     mReaderConnection->Close();
michael@0:     mReaderConnection = nullptr;
michael@0:   }
michael@0: 
michael@0:   if (mWorkerConnection) {
michael@0:     rv = mWorkerConnection->Close();
michael@0:     mWorkerConnection = nullptr;
michael@0:   }
michael@0: 
michael@0:   return rv;
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::ScheduleFlush()
michael@0: {
michael@0:   if (mDirtyEpoch) {
michael@0:     return; // Already scheduled
michael@0:   }
michael@0: 
michael@0:   mDirtyEpoch = PR_IntervalNow() | 1; // Must be non-zero to indicate we are scheduled
michael@0: 
michael@0:   // Wake the monitor from indefinite sleep...
michael@0:   mMonitor.Notify();
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::UnscheduleFlush()
michael@0: {
michael@0:   // We are just about to do the flush, drop flags
michael@0:   mFlushImmediately = false;
michael@0:   mDirtyEpoch = 0;
michael@0: }
michael@0: 
michael@0: PRIntervalTime
michael@0: DOMStorageDBThread::TimeUntilFlush()
michael@0: {
michael@0:   if (mFlushImmediately) {
michael@0:     return 0; // Do it now regardless the timeout.
michael@0:   }
michael@0: 
michael@0:   static_assert(PR_INTERVAL_NO_TIMEOUT != 0,
michael@0:       "PR_INTERVAL_NO_TIMEOUT must be non-zero");
michael@0: 
michael@0:   if (!mDirtyEpoch) {
michael@0:     return PR_INTERVAL_NO_TIMEOUT; // No pending task...
michael@0:   }
michael@0: 
michael@0:   static const PRIntervalTime kMaxAge = PR_MillisecondsToInterval(FLUSHING_INTERVAL_MS);
michael@0: 
michael@0:   PRIntervalTime now = PR_IntervalNow() | 1;
michael@0:   PRIntervalTime age = now - mDirtyEpoch;
michael@0:   if (age > kMaxAge) {
michael@0:     return 0; // It is time.
michael@0:   }
michael@0: 
michael@0:   return kMaxAge - age; // Time left, this is used to sleep the monitor
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::NotifyFlushCompletion()
michael@0: {
michael@0: #ifdef DOM_STORAGE_TESTS
michael@0:   if (!NS_IsMainThread()) {
michael@0:     nsRefPtr<nsRunnableMethod<DOMStorageDBThread, void, false> > event =
michael@0:       NS_NewNonOwningRunnableMethod(this, &DOMStorageDBThread::NotifyFlushCompletion);
michael@0:     NS_DispatchToMainThread(event);
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
michael@0:   if (obs) {
michael@0:     obs->NotifyObservers(nullptr, "domstorage-test-flushed", nullptr);
michael@0:   }
michael@0: #endif
michael@0: }
michael@0: 
michael@0: // DOMStorageDBThread::DBOperation
michael@0: 
michael@0: DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
michael@0:                                              DOMStorageCacheBridge* aCache,
michael@0:                                              const nsAString& aKey,
michael@0:                                              const nsAString& aValue)
michael@0: : mType(aType)
michael@0: , mCache(aCache)
michael@0: , mKey(aKey)
michael@0: , mValue(aValue)
michael@0: {
michael@0:   MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
michael@0: }
michael@0: 
michael@0: DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
michael@0:                                              DOMStorageUsageBridge* aUsage)
michael@0: : mType(aType)
michael@0: , mUsage(aUsage)
michael@0: {
michael@0:   MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
michael@0: }
michael@0: 
michael@0: DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
michael@0:                                              const nsACString& aScope)
michael@0: : mType(aType)
michael@0: , mCache(nullptr)
michael@0: , mScope(aScope)
michael@0: {
michael@0:   MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
michael@0: }
michael@0: 
michael@0: DOMStorageDBThread::DBOperation::~DBOperation()
michael@0: {
michael@0:   MOZ_COUNT_DTOR(DOMStorageDBThread::DBOperation);
michael@0: }
michael@0: 
michael@0: const nsCString
michael@0: DOMStorageDBThread::DBOperation::Scope()
michael@0: {
michael@0:   if (mCache) {
michael@0:     return mCache->Scope();
michael@0:   }
michael@0: 
michael@0:   return mScope;
michael@0: }
michael@0: 
michael@0: const nsCString
michael@0: DOMStorageDBThread::DBOperation::Target()
michael@0: {
michael@0:   switch (mType) {
michael@0:     case opAddItem:
michael@0:     case opUpdateItem:
michael@0:     case opRemoveItem:
michael@0:       return Scope() + NS_LITERAL_CSTRING("|") + NS_ConvertUTF16toUTF8(mKey);
michael@0: 
michael@0:     default:
michael@0:       return Scope();
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::DBOperation::PerformAndFinalize(DOMStorageDBThread* aThread)
michael@0: {
michael@0:   Finalize(Perform(aThread));
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::DBOperation::Perform(DOMStorageDBThread* aThread)
michael@0: {
michael@0:   nsresult rv;
michael@0: 
michael@0:   switch (mType) {
michael@0:   case opPreload:
michael@0:   case opPreloadUrgent:
michael@0:   {
michael@0:     // Already loaded?
michael@0:     if (mCache->Loaded()) {
michael@0:       break;
michael@0:     }
michael@0: 
michael@0:     StatementCache* statements;
michael@0:     if (MOZ_UNLIKELY(NS_IsMainThread())) {
michael@0:       statements = &aThread->mReaderStatements;
michael@0:     } else {
michael@0:       statements = &aThread->mWorkerStatements;
michael@0:     }
michael@0: 
michael@0:     // OFFSET is an optimization when we have to do a sync load
michael@0:     // and cache has already loaded some parts asynchronously.
michael@0:     // It skips keys we have already loaded.
michael@0:     nsCOMPtr<mozIStorageStatement> stmt = statements->GetCachedStatement(
michael@0:         "SELECT key, value FROM webappsstore2 "
michael@0:         "WHERE scope = :scope ORDER BY key "
michael@0:         "LIMIT -1 OFFSET :offset");
michael@0:     NS_ENSURE_STATE(stmt);
michael@0:     mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
michael@0:                                     mCache->Scope());
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("offset"),
michael@0:                                static_cast<int32_t>(mCache->LoadedCount()));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     bool exists;
michael@0:     while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
michael@0:       nsAutoString key;
michael@0:       rv = stmt->GetString(0, key);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:       nsAutoString value;
michael@0:       rv = stmt->GetString(1, value);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:       if (!mCache->LoadItem(key, value)) {
michael@0:         break;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     mCache->LoadDone(NS_OK);
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   case opGetUsage:
michael@0:   {
michael@0:     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
michael@0:       "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2"
michael@0:       " WHERE scope LIKE :scope"
michael@0:     );
michael@0:     NS_ENSURE_STATE(stmt);
michael@0: 
michael@0:     mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
michael@0:                                     mUsage->Scope() + NS_LITERAL_CSTRING("%"));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     bool exists;
michael@0:     rv = stmt->ExecuteStep(&exists);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     int64_t usage = 0;
michael@0:     if (exists) {
michael@0:       rv = stmt->GetInt64(0, &usage);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0:     }
michael@0: 
michael@0:     mUsage->LoadUsage(usage);
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   case opAddItem:
michael@0:   case opUpdateItem:
michael@0:   {
michael@0:     MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
michael@0:       "INSERT OR REPLACE INTO webappsstore2 (scope, key, value) "
michael@0:       "VALUES (:scope, :key, :value) "
michael@0:     );
michael@0:     NS_ENSURE_STATE(stmt);
michael@0: 
michael@0:     mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
michael@0:                                     mCache->Scope());
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
michael@0:                                 mKey);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"),
michael@0:                                 mValue);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     rv = stmt->Execute();
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     aThread->mScopesHavingData.PutEntry(Scope());
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   case opRemoveItem:
michael@0:   {
michael@0:     MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
michael@0:       "DELETE FROM webappsstore2 "
michael@0:       "WHERE scope = :scope "
michael@0:         "AND key = :key "
michael@0:     );
michael@0:     NS_ENSURE_STATE(stmt);
michael@0:     mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
michael@0:                                     mCache->Scope());
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
michael@0:                                 mKey);
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     rv = stmt->Execute();
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   case opClear:
michael@0:   {
michael@0:     MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
michael@0:       "DELETE FROM webappsstore2 "
michael@0:       "WHERE scope = :scope"
michael@0:     );
michael@0:     NS_ENSURE_STATE(stmt);
michael@0:     mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
michael@0:                                     mCache->Scope());
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     rv = stmt->Execute();
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     aThread->mScopesHavingData.RemoveEntry(Scope());
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   case opClearAll:
michael@0:   {
michael@0:     MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
michael@0:       "DELETE FROM webappsstore2"
michael@0:     );
michael@0:     NS_ENSURE_STATE(stmt);
michael@0:     mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:     rv = stmt->Execute();
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     aThread->mScopesHavingData.Clear();
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   case opClearMatchingScope:
michael@0:   {
michael@0:     MOZ_ASSERT(!NS_IsMainThread());
michael@0: 
michael@0:     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
michael@0:       "DELETE FROM webappsstore2"
michael@0:       " WHERE scope GLOB :scope"
michael@0:     );
michael@0:     NS_ENSURE_STATE(stmt);
michael@0:     mozStorageStatementScoper scope(stmt);
michael@0: 
michael@0:     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
michael@0:                                     mScope + NS_LITERAL_CSTRING("*"));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     rv = stmt->Execute();
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   default:
michael@0:     NS_ERROR("Unknown task type");
michael@0:     break;
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::DBOperation::Finalize(nsresult aRv)
michael@0: {
michael@0:   switch (mType) {
michael@0:   case opPreloadUrgent:
michael@0:   case opPreload:
michael@0:     if (NS_FAILED(aRv)) {
michael@0:       // When we are here, something failed when loading from the database.
michael@0:       // Notify that the storage is loaded to prevent deadlock of the main thread,
michael@0:       // even though it is actually empty or incomplete.
michael@0:       NS_WARNING("Failed to preload localStorage");
michael@0:     }
michael@0: 
michael@0:     mCache->LoadDone(aRv);
michael@0:     break;
michael@0: 
michael@0:   case opGetUsage:
michael@0:     if (NS_FAILED(aRv)) {
michael@0:       mUsage->LoadUsage(0);
michael@0:     }
michael@0: 
michael@0:     break;
michael@0: 
michael@0:   default:
michael@0:     if (NS_FAILED(aRv)) {
michael@0:       NS_WARNING("localStorage update/clear operation failed,"
michael@0:                  " data may not persist or clean up");
michael@0:     }
michael@0: 
michael@0:     break;
michael@0:   }
michael@0: }
michael@0: 
michael@0: // DOMStorageDBThread::PendingOperations
michael@0: 
michael@0: DOMStorageDBThread::PendingOperations::PendingOperations()
michael@0: : mFlushFailureCount(0)
michael@0: {
michael@0: }
michael@0: 
michael@0: bool
michael@0: DOMStorageDBThread::PendingOperations::HasTasks()
michael@0: {
michael@0:   return !!mUpdates.Count() || !!mClears.Count();
michael@0: }
michael@0: 
michael@0: namespace { // anon
michael@0: 
michael@0: PLDHashOperator
michael@0: ForgetUpdatesForScope(const nsACString& aMapping,
michael@0:                       nsAutoPtr<DOMStorageDBThread::DBOperation>& aPendingTask,
michael@0:                       void* aArg)
michael@0: {
michael@0:   DOMStorageDBThread::DBOperation* newOp = static_cast<DOMStorageDBThread::DBOperation*>(aArg);
michael@0: 
michael@0:   if (newOp->Type() == DOMStorageDBThread::DBOperation::opClear &&
michael@0:       aPendingTask->Scope() != newOp->Scope()) {
michael@0:     return PL_DHASH_NEXT;
michael@0:   }
michael@0: 
michael@0:   if (newOp->Type() == DOMStorageDBThread::DBOperation::opClearMatchingScope &&
michael@0:       !StringBeginsWith(aPendingTask->Scope(), newOp->Scope())) {
michael@0:     return PL_DHASH_NEXT;
michael@0:   }
michael@0: 
michael@0:   return PL_DHASH_REMOVE;
michael@0: }
michael@0: 
michael@0: } // anon
michael@0: 
michael@0: bool
michael@0: DOMStorageDBThread::PendingOperations::CheckForCoalesceOpportunity(DBOperation* aNewOp,
michael@0:                                                                    DBOperation::OperationType aPendingType,
michael@0:                                                                    DBOperation::OperationType aNewType)
michael@0: {
michael@0:   if (aNewOp->Type() != aNewType) {
michael@0:     return false;
michael@0:   }
michael@0: 
michael@0:   DOMStorageDBThread::DBOperation* pendingTask;
michael@0:   if (!mUpdates.Get(aNewOp->Target(), &pendingTask)) {
michael@0:     return false;
michael@0:   }
michael@0: 
michael@0:   if (pendingTask->Type() != aPendingType) {
michael@0:     return false;
michael@0:   }
michael@0: 
michael@0:   return true;
michael@0: }
michael@0: 
michael@0: void
michael@0: DOMStorageDBThread::PendingOperations::Add(DOMStorageDBThread::DBOperation* aOperation)
michael@0: {
michael@0:   // Optimize: when a key to remove has never been written to disk
michael@0:   // just bypass this operation.  A kew is new when an operation scheduled
michael@0:   // to write it to the database is of type opAddItem.
michael@0:   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem, DBOperation::opRemoveItem)) {
michael@0:     mUpdates.Remove(aOperation->Target());
michael@0:     delete aOperation;
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   // Optimize: when changing a key that is new and has never been
michael@0:   // written to disk, keep type of the operation to store it at opAddItem.
michael@0:   // This allows optimization to just forget adding a new key when
michael@0:   // it is removed from the storage before flush.
michael@0:   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem, DBOperation::opUpdateItem)) {
michael@0:     aOperation->mType = DBOperation::opAddItem;
michael@0:   }
michael@0: 
michael@0:   // Optimize: to prevent lose of remove operation on a key when doing
michael@0:   // remove/set/remove on a previously existing key we have to change
michael@0:   // opAddItem to opUpdateItem on the new operation when there is opRemoveItem
michael@0:   // pending for the key.
michael@0:   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opRemoveItem, DBOperation::opAddItem)) {
michael@0:     aOperation->mType = DBOperation::opUpdateItem;
michael@0:   }
michael@0: 
michael@0:   switch (aOperation->Type())
michael@0:   {
michael@0:   // Operations on single keys
michael@0: 
michael@0:   case DBOperation::opAddItem:
michael@0:   case DBOperation::opUpdateItem:
michael@0:   case DBOperation::opRemoveItem:
michael@0:     // Override any existing operation for the target (=scope+key).
michael@0:     mUpdates.Put(aOperation->Target(), aOperation);
michael@0:     break;
michael@0: 
michael@0:   // Clear operations
michael@0: 
michael@0:   case DBOperation::opClear:
michael@0:   case DBOperation::opClearMatchingScope:
michael@0:     // Drop all update (insert/remove) operations for equivavelent or matching scope.
michael@0:     // We do this as an optimization as well as a must based on the logic,
michael@0:     // if we would not delete the update tasks, changes would have been stored
michael@0:     // to the database after clear operations have been executed.
michael@0:     mUpdates.Enumerate(ForgetUpdatesForScope, aOperation);
michael@0:     mClears.Put(aOperation->Target(), aOperation);
michael@0:     break;
michael@0: 
michael@0:   case DBOperation::opClearAll:
michael@0:     // Drop simply everything, this is a super-operation.
michael@0:     mUpdates.Clear();
michael@0:     mClears.Clear();
michael@0:     mClears.Put(aOperation->Target(), aOperation);
michael@0:     break;
michael@0: 
michael@0:   default:
michael@0:     MOZ_ASSERT(false);
michael@0:     break;
michael@0:   }
michael@0: }
michael@0: 
michael@0: namespace { // anon
michael@0: 
michael@0: PLDHashOperator
michael@0: CollectTasks(const nsACString& aMapping, nsAutoPtr<DOMStorageDBThread::DBOperation>& aOperation, void* aArg)
michael@0: {
michael@0:   nsTArray<nsAutoPtr<DOMStorageDBThread::DBOperation> >* tasks =
michael@0:     static_cast<nsTArray<nsAutoPtr<DOMStorageDBThread::DBOperation> >*>(aArg);
michael@0: 
michael@0:   tasks->AppendElement(aOperation.forget());
michael@0:   return PL_DHASH_NEXT;
michael@0: }
michael@0: 
michael@0: } // anon
michael@0: 
michael@0: bool
michael@0: DOMStorageDBThread::PendingOperations::Prepare()
michael@0: {
michael@0:   // Called under the lock
michael@0: 
michael@0:   // First collect clear operations and then updates, we can
michael@0:   // do this since whenever a clear operation for a scope is
michael@0:   // scheduled, we drop all updates matching that scope. So,
michael@0:   // all scope-related update operations we have here now were
michael@0:   // scheduled after the clear operations.
michael@0:   mClears.Enumerate(CollectTasks, &mExecList);
michael@0:   mClears.Clear();
michael@0: 
michael@0:   mUpdates.Enumerate(CollectTasks, &mExecList);
michael@0:   mUpdates.Clear();
michael@0: 
michael@0:   return !!mExecList.Length();
michael@0: }
michael@0: 
michael@0: nsresult
michael@0: DOMStorageDBThread::PendingOperations::Execute(DOMStorageDBThread* aThread)
michael@0: {
michael@0:   // Called outside the lock
michael@0: 
michael@0:   mozStorageTransaction transaction(aThread->mWorkerConnection, false);
michael@0: 
michael@0:   nsresult rv;
michael@0: 
michael@0:   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
michael@0:     DOMStorageDBThread::DBOperation* task = mExecList[i];
michael@0:     rv = task->Perform(aThread);
michael@0:     if (NS_FAILED(rv)) {
michael@0:       return rv;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   rv = transaction.Commit();
michael@0:   if (NS_FAILED(rv)) {
michael@0:     return rv;
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: bool
michael@0: DOMStorageDBThread::PendingOperations::Finalize(nsresult aRv)
michael@0: {
michael@0:   // Called under the lock
michael@0: 
michael@0:   // The list is kept on a failure to retry it
michael@0:   if (NS_FAILED(aRv)) {
michael@0:     // XXX Followup: we may try to reopen the database and flush these
michael@0:     // pending tasks, however testing showed that even though I/O is actually
michael@0:     // broken some amount of operations is left in sqlite+system buffers and
michael@0:     // seems like successfully flushed to disk.
michael@0:     // Tested by removing a flash card and disconnecting from network while
michael@0:     // using a network drive on Windows system.
michael@0:     NS_WARNING("Flush operation on localStorage database failed");
michael@0: 
michael@0:     ++mFlushFailureCount;
michael@0: 
michael@0:     return mFlushFailureCount >= 5;
michael@0:   }
michael@0: 
michael@0:   mFlushFailureCount = 0;
michael@0:   mExecList.Clear();
michael@0:   return true;
michael@0: }
michael@0: 
michael@0: namespace { // anon
michael@0: 
michael@0: class FindPendingOperationForScopeData
michael@0: {
michael@0: public:
michael@0:   FindPendingOperationForScopeData(const nsACString& aScope) : mScope(aScope), mFound(false) {}
michael@0:   nsCString mScope;
michael@0:   bool mFound;
michael@0: };
michael@0: 
michael@0: PLDHashOperator
michael@0: FindPendingClearForScope(const nsACString& aMapping,
michael@0:                          DOMStorageDBThread::DBOperation* aPendingOperation,
michael@0:                          void* aArg)
michael@0: {
michael@0:   FindPendingOperationForScopeData* data =
michael@0:     static_cast<FindPendingOperationForScopeData*>(aArg);
michael@0: 
michael@0:   if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClearAll) {
michael@0:     data->mFound = true;
michael@0:     return PL_DHASH_STOP;
michael@0:   }
michael@0: 
michael@0:   if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClear &&
michael@0:       data->mScope == aPendingOperation->Scope()) {
michael@0:     data->mFound = true;
michael@0:     return PL_DHASH_STOP;
michael@0:   }
michael@0: 
michael@0:   if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClearMatchingScope &&
michael@0:       StringBeginsWith(data->mScope, aPendingOperation->Scope())) {
michael@0:     data->mFound = true;
michael@0:     return PL_DHASH_STOP;
michael@0:   }
michael@0: 
michael@0:   return PL_DHASH_NEXT;
michael@0: }
michael@0: 
michael@0: } // anon
michael@0: 
michael@0: bool
michael@0: DOMStorageDBThread::PendingOperations::IsScopeClearPending(const nsACString& aScope)
michael@0: {
michael@0:   // Called under the lock
michael@0: 
michael@0:   FindPendingOperationForScopeData data(aScope);
michael@0:   mClears.EnumerateRead(FindPendingClearForScope, &data);
michael@0:   if (data.mFound) {
michael@0:     return true;
michael@0:   }
michael@0: 
michael@0:   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
michael@0:     DOMStorageDBThread::DBOperation* task = mExecList[i];
michael@0:     FindPendingClearForScope(EmptyCString(), task, &data);
michael@0: 
michael@0:     if (data.mFound) {
michael@0:       return true;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   return false;
michael@0: }
michael@0: 
michael@0: namespace { // anon
michael@0: 
michael@0: PLDHashOperator
michael@0: FindPendingUpdateForScope(const nsACString& aMapping,
michael@0:                           DOMStorageDBThread::DBOperation* aPendingOperation,
michael@0:                           void* aArg)
michael@0: {
michael@0:   FindPendingOperationForScopeData* data =
michael@0:     static_cast<FindPendingOperationForScopeData*>(aArg);
michael@0: 
michael@0:   if ((aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opAddItem ||
michael@0:        aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opUpdateItem ||
michael@0:        aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opRemoveItem) &&
michael@0:        data->mScope == aPendingOperation->Scope()) {
michael@0:     data->mFound = true;
michael@0:     return PL_DHASH_STOP;
michael@0:   }
michael@0: 
michael@0:   return PL_DHASH_NEXT;
michael@0: }
michael@0: 
michael@0: } // anon
michael@0: 
michael@0: bool
michael@0: DOMStorageDBThread::PendingOperations::IsScopeUpdatePending(const nsACString& aScope)
michael@0: {
michael@0:   // Called under the lock
michael@0: 
michael@0:   FindPendingOperationForScopeData data(aScope);
michael@0:   mUpdates.EnumerateRead(FindPendingUpdateForScope, &data);
michael@0:   if (data.mFound) {
michael@0:     return true;
michael@0:   }
michael@0: 
michael@0:   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
michael@0:     DOMStorageDBThread::DBOperation* task = mExecList[i];
michael@0:     FindPendingUpdateForScope(EmptyCString(), task, &data);
michael@0: 
michael@0:     if (data.mFound) {
michael@0:       return true;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   return false;
michael@0: }
michael@0: 
michael@0: } // ::dom
michael@0: } // ::mozilla