dom/src/storage/DOMStorageDBThread.cpp

Thu, 22 Jan 2015 13:21:57 +0100

author
Michael Schloh von Bennewitz <michael@schloh.com>
date
Thu, 22 Jan 2015 13:21:57 +0100
branch
TOR_BUG_9701
changeset 15
b8a032363ba2
permissions
-rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

     1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
     2 /* This Source Code Form is subject to the terms of the Mozilla Public
     3  * License, v. 2.0. If a copy of the MPL was not distributed with this
     4  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
     6 #include "DOMStorageDBThread.h"
     7 #include "DOMStorageCache.h"
     9 #include "nsIEffectiveTLDService.h"
    10 #include "nsDirectoryServiceUtils.h"
    11 #include "nsAppDirectoryServiceDefs.h"
    12 #include "nsThreadUtils.h"
    13 #include "nsProxyRelease.h"
    14 #include "mozStorageCID.h"
    15 #include "mozStorageHelper.h"
    16 #include "mozIStorageService.h"
    17 #include "mozIStorageBindingParamsArray.h"
    18 #include "mozIStorageBindingParams.h"
    19 #include "mozIStorageValueArray.h"
    20 #include "mozIStorageFunction.h"
    21 #include "nsIObserverService.h"
    22 #include "nsIVariant.h"
    23 #include "mozilla/IOInterposer.h"
    24 #include "mozilla/Services.h"
    26 // How long we collect write oprerations
    27 // before they are flushed to the database
    28 // In milliseconds.
    29 #define FLUSHING_INTERVAL_MS 5000
    31 // Write Ahead Log's maximum size is 512KB
    32 #define MAX_WAL_SIZE_BYTES 512 * 1024
    34 namespace mozilla {
    35 namespace dom {
    37 DOMStorageDBBridge::DOMStorageDBBridge()
    38 {
    39 }
    42 DOMStorageDBThread::DOMStorageDBThread()
    43 : mThread(nullptr)
    44 , mMonitor("DOMStorageThreadMonitor")
    45 , mStopIOThread(false)
    46 , mWALModeEnabled(false)
    47 , mDBReady(false)
    48 , mStatus(NS_OK)
    49 , mWorkerStatements(mWorkerConnection)
    50 , mReaderStatements(mReaderConnection)
    51 , mDirtyEpoch(0)
    52 , mFlushImmediately(false)
    53 , mPriorityCounter(0)
    54 {
    55 }
    57 nsresult
    58 DOMStorageDBThread::Init()
    59 {
    60   nsresult rv;
    62   // Need to determine location on the main thread, since
    63   // NS_GetSpecialDirectory access the atom table that can
    64   // be accessed only on the main thread.
    65   rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
    66                               getter_AddRefs(mDatabaseFile));
    67   NS_ENSURE_SUCCESS(rv, rv);
    69   rv = mDatabaseFile->Append(NS_LITERAL_STRING("webappsstore.sqlite"));
    70   NS_ENSURE_SUCCESS(rv, rv);
    72   // Ensure mozIStorageService init on the main thread first.
    73   nsCOMPtr<mozIStorageService> service =
    74     do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
    75   NS_ENSURE_SUCCESS(rv, rv);
    77   // Need to keep the lock to avoid setting mThread later then
    78   // the thread body executes.
    79   MonitorAutoLock monitor(mMonitor);
    81   mThread = PR_CreateThread(PR_USER_THREAD, &DOMStorageDBThread::ThreadFunc, this,
    82                             PR_PRIORITY_LOW, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD,
    83                             262144);
    84   if (!mThread) {
    85     return NS_ERROR_OUT_OF_MEMORY;
    86   }
    88   return NS_OK;
    89 }
    91 nsresult
    92 DOMStorageDBThread::Shutdown()
    93 {
    94   if (!mThread) {
    95     return NS_ERROR_NOT_INITIALIZED;
    96   }
    98   Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_SHUTDOWN_DATABASE_MS> timer;
   100   {
   101     MonitorAutoLock monitor(mMonitor);
   103     // After we stop, no other operations can be accepted
   104     mFlushImmediately = true;
   105     mStopIOThread = true;
   106     monitor.Notify();
   107   }
   109   PR_JoinThread(mThread);
   110   mThread = nullptr;
   112   return mStatus;
   113 }
   115 void
   116 DOMStorageDBThread::SyncPreload(DOMStorageCacheBridge* aCache, bool aForceSync)
   117 {
   118   if (!aForceSync && aCache->LoadedCount()) {
   119     // Preload already started for this cache, just wait for it to finish.
   120     // LoadWait will exit after LoadDone on the cache has been called.
   121     SetHigherPriority();
   122     aCache->LoadWait();
   123     SetDefaultPriority();
   124     return;
   125   }
   127   // Bypass sync load when an update is pending in the queue to write, we would
   128   // get incosistent data in the cache.  Also don't allow sync main-thread preload
   129   // when DB open and init is still pending on the background thread.
   130   if (mDBReady && mWALModeEnabled) {
   131     bool pendingTasks;
   132     {
   133       MonitorAutoLock monitor(mMonitor);
   134       pendingTasks = mPendingTasks.IsScopeUpdatePending(aCache->Scope()) ||
   135                      mPendingTasks.IsScopeClearPending(aCache->Scope());
   136     }
   138     if (!pendingTasks) {
   139       // WAL is enabled, thus do the load synchronously on the main thread.
   140       DBOperation preload(DBOperation::opPreload, aCache);
   141       preload.PerformAndFinalize(this);
   142       return;
   143     }
   144   }
   146   // Need to go asynchronously since WAL is not allowed or scheduled updates
   147   // need to be flushed first.
   148   // Schedule preload for this cache as the first operation.
   149   nsresult rv = InsertDBOp(new DBOperation(DBOperation::opPreloadUrgent, aCache));
   151   // LoadWait exits after LoadDone of the cache has been called.
   152   if (NS_SUCCEEDED(rv)) {
   153     aCache->LoadWait();
   154   }
   155 }
   157 void
   158 DOMStorageDBThread::AsyncFlush()
   159 {
   160   MonitorAutoLock monitor(mMonitor);
   161   mFlushImmediately = true;
   162   monitor.Notify();
   163 }
   165 bool
   166 DOMStorageDBThread::ShouldPreloadScope(const nsACString& aScope)
   167 {
   168   MonitorAutoLock monitor(mMonitor);
   169   return mScopesHavingData.Contains(aScope);
   170 }
   172 namespace { // anon
   174 PLDHashOperator
   175 GetScopesHavingDataEnum(nsCStringHashKey* aKey, void* aArg)
   176 {
   177   InfallibleTArray<nsCString>* scopes =
   178       static_cast<InfallibleTArray<nsCString>*>(aArg);
   179   scopes->AppendElement(aKey->GetKey());
   180   return PL_DHASH_NEXT;
   181 }
   183 } // anon
   185 void
   186 DOMStorageDBThread::GetScopesHavingData(InfallibleTArray<nsCString>* aScopes)
   187 {
   188   MonitorAutoLock monitor(mMonitor);
   189   mScopesHavingData.EnumerateEntries(GetScopesHavingDataEnum, aScopes);
   190 }
   192 nsresult
   193 DOMStorageDBThread::InsertDBOp(DOMStorageDBThread::DBOperation* aOperation)
   194 {
   195   MonitorAutoLock monitor(mMonitor);
   197   // Sentinel to don't forget to delete the operation when we exit early.
   198   nsAutoPtr<DOMStorageDBThread::DBOperation> opScope(aOperation);
   200   if (mStopIOThread) {
   201     // Thread use after shutdown demanded.
   202     MOZ_ASSERT(false);
   203     return NS_ERROR_NOT_INITIALIZED;
   204   }
   206   if (NS_FAILED(mStatus)) {
   207     MonitorAutoUnlock unlock(mMonitor);
   208     aOperation->Finalize(mStatus);
   209     return mStatus;
   210   }
   212   switch (aOperation->Type()) {
   213   case DBOperation::opPreload:
   214   case DBOperation::opPreloadUrgent:
   215     if (mPendingTasks.IsScopeUpdatePending(aOperation->Scope())) {
   216       // If there is a pending update operation for the scope first do the flush
   217       // before we preload the cache.  This may happen in an extremely rare case
   218       // when a child process throws away its cache before flush on the parent
   219       // has finished.  If we would preloaded the cache as a priority operation 
   220       // before the pending flush, we would have got an inconsistent cache content.
   221       mFlushImmediately = true;
   222     } else if (mPendingTasks.IsScopeClearPending(aOperation->Scope())) {
   223       // The scope is scheduled to be cleared, so just quickly load as empty.
   224       // We need to do this to prevent load of the DB data before the scope has
   225       // actually been cleared from the database.  Preloads are processed
   226       // immediately before update and clear operations on the database that
   227       // are flushed periodically in batches.
   228       MonitorAutoUnlock unlock(mMonitor);
   229       aOperation->Finalize(NS_OK);
   230       return NS_OK;
   231     }
   232     // NO BREAK
   234   case DBOperation::opGetUsage:
   235     if (aOperation->Type() == DBOperation::opPreloadUrgent) {
   236       SetHigherPriority(); // Dropped back after urgent preload execution
   237       mPreloads.InsertElementAt(0, aOperation);
   238     } else {
   239       mPreloads.AppendElement(aOperation);
   240     }
   242     // DB operation adopted, don't delete it.
   243     opScope.forget();
   245     // Immediately start executing this.
   246     monitor.Notify();
   247     break;
   249   default:
   250     // Update operations are first collected, coalesced and then flushed
   251     // after a short time.
   252     mPendingTasks.Add(aOperation);
   254     // DB operation adopted, don't delete it.
   255     opScope.forget();
   257     ScheduleFlush();
   258     break;
   259   }
   261   return NS_OK;
   262 }
   264 void
   265 DOMStorageDBThread::SetHigherPriority()
   266 {
   267   ++mPriorityCounter;
   268   PR_SetThreadPriority(mThread, PR_PRIORITY_URGENT);
   269 }
   271 void
   272 DOMStorageDBThread::SetDefaultPriority()
   273 {
   274   if (--mPriorityCounter <= 0) {
   275     PR_SetThreadPriority(mThread, PR_PRIORITY_LOW);
   276   }
   277 }
   279 void
   280 DOMStorageDBThread::ThreadFunc(void* aArg)
   281 {
   282   PR_SetCurrentThreadName("localStorage DB");
   283   mozilla::IOInterposer::RegisterCurrentThread();
   285   DOMStorageDBThread* thread = static_cast<DOMStorageDBThread*>(aArg);
   286   thread->ThreadFunc();
   287   mozilla::IOInterposer::UnregisterCurrentThread();
   288 }
   290 void
   291 DOMStorageDBThread::ThreadFunc()
   292 {
   293   nsresult rv = InitDatabase();
   295   MonitorAutoLock lockMonitor(mMonitor);
   297   if (NS_FAILED(rv)) {
   298     mStatus = rv;
   299     mStopIOThread = true;
   300     return;
   301   }
   303   while (MOZ_LIKELY(!mStopIOThread || mPreloads.Length() || mPendingTasks.HasTasks())) {
   304     if (MOZ_UNLIKELY(TimeUntilFlush() == 0)) {
   305       // Flush time is up or flush has been forced, do it now.
   306       UnscheduleFlush();
   307       if (mPendingTasks.Prepare()) {
   308         {
   309           MonitorAutoUnlock unlockMonitor(mMonitor);
   310           rv = mPendingTasks.Execute(this);
   311         }
   313         if (!mPendingTasks.Finalize(rv)) {
   314           mStatus = rv;
   315           NS_WARNING("localStorage DB access broken");
   316         }
   317       }
   318       NotifyFlushCompletion();
   319     } else if (MOZ_LIKELY(mPreloads.Length())) {
   320       nsAutoPtr<DBOperation> op(mPreloads[0]);
   321       mPreloads.RemoveElementAt(0);
   322       {
   323         MonitorAutoUnlock unlockMonitor(mMonitor);
   324         op->PerformAndFinalize(this);
   325       }
   327       if (op->Type() == DBOperation::opPreloadUrgent) {
   328         SetDefaultPriority(); // urgent preload unscheduled
   329       }
   330     } else if (MOZ_UNLIKELY(!mStopIOThread)) {
   331       lockMonitor.Wait(TimeUntilFlush());
   332     }
   333   } // thread loop
   335   mStatus = ShutdownDatabase();
   336 }
   338 extern void
   339 ReverseString(const nsCSubstring& aSource, nsCSubstring& aResult);
   341 namespace { // anon
   343 class nsReverseStringSQLFunction MOZ_FINAL : public mozIStorageFunction
   344 {
   345   NS_DECL_ISUPPORTS
   346   NS_DECL_MOZISTORAGEFUNCTION
   347 };
   349 NS_IMPL_ISUPPORTS(nsReverseStringSQLFunction, mozIStorageFunction)
   351 NS_IMETHODIMP
   352 nsReverseStringSQLFunction::OnFunctionCall(
   353     mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult)
   354 {
   355   nsresult rv;
   357   nsAutoCString stringToReverse;
   358   rv = aFunctionArguments->GetUTF8String(0, stringToReverse);
   359   NS_ENSURE_SUCCESS(rv, rv);
   361   nsAutoCString result;
   362   ReverseString(stringToReverse, result);
   364   nsCOMPtr<nsIWritableVariant> outVar(do_CreateInstance(
   365       NS_VARIANT_CONTRACTID, &rv));
   366   NS_ENSURE_SUCCESS(rv, rv);
   368   rv = outVar->SetAsAUTF8String(result);
   369   NS_ENSURE_SUCCESS(rv, rv);
   371   *aResult = outVar.get();
   372   outVar.forget();
   373   return NS_OK;
   374 }
   376 } // anon
   378 nsresult
   379 DOMStorageDBThread::OpenDatabaseConnection()
   380 {
   381   nsresult rv;
   383   MOZ_ASSERT(!NS_IsMainThread());
   385   nsCOMPtr<mozIStorageService> service
   386       = do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
   387   NS_ENSURE_SUCCESS(rv, rv);
   389   nsCOMPtr<mozIStorageConnection> connection;
   390   rv = service->OpenUnsharedDatabase(mDatabaseFile, getter_AddRefs(mWorkerConnection));
   391   if (rv == NS_ERROR_FILE_CORRUPTED) {
   392     // delete the db and try opening again
   393     rv = mDatabaseFile->Remove(false);
   394     NS_ENSURE_SUCCESS(rv, rv);
   395     rv = service->OpenUnsharedDatabase(mDatabaseFile, getter_AddRefs(mWorkerConnection));
   396   }
   397   NS_ENSURE_SUCCESS(rv, rv);
   399   return NS_OK;
   400 }
   402 nsresult
   403 DOMStorageDBThread::InitDatabase()
   404 {
   405   Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_INIT_DATABASE_MS> timer;
   407   nsresult rv;
   409   // Here we are on the worker thread. This opens the worker connection.
   410   MOZ_ASSERT(!NS_IsMainThread());
   412   rv = OpenDatabaseConnection();
   413   NS_ENSURE_SUCCESS(rv, rv);
   415   rv = TryJournalMode();
   416   NS_ENSURE_SUCCESS(rv, rv);
   418   // Create a read-only clone
   419   (void)mWorkerConnection->Clone(true, getter_AddRefs(mReaderConnection));
   420   NS_ENSURE_TRUE(mReaderConnection, NS_ERROR_FAILURE);
   422   mozStorageTransaction transaction(mWorkerConnection, false);
   424   // Ensure Gecko 1.9.1 storage table
   425   rv = mWorkerConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
   426          "CREATE TABLE IF NOT EXISTS webappsstore2 ("
   427          "scope TEXT, "
   428          "key TEXT, "
   429          "value TEXT, "
   430          "secure INTEGER, "
   431          "owner TEXT)"));
   432   NS_ENSURE_SUCCESS(rv, rv);
   434   rv = mWorkerConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
   435         "CREATE UNIQUE INDEX IF NOT EXISTS scope_key_index"
   436         " ON webappsstore2(scope, key)"));
   437   NS_ENSURE_SUCCESS(rv, rv);
   439   nsCOMPtr<mozIStorageFunction> function1(new nsReverseStringSQLFunction());
   440   NS_ENSURE_TRUE(function1, NS_ERROR_OUT_OF_MEMORY);
   442   rv = mWorkerConnection->CreateFunction(NS_LITERAL_CSTRING("REVERSESTRING"), 1, function1);
   443   NS_ENSURE_SUCCESS(rv, rv);
   445   bool exists;
   447   // Check if there is storage of Gecko 1.9.0 and if so, upgrade that storage
   448   // to actual webappsstore2 table and drop the obsolete table. First process
   449   // this newer table upgrade to priority potential duplicates from older
   450   // storage table.
   451   rv = mWorkerConnection->TableExists(NS_LITERAL_CSTRING("webappsstore"),
   452                                 &exists);
   453   NS_ENSURE_SUCCESS(rv, rv);
   455   if (exists) {
   456     rv = mWorkerConnection->ExecuteSimpleSQL(
   457       NS_LITERAL_CSTRING("INSERT OR IGNORE INTO "
   458                          "webappsstore2(scope, key, value, secure, owner) "
   459                          "SELECT REVERSESTRING(domain) || '.:', key, value, secure, owner "
   460                          "FROM webappsstore"));
   461     NS_ENSURE_SUCCESS(rv, rv);
   463     rv = mWorkerConnection->ExecuteSimpleSQL(
   464       NS_LITERAL_CSTRING("DROP TABLE webappsstore"));
   465     NS_ENSURE_SUCCESS(rv, rv);
   466   }
   468   // Check if there is storage of Gecko 1.8 and if so, upgrade that storage
   469   // to actual webappsstore2 table and drop the obsolete table. Potential
   470   // duplicates will be ignored.
   471   rv = mWorkerConnection->TableExists(NS_LITERAL_CSTRING("moz_webappsstore"),
   472                                 &exists);
   473   NS_ENSURE_SUCCESS(rv, rv);
   475   if (exists) {
   476     rv = mWorkerConnection->ExecuteSimpleSQL(
   477       NS_LITERAL_CSTRING("INSERT OR IGNORE INTO "
   478                          "webappsstore2(scope, key, value, secure, owner) "
   479                          "SELECT REVERSESTRING(domain) || '.:', key, value, secure, domain "
   480                          "FROM moz_webappsstore"));
   481     NS_ENSURE_SUCCESS(rv, rv);
   483     rv = mWorkerConnection->ExecuteSimpleSQL(
   484       NS_LITERAL_CSTRING("DROP TABLE moz_webappsstore"));
   485     NS_ENSURE_SUCCESS(rv, rv);
   486   }
   488   rv = transaction.Commit();
   489   NS_ENSURE_SUCCESS(rv, rv);
   491   // Database open and all initiation operation are done.  Switching this flag
   492   // to true allow main thread to read directly from the database.
   493   // If we would allow this sooner, we would have opened a window where main thread
   494   // read might operate on a totaly broken and incosistent database.
   495   mDBReady = true;
   497   // List scopes having any stored data
   498   nsCOMPtr<mozIStorageStatement> stmt;
   499   rv = mWorkerConnection->CreateStatement(NS_LITERAL_CSTRING("SELECT DISTINCT scope FROM webappsstore2"),
   500                                     getter_AddRefs(stmt));
   501   NS_ENSURE_SUCCESS(rv, rv);
   502   mozStorageStatementScoper scope(stmt);
   504   while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
   505     nsAutoCString foundScope;
   506     rv = stmt->GetUTF8String(0, foundScope);
   507     NS_ENSURE_SUCCESS(rv, rv);
   509     MonitorAutoLock monitor(mMonitor);
   510     mScopesHavingData.PutEntry(foundScope);
   511   }
   513   return NS_OK;
   514 }
   516 nsresult
   517 DOMStorageDBThread::SetJournalMode(bool aIsWal)
   518 {
   519   nsresult rv;
   521   nsAutoCString stmtString(
   522     MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA journal_mode = ");
   523   if (aIsWal) {
   524     stmtString.AppendLiteral("wal");
   525   } else {
   526     stmtString.AppendLiteral("truncate");
   527   }
   529   nsCOMPtr<mozIStorageStatement> stmt;
   530   rv = mWorkerConnection->CreateStatement(stmtString, getter_AddRefs(stmt));
   531   NS_ENSURE_SUCCESS(rv, rv);
   532   mozStorageStatementScoper scope(stmt);
   534   bool hasResult = false;
   535   rv = stmt->ExecuteStep(&hasResult);
   536   NS_ENSURE_SUCCESS(rv, rv);
   537   if (!hasResult) {
   538     return NS_ERROR_FAILURE;
   539   }
   541   nsAutoCString journalMode;
   542   rv = stmt->GetUTF8String(0, journalMode);
   543   NS_ENSURE_SUCCESS(rv, rv);
   544   if ((aIsWal && !journalMode.EqualsLiteral("wal")) ||
   545       (!aIsWal && !journalMode.EqualsLiteral("truncate"))) {
   546     return NS_ERROR_FAILURE;
   547   }
   549   return NS_OK;
   550 }
   552 nsresult
   553 DOMStorageDBThread::TryJournalMode()
   554 {
   555   nsresult rv;
   557   rv = SetJournalMode(true);
   558   if (NS_FAILED(rv)) {
   559     mWALModeEnabled = false;
   561     rv = SetJournalMode(false);
   562     NS_ENSURE_SUCCESS(rv, rv);
   563   } else {
   564     mWALModeEnabled = true;
   566     rv = ConfigureWALBehavior();
   567     NS_ENSURE_SUCCESS(rv, rv);
   568   }
   570   return NS_OK;
   571 }
   573 nsresult
   574 DOMStorageDBThread::ConfigureWALBehavior()
   575 {
   576   // Get the DB's page size
   577   nsCOMPtr<mozIStorageStatement> stmt;
   578   nsresult rv = mWorkerConnection->CreateStatement(NS_LITERAL_CSTRING(
   579     MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"
   580   ), getter_AddRefs(stmt));
   581   NS_ENSURE_SUCCESS(rv, rv);
   583   bool hasResult = false;
   584   rv = stmt->ExecuteStep(&hasResult);
   585   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
   587   int32_t pageSize = 0;
   588   rv = stmt->GetInt32(0, &pageSize);
   589   NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && pageSize > 0, NS_ERROR_UNEXPECTED);
   591   // Set the threshold for auto-checkpointing the WAL.
   592   // We don't want giant logs slowing down reads & shutdown.
   593   int32_t thresholdInPages = static_cast<int32_t>(MAX_WAL_SIZE_BYTES / pageSize);
   594   nsAutoCString thresholdPragma("PRAGMA wal_autocheckpoint = ");
   595   thresholdPragma.AppendInt(thresholdInPages);
   596   rv = mWorkerConnection->ExecuteSimpleSQL(thresholdPragma);
   597   NS_ENSURE_SUCCESS(rv, rv);
   599   // Set the maximum WAL log size to reduce footprint on mobile (large empty
   600   // WAL files will be truncated)
   601   nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
   602   // bug 600307: mak recommends setting this to 3 times the auto-checkpoint threshold
   603   journalSizePragma.AppendInt(MAX_WAL_SIZE_BYTES * 3);
   604   rv = mWorkerConnection->ExecuteSimpleSQL(journalSizePragma);
   605   NS_ENSURE_SUCCESS(rv, rv);
   607   return NS_OK;
   608 }
   610 nsresult
   611 DOMStorageDBThread::ShutdownDatabase()
   612 {
   613   // Has to be called on the worker thread.
   614   MOZ_ASSERT(!NS_IsMainThread());
   616   nsresult rv = mStatus;
   618   mDBReady = false;
   620   // Finalize the cached statements.
   621   mReaderStatements.FinalizeStatements();
   622   mWorkerStatements.FinalizeStatements();
   624   if (mReaderConnection) {
   625     // No need to sync access to mReaderConnection since the main thread
   626     // is right now joining this thread, unable to execute any events.
   627     mReaderConnection->Close();
   628     mReaderConnection = nullptr;
   629   }
   631   if (mWorkerConnection) {
   632     rv = mWorkerConnection->Close();
   633     mWorkerConnection = nullptr;
   634   }
   636   return rv;
   637 }
   639 void
   640 DOMStorageDBThread::ScheduleFlush()
   641 {
   642   if (mDirtyEpoch) {
   643     return; // Already scheduled
   644   }
   646   mDirtyEpoch = PR_IntervalNow() | 1; // Must be non-zero to indicate we are scheduled
   648   // Wake the monitor from indefinite sleep...
   649   mMonitor.Notify();
   650 }
   652 void
   653 DOMStorageDBThread::UnscheduleFlush()
   654 {
   655   // We are just about to do the flush, drop flags
   656   mFlushImmediately = false;
   657   mDirtyEpoch = 0;
   658 }
   660 PRIntervalTime
   661 DOMStorageDBThread::TimeUntilFlush()
   662 {
   663   if (mFlushImmediately) {
   664     return 0; // Do it now regardless the timeout.
   665   }
   667   static_assert(PR_INTERVAL_NO_TIMEOUT != 0,
   668       "PR_INTERVAL_NO_TIMEOUT must be non-zero");
   670   if (!mDirtyEpoch) {
   671     return PR_INTERVAL_NO_TIMEOUT; // No pending task...
   672   }
   674   static const PRIntervalTime kMaxAge = PR_MillisecondsToInterval(FLUSHING_INTERVAL_MS);
   676   PRIntervalTime now = PR_IntervalNow() | 1;
   677   PRIntervalTime age = now - mDirtyEpoch;
   678   if (age > kMaxAge) {
   679     return 0; // It is time.
   680   }
   682   return kMaxAge - age; // Time left, this is used to sleep the monitor
   683 }
   685 void
   686 DOMStorageDBThread::NotifyFlushCompletion()
   687 {
   688 #ifdef DOM_STORAGE_TESTS
   689   if (!NS_IsMainThread()) {
   690     nsRefPtr<nsRunnableMethod<DOMStorageDBThread, void, false> > event =
   691       NS_NewNonOwningRunnableMethod(this, &DOMStorageDBThread::NotifyFlushCompletion);
   692     NS_DispatchToMainThread(event);
   693     return;
   694   }
   696   nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
   697   if (obs) {
   698     obs->NotifyObservers(nullptr, "domstorage-test-flushed", nullptr);
   699   }
   700 #endif
   701 }
   703 // DOMStorageDBThread::DBOperation
   705 DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
   706                                              DOMStorageCacheBridge* aCache,
   707                                              const nsAString& aKey,
   708                                              const nsAString& aValue)
   709 : mType(aType)
   710 , mCache(aCache)
   711 , mKey(aKey)
   712 , mValue(aValue)
   713 {
   714   MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
   715 }
   717 DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
   718                                              DOMStorageUsageBridge* aUsage)
   719 : mType(aType)
   720 , mUsage(aUsage)
   721 {
   722   MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
   723 }
   725 DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
   726                                              const nsACString& aScope)
   727 : mType(aType)
   728 , mCache(nullptr)
   729 , mScope(aScope)
   730 {
   731   MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
   732 }
   734 DOMStorageDBThread::DBOperation::~DBOperation()
   735 {
   736   MOZ_COUNT_DTOR(DOMStorageDBThread::DBOperation);
   737 }
   739 const nsCString
   740 DOMStorageDBThread::DBOperation::Scope()
   741 {
   742   if (mCache) {
   743     return mCache->Scope();
   744   }
   746   return mScope;
   747 }
   749 const nsCString
   750 DOMStorageDBThread::DBOperation::Target()
   751 {
   752   switch (mType) {
   753     case opAddItem:
   754     case opUpdateItem:
   755     case opRemoveItem:
   756       return Scope() + NS_LITERAL_CSTRING("|") + NS_ConvertUTF16toUTF8(mKey);
   758     default:
   759       return Scope();
   760   }
   761 }
   763 void
   764 DOMStorageDBThread::DBOperation::PerformAndFinalize(DOMStorageDBThread* aThread)
   765 {
   766   Finalize(Perform(aThread));
   767 }
   769 nsresult
   770 DOMStorageDBThread::DBOperation::Perform(DOMStorageDBThread* aThread)
   771 {
   772   nsresult rv;
   774   switch (mType) {
   775   case opPreload:
   776   case opPreloadUrgent:
   777   {
   778     // Already loaded?
   779     if (mCache->Loaded()) {
   780       break;
   781     }
   783     StatementCache* statements;
   784     if (MOZ_UNLIKELY(NS_IsMainThread())) {
   785       statements = &aThread->mReaderStatements;
   786     } else {
   787       statements = &aThread->mWorkerStatements;
   788     }
   790     // OFFSET is an optimization when we have to do a sync load
   791     // and cache has already loaded some parts asynchronously.
   792     // It skips keys we have already loaded.
   793     nsCOMPtr<mozIStorageStatement> stmt = statements->GetCachedStatement(
   794         "SELECT key, value FROM webappsstore2 "
   795         "WHERE scope = :scope ORDER BY key "
   796         "LIMIT -1 OFFSET :offset");
   797     NS_ENSURE_STATE(stmt);
   798     mozStorageStatementScoper scope(stmt);
   800     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
   801                                     mCache->Scope());
   802     NS_ENSURE_SUCCESS(rv, rv);
   804     rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("offset"),
   805                                static_cast<int32_t>(mCache->LoadedCount()));
   806     NS_ENSURE_SUCCESS(rv, rv);
   808     bool exists;
   809     while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
   810       nsAutoString key;
   811       rv = stmt->GetString(0, key);
   812       NS_ENSURE_SUCCESS(rv, rv);
   814       nsAutoString value;
   815       rv = stmt->GetString(1, value);
   816       NS_ENSURE_SUCCESS(rv, rv);
   818       if (!mCache->LoadItem(key, value)) {
   819         break;
   820       }
   821     }
   823     mCache->LoadDone(NS_OK);
   824     break;
   825   }
   827   case opGetUsage:
   828   {
   829     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
   830       "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2"
   831       " WHERE scope LIKE :scope"
   832     );
   833     NS_ENSURE_STATE(stmt);
   835     mozStorageStatementScoper scope(stmt);
   837     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
   838                                     mUsage->Scope() + NS_LITERAL_CSTRING("%"));
   839     NS_ENSURE_SUCCESS(rv, rv);
   841     bool exists;
   842     rv = stmt->ExecuteStep(&exists);
   843     NS_ENSURE_SUCCESS(rv, rv);
   845     int64_t usage = 0;
   846     if (exists) {
   847       rv = stmt->GetInt64(0, &usage);
   848       NS_ENSURE_SUCCESS(rv, rv);
   849     }
   851     mUsage->LoadUsage(usage);
   852     break;
   853   }
   855   case opAddItem:
   856   case opUpdateItem:
   857   {
   858     MOZ_ASSERT(!NS_IsMainThread());
   860     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
   861       "INSERT OR REPLACE INTO webappsstore2 (scope, key, value) "
   862       "VALUES (:scope, :key, :value) "
   863     );
   864     NS_ENSURE_STATE(stmt);
   866     mozStorageStatementScoper scope(stmt);
   868     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
   869                                     mCache->Scope());
   870     NS_ENSURE_SUCCESS(rv, rv);
   871     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
   872                                 mKey);
   873     NS_ENSURE_SUCCESS(rv, rv);
   874     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"),
   875                                 mValue);
   876     NS_ENSURE_SUCCESS(rv, rv);
   878     rv = stmt->Execute();
   879     NS_ENSURE_SUCCESS(rv, rv);
   881     aThread->mScopesHavingData.PutEntry(Scope());
   882     break;
   883   }
   885   case opRemoveItem:
   886   {
   887     MOZ_ASSERT(!NS_IsMainThread());
   889     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
   890       "DELETE FROM webappsstore2 "
   891       "WHERE scope = :scope "
   892         "AND key = :key "
   893     );
   894     NS_ENSURE_STATE(stmt);
   895     mozStorageStatementScoper scope(stmt);
   897     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
   898                                     mCache->Scope());
   899     NS_ENSURE_SUCCESS(rv, rv);
   900     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
   901                                 mKey);
   902     NS_ENSURE_SUCCESS(rv, rv);
   904     rv = stmt->Execute();
   905     NS_ENSURE_SUCCESS(rv, rv);
   907     break;
   908   }
   910   case opClear:
   911   {
   912     MOZ_ASSERT(!NS_IsMainThread());
   914     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
   915       "DELETE FROM webappsstore2 "
   916       "WHERE scope = :scope"
   917     );
   918     NS_ENSURE_STATE(stmt);
   919     mozStorageStatementScoper scope(stmt);
   921     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
   922                                     mCache->Scope());
   923     NS_ENSURE_SUCCESS(rv, rv);
   925     rv = stmt->Execute();
   926     NS_ENSURE_SUCCESS(rv, rv);
   928     aThread->mScopesHavingData.RemoveEntry(Scope());
   929     break;
   930   }
   932   case opClearAll:
   933   {
   934     MOZ_ASSERT(!NS_IsMainThread());
   936     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
   937       "DELETE FROM webappsstore2"
   938     );
   939     NS_ENSURE_STATE(stmt);
   940     mozStorageStatementScoper scope(stmt);
   942     rv = stmt->Execute();
   943     NS_ENSURE_SUCCESS(rv, rv);
   945     aThread->mScopesHavingData.Clear();
   946     break;
   947   }
   949   case opClearMatchingScope:
   950   {
   951     MOZ_ASSERT(!NS_IsMainThread());
   953     nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
   954       "DELETE FROM webappsstore2"
   955       " WHERE scope GLOB :scope"
   956     );
   957     NS_ENSURE_STATE(stmt);
   958     mozStorageStatementScoper scope(stmt);
   960     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
   961                                     mScope + NS_LITERAL_CSTRING("*"));
   962     NS_ENSURE_SUCCESS(rv, rv);
   964     rv = stmt->Execute();
   965     NS_ENSURE_SUCCESS(rv, rv);
   967     break;
   968   }
   970   default:
   971     NS_ERROR("Unknown task type");
   972     break;
   973   }
   975   return NS_OK;
   976 }
   978 void
   979 DOMStorageDBThread::DBOperation::Finalize(nsresult aRv)
   980 {
   981   switch (mType) {
   982   case opPreloadUrgent:
   983   case opPreload:
   984     if (NS_FAILED(aRv)) {
   985       // When we are here, something failed when loading from the database.
   986       // Notify that the storage is loaded to prevent deadlock of the main thread,
   987       // even though it is actually empty or incomplete.
   988       NS_WARNING("Failed to preload localStorage");
   989     }
   991     mCache->LoadDone(aRv);
   992     break;
   994   case opGetUsage:
   995     if (NS_FAILED(aRv)) {
   996       mUsage->LoadUsage(0);
   997     }
   999     break;
  1001   default:
  1002     if (NS_FAILED(aRv)) {
  1003       NS_WARNING("localStorage update/clear operation failed,"
  1004                  " data may not persist or clean up");
  1007     break;
  1011 // DOMStorageDBThread::PendingOperations
  1013 DOMStorageDBThread::PendingOperations::PendingOperations()
  1014 : mFlushFailureCount(0)
  1018 bool
  1019 DOMStorageDBThread::PendingOperations::HasTasks()
  1021   return !!mUpdates.Count() || !!mClears.Count();
  1024 namespace { // anon
  1026 PLDHashOperator
  1027 ForgetUpdatesForScope(const nsACString& aMapping,
  1028                       nsAutoPtr<DOMStorageDBThread::DBOperation>& aPendingTask,
  1029                       void* aArg)
  1031   DOMStorageDBThread::DBOperation* newOp = static_cast<DOMStorageDBThread::DBOperation*>(aArg);
  1033   if (newOp->Type() == DOMStorageDBThread::DBOperation::opClear &&
  1034       aPendingTask->Scope() != newOp->Scope()) {
  1035     return PL_DHASH_NEXT;
  1038   if (newOp->Type() == DOMStorageDBThread::DBOperation::opClearMatchingScope &&
  1039       !StringBeginsWith(aPendingTask->Scope(), newOp->Scope())) {
  1040     return PL_DHASH_NEXT;
  1043   return PL_DHASH_REMOVE;
  1046 } // anon
  1048 bool
  1049 DOMStorageDBThread::PendingOperations::CheckForCoalesceOpportunity(DBOperation* aNewOp,
  1050                                                                    DBOperation::OperationType aPendingType,
  1051                                                                    DBOperation::OperationType aNewType)
  1053   if (aNewOp->Type() != aNewType) {
  1054     return false;
  1057   DOMStorageDBThread::DBOperation* pendingTask;
  1058   if (!mUpdates.Get(aNewOp->Target(), &pendingTask)) {
  1059     return false;
  1062   if (pendingTask->Type() != aPendingType) {
  1063     return false;
  1066   return true;
  1069 void
  1070 DOMStorageDBThread::PendingOperations::Add(DOMStorageDBThread::DBOperation* aOperation)
  1072   // Optimize: when a key to remove has never been written to disk
  1073   // just bypass this operation.  A kew is new when an operation scheduled
  1074   // to write it to the database is of type opAddItem.
  1075   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem, DBOperation::opRemoveItem)) {
  1076     mUpdates.Remove(aOperation->Target());
  1077     delete aOperation;
  1078     return;
  1081   // Optimize: when changing a key that is new and has never been
  1082   // written to disk, keep type of the operation to store it at opAddItem.
  1083   // This allows optimization to just forget adding a new key when
  1084   // it is removed from the storage before flush.
  1085   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem, DBOperation::opUpdateItem)) {
  1086     aOperation->mType = DBOperation::opAddItem;
  1089   // Optimize: to prevent lose of remove operation on a key when doing
  1090   // remove/set/remove on a previously existing key we have to change
  1091   // opAddItem to opUpdateItem on the new operation when there is opRemoveItem
  1092   // pending for the key.
  1093   if (CheckForCoalesceOpportunity(aOperation, DBOperation::opRemoveItem, DBOperation::opAddItem)) {
  1094     aOperation->mType = DBOperation::opUpdateItem;
  1097   switch (aOperation->Type())
  1099   // Operations on single keys
  1101   case DBOperation::opAddItem:
  1102   case DBOperation::opUpdateItem:
  1103   case DBOperation::opRemoveItem:
  1104     // Override any existing operation for the target (=scope+key).
  1105     mUpdates.Put(aOperation->Target(), aOperation);
  1106     break;
  1108   // Clear operations
  1110   case DBOperation::opClear:
  1111   case DBOperation::opClearMatchingScope:
  1112     // Drop all update (insert/remove) operations for equivavelent or matching scope.
  1113     // We do this as an optimization as well as a must based on the logic,
  1114     // if we would not delete the update tasks, changes would have been stored
  1115     // to the database after clear operations have been executed.
  1116     mUpdates.Enumerate(ForgetUpdatesForScope, aOperation);
  1117     mClears.Put(aOperation->Target(), aOperation);
  1118     break;
  1120   case DBOperation::opClearAll:
  1121     // Drop simply everything, this is a super-operation.
  1122     mUpdates.Clear();
  1123     mClears.Clear();
  1124     mClears.Put(aOperation->Target(), aOperation);
  1125     break;
  1127   default:
  1128     MOZ_ASSERT(false);
  1129     break;
  1133 namespace { // anon
  1135 PLDHashOperator
  1136 CollectTasks(const nsACString& aMapping, nsAutoPtr<DOMStorageDBThread::DBOperation>& aOperation, void* aArg)
  1138   nsTArray<nsAutoPtr<DOMStorageDBThread::DBOperation> >* tasks =
  1139     static_cast<nsTArray<nsAutoPtr<DOMStorageDBThread::DBOperation> >*>(aArg);
  1141   tasks->AppendElement(aOperation.forget());
  1142   return PL_DHASH_NEXT;
  1145 } // anon
  1147 bool
  1148 DOMStorageDBThread::PendingOperations::Prepare()
  1150   // Called under the lock
  1152   // First collect clear operations and then updates, we can
  1153   // do this since whenever a clear operation for a scope is
  1154   // scheduled, we drop all updates matching that scope. So,
  1155   // all scope-related update operations we have here now were
  1156   // scheduled after the clear operations.
  1157   mClears.Enumerate(CollectTasks, &mExecList);
  1158   mClears.Clear();
  1160   mUpdates.Enumerate(CollectTasks, &mExecList);
  1161   mUpdates.Clear();
  1163   return !!mExecList.Length();
  1166 nsresult
  1167 DOMStorageDBThread::PendingOperations::Execute(DOMStorageDBThread* aThread)
  1169   // Called outside the lock
  1171   mozStorageTransaction transaction(aThread->mWorkerConnection, false);
  1173   nsresult rv;
  1175   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
  1176     DOMStorageDBThread::DBOperation* task = mExecList[i];
  1177     rv = task->Perform(aThread);
  1178     if (NS_FAILED(rv)) {
  1179       return rv;
  1183   rv = transaction.Commit();
  1184   if (NS_FAILED(rv)) {
  1185     return rv;
  1188   return NS_OK;
  1191 bool
  1192 DOMStorageDBThread::PendingOperations::Finalize(nsresult aRv)
  1194   // Called under the lock
  1196   // The list is kept on a failure to retry it
  1197   if (NS_FAILED(aRv)) {
  1198     // XXX Followup: we may try to reopen the database and flush these
  1199     // pending tasks, however testing showed that even though I/O is actually
  1200     // broken some amount of operations is left in sqlite+system buffers and
  1201     // seems like successfully flushed to disk.
  1202     // Tested by removing a flash card and disconnecting from network while
  1203     // using a network drive on Windows system.
  1204     NS_WARNING("Flush operation on localStorage database failed");
  1206     ++mFlushFailureCount;
  1208     return mFlushFailureCount >= 5;
  1211   mFlushFailureCount = 0;
  1212   mExecList.Clear();
  1213   return true;
  1216 namespace { // anon
  1218 class FindPendingOperationForScopeData
  1220 public:
  1221   FindPendingOperationForScopeData(const nsACString& aScope) : mScope(aScope), mFound(false) {}
  1222   nsCString mScope;
  1223   bool mFound;
  1224 };
  1226 PLDHashOperator
  1227 FindPendingClearForScope(const nsACString& aMapping,
  1228                          DOMStorageDBThread::DBOperation* aPendingOperation,
  1229                          void* aArg)
  1231   FindPendingOperationForScopeData* data =
  1232     static_cast<FindPendingOperationForScopeData*>(aArg);
  1234   if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClearAll) {
  1235     data->mFound = true;
  1236     return PL_DHASH_STOP;
  1239   if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClear &&
  1240       data->mScope == aPendingOperation->Scope()) {
  1241     data->mFound = true;
  1242     return PL_DHASH_STOP;
  1245   if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClearMatchingScope &&
  1246       StringBeginsWith(data->mScope, aPendingOperation->Scope())) {
  1247     data->mFound = true;
  1248     return PL_DHASH_STOP;
  1251   return PL_DHASH_NEXT;
  1254 } // anon
  1256 bool
  1257 DOMStorageDBThread::PendingOperations::IsScopeClearPending(const nsACString& aScope)
  1259   // Called under the lock
  1261   FindPendingOperationForScopeData data(aScope);
  1262   mClears.EnumerateRead(FindPendingClearForScope, &data);
  1263   if (data.mFound) {
  1264     return true;
  1267   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
  1268     DOMStorageDBThread::DBOperation* task = mExecList[i];
  1269     FindPendingClearForScope(EmptyCString(), task, &data);
  1271     if (data.mFound) {
  1272       return true;
  1276   return false;
  1279 namespace { // anon
  1281 PLDHashOperator
  1282 FindPendingUpdateForScope(const nsACString& aMapping,
  1283                           DOMStorageDBThread::DBOperation* aPendingOperation,
  1284                           void* aArg)
  1286   FindPendingOperationForScopeData* data =
  1287     static_cast<FindPendingOperationForScopeData*>(aArg);
  1289   if ((aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opAddItem ||
  1290        aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opUpdateItem ||
  1291        aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opRemoveItem) &&
  1292        data->mScope == aPendingOperation->Scope()) {
  1293     data->mFound = true;
  1294     return PL_DHASH_STOP;
  1297   return PL_DHASH_NEXT;
  1300 } // anon
  1302 bool
  1303 DOMStorageDBThread::PendingOperations::IsScopeUpdatePending(const nsACString& aScope)
  1305   // Called under the lock
  1307   FindPendingOperationForScopeData data(aScope);
  1308   mUpdates.EnumerateRead(FindPendingUpdateForScope, &data);
  1309   if (data.mFound) {
  1310     return true;
  1313   for (uint32_t i = 0; i < mExecList.Length(); ++i) {
  1314     DOMStorageDBThread::DBOperation* task = mExecList[i];
  1315     FindPendingUpdateForScope(EmptyCString(), task, &data);
  1317     if (data.mFound) {
  1318       return true;
  1322   return false;
  1325 } // ::dom
  1326 } // ::mozilla

mercurial