michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
michael@0:  * vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: #include "mozilla/Attributes.h"
michael@0: #include "mozilla/DebugOnly.h"
michael@0: 
michael@0: #include "mozStorageService.h"
michael@0: #include "mozStorageConnection.h"
michael@0: #include "prinit.h"
michael@0: #include "nsAutoPtr.h"
michael@0: #include "nsCollationCID.h"
michael@0: #include "nsEmbedCID.h"
michael@0: #include "nsThreadUtils.h"
michael@0: #include "mozStoragePrivateHelpers.h"
michael@0: #include "nsILocale.h"
michael@0: #include "nsILocaleService.h"
michael@0: #include "nsIXPConnect.h"
michael@0: #include "nsIObserverService.h"
michael@0: #include "nsIPropertyBag2.h"
michael@0: #include "mozilla/Services.h"
michael@0: #include "mozilla/Preferences.h"
michael@0: #include "mozilla/LateWriteChecks.h"
michael@0: #include "mozIStorageCompletionCallback.h"
michael@0: #include "mozIStoragePendingStatement.h"
michael@0: 
michael@0: #include "sqlite3.h"
michael@0: 
michael@0: #ifdef SQLITE_OS_WIN
michael@0: // "windows.h" was included and it can #define lots of things we care about...
michael@0: #undef CompareString
michael@0: #endif
michael@0: 
michael@0: #include "nsIPromptService.h"
michael@0: 
michael@0: #ifdef MOZ_STORAGE_MEMORY
michael@0: #  include "mozmemory.h"
michael@0: #  ifdef MOZ_DMD
michael@0: #    include "DMD.h"
michael@0: #  endif
michael@0: #endif
michael@0: 
michael@0: ////////////////////////////////////////////////////////////////////////////////
michael@0: //// Defines
michael@0: 
michael@0: #define PREF_TS_SYNCHRONOUS "toolkit.storage.synchronous"
michael@0: #define PREF_TS_SYNCHRONOUS_DEFAULT 1
michael@0: 
michael@0: #define PREF_TS_PAGESIZE "toolkit.storage.pageSize"
michael@0: 
michael@0: // This value must be kept in sync with the value of SQLITE_DEFAULT_PAGE_SIZE in
michael@0: // db/sqlite3/src/Makefile.in.
michael@0: #define PREF_TS_PAGESIZE_DEFAULT 32768
michael@0: 
michael@0: namespace mozilla {
michael@0: namespace storage {
michael@0: 
michael@0: ////////////////////////////////////////////////////////////////////////////////
michael@0: //// Memory Reporting
michael@0: 
michael@0: #ifdef MOZ_DMD
michael@0: static mozilla::Atomic<size_t> gSqliteMemoryUsed;
michael@0: #endif
michael@0: 
michael@0: static int64_t
michael@0: StorageSQLiteDistinguishedAmount()
michael@0: {
michael@0:   return ::sqlite3_memory_used();
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Passes a single SQLite memory statistic to a memory reporter callback.
michael@0:  *
michael@0:  * @param aHandleReport
michael@0:  *        The callback.
michael@0:  * @param aData
michael@0:  *        The data for the callback.
michael@0:  * @param aConn
michael@0:  *        The SQLite connection.
michael@0:  * @param aPathHead
michael@0:  *        Head of the path for the memory report.
michael@0:  * @param aKind
michael@0:  *        The memory report statistic kind, one of "stmt", "cache" or
michael@0:  *        "schema".
michael@0:  * @param aDesc
michael@0:  *        The memory report description.
michael@0:  * @param aOption
michael@0:  *        The SQLite constant for getting the measurement.
michael@0:  * @param aTotal
michael@0:  *        The accumulator for the measurement.
michael@0:  */
michael@0: nsresult
michael@0: ReportConn(nsIHandleReportCallback *aHandleReport,
michael@0:            nsISupports *aData,
michael@0:            Connection *aConn,
michael@0:            const nsACString &aPathHead,
michael@0:            const nsACString &aKind,
michael@0:            const nsACString &aDesc,
michael@0:            int32_t aOption,
michael@0:            size_t *aTotal)
michael@0: {
michael@0:   nsCString path(aPathHead);
michael@0:   path.Append(aKind);
michael@0:   path.AppendLiteral("-used");
michael@0: 
michael@0:   int32_t val = aConn->getSqliteRuntimeStatus(aOption);
michael@0:   nsresult rv = aHandleReport->Callback(EmptyCString(), path,
michael@0:                                         nsIMemoryReporter::KIND_HEAP,
michael@0:                                         nsIMemoryReporter::UNITS_BYTES,
michael@0:                                         int64_t(val), aDesc, aData);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0:   *aTotal += val;
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: // Warning: To get a Connection's measurements requires holding its lock.
michael@0: // There may be a delay getting the lock if another thread is accessing the
michael@0: // Connection.  This isn't very nice if CollectReports is called from the main
michael@0: // thread!  But at the time of writing this function is only called when
michael@0: // about:memory is loaded (not, for example, when telemetry pings occur) and
michael@0: // any delays in that case aren't so bad.
michael@0: NS_IMETHODIMP
michael@0: Service::CollectReports(nsIHandleReportCallback *aHandleReport,
michael@0:                         nsISupports *aData)
michael@0: {
michael@0:   nsresult rv;
michael@0:   size_t totalConnSize = 0;
michael@0:   {
michael@0:     nsTArray<nsRefPtr<Connection> > connections;
michael@0:     getConnections(connections);
michael@0: 
michael@0:     for (uint32_t i = 0; i < connections.Length(); i++) {
michael@0:       nsRefPtr<Connection> &conn = connections[i];
michael@0: 
michael@0:       // Someone may have closed the Connection, in which case we skip it.
michael@0:       bool isReady;
michael@0:       (void)conn->GetConnectionReady(&isReady);
michael@0:       if (!isReady) {
michael@0:           continue;
michael@0:       }
michael@0: 
michael@0:       nsCString pathHead("explicit/storage/sqlite/");
michael@0:       pathHead.Append(conn->getFilename());
michael@0:       pathHead.AppendLiteral("/");
michael@0: 
michael@0:       SQLiteMutexAutoLock lockedScope(conn->sharedDBMutex);
michael@0: 
michael@0:       NS_NAMED_LITERAL_CSTRING(stmtDesc,
michael@0:         "Memory (approximate) used by all prepared statements used by "
michael@0:         "connections to this database.");
michael@0:       rv = ReportConn(aHandleReport, aData, conn, pathHead,
michael@0:                       NS_LITERAL_CSTRING("stmt"), stmtDesc,
michael@0:                       SQLITE_DBSTATUS_STMT_USED, &totalConnSize);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:       NS_NAMED_LITERAL_CSTRING(cacheDesc,
michael@0:         "Memory (approximate) used by all pager caches used by connections "
michael@0:         "to this database.");
michael@0:       rv = ReportConn(aHandleReport, aData, conn, pathHead,
michael@0:                       NS_LITERAL_CSTRING("cache"), cacheDesc,
michael@0:                       SQLITE_DBSTATUS_CACHE_USED, &totalConnSize);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:       NS_NAMED_LITERAL_CSTRING(schemaDesc,
michael@0:         "Memory (approximate) used to store the schema for all databases "
michael@0:         "associated with connections to this database.");
michael@0:       rv = ReportConn(aHandleReport, aData, conn, pathHead,
michael@0:                       NS_LITERAL_CSTRING("schema"), schemaDesc,
michael@0:                       SQLITE_DBSTATUS_SCHEMA_USED, &totalConnSize);
michael@0:       NS_ENSURE_SUCCESS(rv, rv);
michael@0:     }
michael@0: 
michael@0: #ifdef MOZ_DMD
michael@0:     if (::sqlite3_memory_used() != int64_t(gSqliteMemoryUsed)) {
michael@0:       NS_WARNING("memory consumption reported by SQLite doesn't match "
michael@0:                  "our measurements");
michael@0:     }
michael@0: #endif
michael@0:   }
michael@0: 
michael@0:   int64_t other = ::sqlite3_memory_used() - totalConnSize;
michael@0: 
michael@0:   rv = aHandleReport->Callback(
michael@0:           EmptyCString(),
michael@0:           NS_LITERAL_CSTRING("explicit/storage/sqlite/other"),
michael@0:           KIND_HEAP, UNITS_BYTES, other,
michael@0:           NS_LITERAL_CSTRING("All unclassified sqlite memory."),
michael@0:           aData);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: ////////////////////////////////////////////////////////////////////////////////
michael@0: //// Service
michael@0: 
michael@0: NS_IMPL_ISUPPORTS(
michael@0:   Service,
michael@0:   mozIStorageService,
michael@0:   nsIObserver,
michael@0:   nsIMemoryReporter
michael@0: )
michael@0: 
michael@0: Service *Service::gService = nullptr;
michael@0: 
michael@0: Service *
michael@0: Service::getSingleton()
michael@0: {
michael@0:   if (gService) {
michael@0:     NS_ADDREF(gService);
michael@0:     return gService;
michael@0:   }
michael@0: 
michael@0:   // Ensure that we are using the same version of SQLite that we compiled with
michael@0:   // or newer.  Our configure check ensures we are using a new enough version
michael@0:   // at compile time.
michael@0:   if (SQLITE_VERSION_NUMBER > ::sqlite3_libversion_number()) {
michael@0:     nsCOMPtr<nsIPromptService> ps(do_GetService(NS_PROMPTSERVICE_CONTRACTID));
michael@0:     if (ps) {
michael@0:       nsAutoString title, message;
michael@0:       title.AppendLiteral("SQLite Version Error");
michael@0:       message.AppendLiteral("The application has been updated, but your version "
michael@0:                           "of SQLite is too old and the application cannot "
michael@0:                           "run.");
michael@0:       (void)ps->Alert(nullptr, title.get(), message.get());
michael@0:     }
michael@0:     ::PR_Abort();
michael@0:   }
michael@0: 
michael@0:   // The first reference to the storage service must be obtained on the
michael@0:   // main thread.
michael@0:   NS_ENSURE_TRUE(NS_IsMainThread(), nullptr);
michael@0:   gService = new Service();
michael@0:   if (gService) {
michael@0:     NS_ADDREF(gService);
michael@0:     if (NS_FAILED(gService->initialize()))
michael@0:       NS_RELEASE(gService);
michael@0:   }
michael@0: 
michael@0:   return gService;
michael@0: }
michael@0: 
michael@0: nsIXPConnect *Service::sXPConnect = nullptr;
michael@0: 
michael@0: // static
michael@0: already_AddRefed<nsIXPConnect>
michael@0: Service::getXPConnect()
michael@0: {
michael@0:   NS_PRECONDITION(NS_IsMainThread(),
michael@0:                   "Must only get XPConnect on the main thread!");
michael@0:   NS_PRECONDITION(gService,
michael@0:                   "Can not get XPConnect without an instance of our service!");
michael@0: 
michael@0:   // If we've been shutdown, sXPConnect will be null.  To prevent leaks, we do
michael@0:   // not cache the service after this point.
michael@0:   nsCOMPtr<nsIXPConnect> xpc(sXPConnect);
michael@0:   if (!xpc)
michael@0:     xpc = do_GetService(nsIXPConnect::GetCID());
michael@0:   NS_ASSERTION(xpc, "Could not get XPConnect!");
michael@0:   return xpc.forget();
michael@0: }
michael@0: 
michael@0: int32_t Service::sSynchronousPref;
michael@0: 
michael@0: // static
michael@0: int32_t
michael@0: Service::getSynchronousPref()
michael@0: {
michael@0:   return sSynchronousPref;
michael@0: }
michael@0: 
michael@0: int32_t Service::sDefaultPageSize = PREF_TS_PAGESIZE_DEFAULT;
michael@0: 
michael@0: Service::Service()
michael@0: : mMutex("Service::mMutex")
michael@0: , mSqliteVFS(nullptr)
michael@0: , mRegistrationMutex("Service::mRegistrationMutex")
michael@0: , mConnections()
michael@0: {
michael@0: }
michael@0: 
michael@0: Service::~Service()
michael@0: {
michael@0:   mozilla::UnregisterWeakMemoryReporter(this);
michael@0:   mozilla::UnregisterStorageSQLiteDistinguishedAmount();
michael@0: 
michael@0:   int rc = sqlite3_vfs_unregister(mSqliteVFS);
michael@0:   if (rc != SQLITE_OK)
michael@0:     NS_WARNING("Failed to unregister sqlite vfs wrapper.");
michael@0: 
michael@0:   // Shutdown the sqlite3 API.  Warn if shutdown did not turn out okay, but
michael@0:   // there is nothing actionable we can do in that case.
michael@0:   rc = ::sqlite3_shutdown();
michael@0:   if (rc != SQLITE_OK)
michael@0:     NS_WARNING("sqlite3 did not shutdown cleanly.");
michael@0: 
michael@0:   DebugOnly<bool> shutdownObserved = !sXPConnect;
michael@0:   NS_ASSERTION(shutdownObserved, "Shutdown was not observed!");
michael@0: 
michael@0:   gService = nullptr;
michael@0:   delete mSqliteVFS;
michael@0:   mSqliteVFS = nullptr;
michael@0: }
michael@0: 
michael@0: void
michael@0: Service::registerConnection(Connection *aConnection)
michael@0: {
michael@0:   mRegistrationMutex.AssertNotCurrentThreadOwns();
michael@0:   MutexAutoLock mutex(mRegistrationMutex);
michael@0:   (void)mConnections.AppendElement(aConnection);
michael@0: }
michael@0: 
michael@0: void
michael@0: Service::unregisterConnection(Connection *aConnection)
michael@0: {
michael@0:   // If this is the last Connection it might be the only thing keeping Service
michael@0:   // alive.  So ensure that Service is destroyed only after the Connection is
michael@0:   // cleanly unregistered and destroyed.
michael@0:   nsRefPtr<Service> kungFuDeathGrip(this);
michael@0:   {
michael@0:     mRegistrationMutex.AssertNotCurrentThreadOwns();
michael@0:     MutexAutoLock mutex(mRegistrationMutex);
michael@0:     DebugOnly<bool> removed = mConnections.RemoveElement(aConnection);
michael@0:     // Assert if we try to unregister a non-existent connection.
michael@0:     MOZ_ASSERT(removed);
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: Service::getConnections(/* inout */ nsTArray<nsRefPtr<Connection> >& aConnections)
michael@0: {
michael@0:   mRegistrationMutex.AssertNotCurrentThreadOwns();
michael@0:   MutexAutoLock mutex(mRegistrationMutex);
michael@0:   aConnections.Clear();
michael@0:   aConnections.AppendElements(mConnections);
michael@0: }
michael@0: 
michael@0: void
michael@0: Service::minimizeMemory()
michael@0: {
michael@0:   nsTArray<nsRefPtr<Connection> > connections;
michael@0:   getConnections(connections);
michael@0: 
michael@0:   for (uint32_t i = 0; i < connections.Length(); i++) {
michael@0:     nsRefPtr<Connection> conn = connections[i];
michael@0:     if (conn->connectionReady()) {
michael@0:       NS_NAMED_LITERAL_CSTRING(shrinkPragma, "PRAGMA shrink_memory");
michael@0:       nsCOMPtr<mozIStorageConnection> syncConn = do_QueryInterface(
michael@0:         NS_ISUPPORTS_CAST(mozIStorageAsyncConnection*, conn));
michael@0:       DebugOnly<nsresult> rv;
michael@0: 
michael@0:       if (!syncConn) {
michael@0:         nsCOMPtr<mozIStoragePendingStatement> ps;
michael@0:         rv = connections[i]->ExecuteSimpleSQLAsync(shrinkPragma, nullptr,
michael@0:           getter_AddRefs(ps));
michael@0:       } else {
michael@0:         rv = connections[i]->ExecuteSimpleSQL(shrinkPragma);
michael@0:       }
michael@0: 
michael@0:       MOZ_ASSERT(NS_SUCCEEDED(rv),
michael@0:         "Should have been able to purge sqlite caches");
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: void
michael@0: Service::shutdown()
michael@0: {
michael@0:   NS_IF_RELEASE(sXPConnect);
michael@0: }
michael@0: 
michael@0: sqlite3_vfs *ConstructTelemetryVFS();
michael@0: 
michael@0: #ifdef MOZ_STORAGE_MEMORY
michael@0: 
michael@0: namespace {
michael@0: 
michael@0: // By default, SQLite tracks the size of all its heap blocks by adding an extra
michael@0: // 8 bytes at the start of the block to hold the size.  Unfortunately, this
michael@0: // causes a lot of 2^N-sized allocations to be rounded up by jemalloc
michael@0: // allocator, wasting memory.  For example, a request for 1024 bytes has 8
michael@0: // bytes added, becoming a request for 1032 bytes, and jemalloc rounds this up
michael@0: // to 2048 bytes, wasting 1012 bytes.  (See bug 676189 for more details.)
michael@0: //
michael@0: // So we register jemalloc as the malloc implementation, which avoids this
michael@0: // 8-byte overhead, and thus a lot of waste.  This requires us to provide a
michael@0: // function, sqliteMemRoundup(), which computes the actual size that will be
michael@0: // allocated for a given request.  SQLite uses this function before all
michael@0: // allocations, and may be able to use any excess bytes caused by the rounding.
michael@0: //
michael@0: // Note: the wrappers for moz_malloc, moz_realloc and moz_malloc_usable_size
michael@0: // are necessary because the sqlite_mem_methods type signatures differ slightly
michael@0: // from the standard ones -- they use int instead of size_t.  But we don't need
michael@0: // a wrapper for moz_free.
michael@0: 
michael@0: #ifdef MOZ_DMD
michael@0: 
michael@0: // sqlite does its own memory accounting, and we use its numbers in our memory
michael@0: // reporters.  But we don't want sqlite's heap blocks to show up in DMD's
michael@0: // output as unreported, so we mark them as reported when they're allocated and
michael@0: // mark them as unreported when they are freed.
michael@0: //
michael@0: // In other words, we are marking all sqlite heap blocks as reported even
michael@0: // though we're not reporting them ourselves.  Instead we're trusting that
michael@0: // sqlite is fully and correctly accounting for all of its heap blocks via its
michael@0: // own memory accounting.  Well, we don't have to trust it entirely, because
michael@0: // it's easy to keep track (while doing this DMD-specific marking) of exactly
michael@0: // how much memory SQLite is using.  And we can compare that against what
michael@0: // SQLite reports it is using.
michael@0: 
michael@0: MOZ_DEFINE_MALLOC_SIZE_OF_ON_ALLOC(SqliteMallocSizeOfOnAlloc)
michael@0: MOZ_DEFINE_MALLOC_SIZE_OF_ON_FREE(SqliteMallocSizeOfOnFree)
michael@0: 
michael@0: #endif
michael@0: 
michael@0: static void *sqliteMemMalloc(int n)
michael@0: {
michael@0:   void* p = ::moz_malloc(n);
michael@0: #ifdef MOZ_DMD
michael@0:   gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p);
michael@0: #endif
michael@0:   return p;
michael@0: }
michael@0: 
michael@0: static void sqliteMemFree(void *p)
michael@0: {
michael@0: #ifdef MOZ_DMD
michael@0:   gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p);
michael@0: #endif
michael@0:   ::moz_free(p);
michael@0: }
michael@0: 
michael@0: static void *sqliteMemRealloc(void *p, int n)
michael@0: {
michael@0: #ifdef MOZ_DMD
michael@0:   gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p);
michael@0:   void *pnew = ::moz_realloc(p, n);
michael@0:   if (pnew) {
michael@0:     gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(pnew);
michael@0:   } else {
michael@0:     // realloc failed;  undo the SqliteMallocSizeOfOnFree from above
michael@0:     gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p);
michael@0:   }
michael@0:   return pnew;
michael@0: #else
michael@0:   return ::moz_realloc(p, n);
michael@0: #endif
michael@0: }
michael@0: 
michael@0: static int sqliteMemSize(void *p)
michael@0: {
michael@0:   return ::moz_malloc_usable_size(p);
michael@0: }
michael@0: 
michael@0: static int sqliteMemRoundup(int n)
michael@0: {
michael@0:   n = malloc_good_size(n);
michael@0: 
michael@0:   // jemalloc can return blocks of size 2 and 4, but SQLite requires that all
michael@0:   // allocations be 8-aligned.  So we round up sub-8 requests to 8.  This
michael@0:   // wastes a small amount of memory but is obviously safe.
michael@0:   return n <= 8 ? 8 : n;
michael@0: }
michael@0: 
michael@0: static int sqliteMemInit(void *p)
michael@0: {
michael@0:   return 0;
michael@0: }
michael@0: 
michael@0: static void sqliteMemShutdown(void *p)
michael@0: {
michael@0: }
michael@0: 
michael@0: const sqlite3_mem_methods memMethods = {
michael@0:   &sqliteMemMalloc,
michael@0:   &sqliteMemFree,
michael@0:   &sqliteMemRealloc,
michael@0:   &sqliteMemSize,
michael@0:   &sqliteMemRoundup,
michael@0:   &sqliteMemInit,
michael@0:   &sqliteMemShutdown,
michael@0:   nullptr
michael@0: };
michael@0: 
michael@0: } // anonymous namespace
michael@0: 
michael@0: #endif  // MOZ_STORAGE_MEMORY
michael@0: 
michael@0: static const char* sObserverTopics[] = {
michael@0:   "memory-pressure",
michael@0:   "xpcom-shutdown",
michael@0:   "xpcom-shutdown-threads"
michael@0: };
michael@0: 
michael@0: nsresult
michael@0: Service::initialize()
michael@0: {
michael@0:   MOZ_ASSERT(NS_IsMainThread(), "Must be initialized on the main thread");
michael@0: 
michael@0:   int rc;
michael@0: 
michael@0: #ifdef MOZ_STORAGE_MEMORY
michael@0:   rc = ::sqlite3_config(SQLITE_CONFIG_MALLOC, &memMethods);
michael@0:   if (rc != SQLITE_OK)
michael@0:     return convertResultCode(rc);
michael@0: #endif
michael@0: 
michael@0:   // Explicitly initialize sqlite3.  Although this is implicitly called by
michael@0:   // various sqlite3 functions (and the sqlite3_open calls in our case),
michael@0:   // the documentation suggests calling this directly.  So we do.
michael@0:   rc = ::sqlite3_initialize();
michael@0:   if (rc != SQLITE_OK)
michael@0:     return convertResultCode(rc);
michael@0: 
michael@0:   mSqliteVFS = ConstructTelemetryVFS();
michael@0:   if (mSqliteVFS) {
michael@0:     rc = sqlite3_vfs_register(mSqliteVFS, 1);
michael@0:     if (rc != SQLITE_OK)
michael@0:       return convertResultCode(rc);
michael@0:   } else {
michael@0:     NS_WARNING("Failed to register telemetry VFS");
michael@0:   }
michael@0: 
michael@0:   // Register for xpcom-shutdown so we can cleanup after ourselves.  The
michael@0:   // observer service can only be used on the main thread.
michael@0:   nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
michael@0:   NS_ENSURE_TRUE(os, NS_ERROR_FAILURE);
michael@0: 
michael@0:   for (size_t i = 0; i < ArrayLength(sObserverTopics); ++i) {
michael@0:     nsresult rv = os->AddObserver(this, sObserverTopics[i], false);
michael@0:     if (NS_WARN_IF(NS_FAILED(rv))) {
michael@0:       return rv;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // We cache XPConnect for our language helpers.  XPConnect can only be
michael@0:   // used on the main thread.
michael@0:   (void)CallGetService(nsIXPConnect::GetCID(), &sXPConnect);
michael@0: 
michael@0:   // We need to obtain the toolkit.storage.synchronous preferences on the main
michael@0:   // thread because the preference service can only be accessed there.  This
michael@0:   // is cached in the service for all future Open[Unshared]Database calls.
michael@0:   sSynchronousPref =
michael@0:     Preferences::GetInt(PREF_TS_SYNCHRONOUS, PREF_TS_SYNCHRONOUS_DEFAULT);
michael@0: 
michael@0:   // We need to obtain the toolkit.storage.pageSize preferences on the main
michael@0:   // thread because the preference service can only be accessed there.  This
michael@0:   // is cached in the service for all future Open[Unshared]Database calls.
michael@0:   sDefaultPageSize =
michael@0:       Preferences::GetInt(PREF_TS_PAGESIZE, PREF_TS_PAGESIZE_DEFAULT);
michael@0: 
michael@0:   mozilla::RegisterWeakMemoryReporter(this);
michael@0:   mozilla::RegisterStorageSQLiteDistinguishedAmount(StorageSQLiteDistinguishedAmount);
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: int
michael@0: Service::localeCompareStrings(const nsAString &aStr1,
michael@0:                               const nsAString &aStr2,
michael@0:                               int32_t aComparisonStrength)
michael@0: {
michael@0:   // The implementation of nsICollation.CompareString() is platform-dependent.
michael@0:   // On Linux it's not thread-safe.  It may not be on Windows and OS X either,
michael@0:   // but it's more difficult to tell.  We therefore synchronize this method.
michael@0:   MutexAutoLock mutex(mMutex);
michael@0: 
michael@0:   nsICollation *coll = getLocaleCollation();
michael@0:   if (!coll) {
michael@0:     NS_ERROR("Storage service has no collation");
michael@0:     return 0;
michael@0:   }
michael@0: 
michael@0:   int32_t res;
michael@0:   nsresult rv = coll->CompareString(aComparisonStrength, aStr1, aStr2, &res);
michael@0:   if (NS_FAILED(rv)) {
michael@0:     NS_ERROR("Collation compare string failed");
michael@0:     return 0;
michael@0:   }
michael@0: 
michael@0:   return res;
michael@0: }
michael@0: 
michael@0: nsICollation *
michael@0: Service::getLocaleCollation()
michael@0: {
michael@0:   mMutex.AssertCurrentThreadOwns();
michael@0: 
michael@0:   if (mLocaleCollation)
michael@0:     return mLocaleCollation;
michael@0: 
michael@0:   nsCOMPtr<nsILocaleService> svc(do_GetService(NS_LOCALESERVICE_CONTRACTID));
michael@0:   if (!svc) {
michael@0:     NS_WARNING("Could not get locale service");
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   nsCOMPtr<nsILocale> appLocale;
michael@0:   nsresult rv = svc->GetApplicationLocale(getter_AddRefs(appLocale));
michael@0:   if (NS_FAILED(rv)) {
michael@0:     NS_WARNING("Could not get application locale");
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   nsCOMPtr<nsICollationFactory> collFact =
michael@0:     do_CreateInstance(NS_COLLATIONFACTORY_CONTRACTID);
michael@0:   if (!collFact) {
michael@0:     NS_WARNING("Could not create collation factory");
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   rv = collFact->CreateCollation(appLocale, getter_AddRefs(mLocaleCollation));
michael@0:   if (NS_FAILED(rv)) {
michael@0:     NS_WARNING("Could not create collation");
michael@0:     return nullptr;
michael@0:   }
michael@0: 
michael@0:   return mLocaleCollation;
michael@0: }
michael@0: 
michael@0: ////////////////////////////////////////////////////////////////////////////////
michael@0: //// mozIStorageService
michael@0: 
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: Service::OpenSpecialDatabase(const char *aStorageKey,
michael@0:                              mozIStorageConnection **_connection)
michael@0: {
michael@0:   nsresult rv;
michael@0: 
michael@0:   nsCOMPtr<nsIFile> storageFile;
michael@0:   if (::strcmp(aStorageKey, "memory") == 0) {
michael@0:     // just fall through with nullptr storageFile, this will cause the storage
michael@0:     // connection to use a memory DB.
michael@0:   }
michael@0:   else {
michael@0:     return NS_ERROR_INVALID_ARG;
michael@0:   }
michael@0: 
michael@0:   nsRefPtr<Connection> msc = new Connection(this, SQLITE_OPEN_READWRITE, false);
michael@0: 
michael@0:   rv = storageFile ? msc->initialize(storageFile) : msc->initialize();
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   msc.forget(_connection);
michael@0:   return NS_OK;
michael@0: 
michael@0: }
michael@0: 
michael@0: namespace {
michael@0: 
michael@0: class AsyncInitDatabase MOZ_FINAL : public nsRunnable
michael@0: {
michael@0: public:
michael@0:   AsyncInitDatabase(Connection* aConnection,
michael@0:                     nsIFile* aStorageFile,
michael@0:                     int32_t aGrowthIncrement,
michael@0:                     mozIStorageCompletionCallback* aCallback)
michael@0:     : mConnection(aConnection)
michael@0:     , mStorageFile(aStorageFile)
michael@0:     , mGrowthIncrement(aGrowthIncrement)
michael@0:     , mCallback(aCallback)
michael@0:   {
michael@0:     MOZ_ASSERT(NS_IsMainThread());
michael@0:   }
michael@0: 
michael@0:   NS_IMETHOD Run()
michael@0:   {
michael@0:     MOZ_ASSERT(!NS_IsMainThread());
michael@0:     nsresult rv = mStorageFile ? mConnection->initialize(mStorageFile)
michael@0:                                : mConnection->initialize();
michael@0:     if (NS_FAILED(rv)) {
michael@0:       return DispatchResult(rv, nullptr);
michael@0:     }
michael@0: 
michael@0:     if (mGrowthIncrement >= 0) {
michael@0:       // Ignore errors. In the future, we might wish to log them.
michael@0:       (void)mConnection->SetGrowthIncrement(mGrowthIncrement, EmptyCString());
michael@0:     }
michael@0: 
michael@0:     return DispatchResult(NS_OK, NS_ISUPPORTS_CAST(mozIStorageAsyncConnection*,
michael@0:                           mConnection));
michael@0:   }
michael@0: 
michael@0: private:
michael@0:   nsresult DispatchResult(nsresult aStatus, nsISupports* aValue) {
michael@0:     nsRefPtr<CallbackComplete> event =
michael@0:       new CallbackComplete(aStatus,
michael@0:                            aValue,
michael@0:                            mCallback.forget());
michael@0:     return NS_DispatchToMainThread(event);
michael@0:   }
michael@0: 
michael@0:   ~AsyncInitDatabase()
michael@0:   {
michael@0:     nsCOMPtr<nsIThread> thread;
michael@0:     DebugOnly<nsresult> rv = NS_GetMainThread(getter_AddRefs(thread));
michael@0:     MOZ_ASSERT(NS_SUCCEEDED(rv));
michael@0:     (void)NS_ProxyRelease(thread, mStorageFile);
michael@0: 
michael@0:     // Handle ambiguous nsISupports inheritance.
michael@0:     Connection *rawConnection = nullptr;
michael@0:     mConnection.swap(rawConnection);
michael@0:     (void)NS_ProxyRelease(thread, NS_ISUPPORTS_CAST(mozIStorageConnection *,
michael@0:                                                     rawConnection));
michael@0: 
michael@0:     // Generally, the callback will be released by CallbackComplete.
michael@0:     // However, if for some reason Run() is not executed, we still
michael@0:     // need to ensure that it is released here.
michael@0:     mozIStorageCompletionCallback *rawCallback = nullptr;
michael@0:     mCallback.swap(rawCallback);
michael@0:     (void)NS_ProxyRelease(thread, rawCallback);
michael@0:   }
michael@0: 
michael@0:   nsRefPtr<Connection> mConnection;
michael@0:   nsCOMPtr<nsIFile> mStorageFile;
michael@0:   int32_t mGrowthIncrement;
michael@0:   nsRefPtr<mozIStorageCompletionCallback> mCallback;
michael@0: };
michael@0: 
michael@0: } // anonymous namespace
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: Service::OpenAsyncDatabase(nsIVariant *aDatabaseStore,
michael@0:                            nsIPropertyBag2 *aOptions,
michael@0:                            mozIStorageCompletionCallback *aCallback)
michael@0: {
michael@0:   if (!NS_IsMainThread()) {
michael@0:     return NS_ERROR_NOT_SAME_THREAD;
michael@0:   }
michael@0:   NS_ENSURE_ARG(aDatabaseStore);
michael@0:   NS_ENSURE_ARG(aCallback);
michael@0: 
michael@0:   nsCOMPtr<nsIFile> storageFile;
michael@0:   int flags = SQLITE_OPEN_READWRITE;
michael@0: 
michael@0:   nsCOMPtr<nsISupports> dbStore;
michael@0:   nsresult rv = aDatabaseStore->GetAsISupports(getter_AddRefs(dbStore));
michael@0:   if (NS_SUCCEEDED(rv)) {
michael@0:     // Generally, aDatabaseStore holds the database nsIFile.
michael@0:     storageFile = do_QueryInterface(dbStore, &rv);
michael@0:     if (NS_FAILED(rv)) {
michael@0:       return NS_ERROR_INVALID_ARG;
michael@0:     }
michael@0: 
michael@0:     rv = storageFile->Clone(getter_AddRefs(storageFile));
michael@0:     MOZ_ASSERT(NS_SUCCEEDED(rv));
michael@0: 
michael@0:     // Ensure that SQLITE_OPEN_CREATE is passed in for compatibility reasons.
michael@0:     flags |= SQLITE_OPEN_CREATE;
michael@0: 
michael@0:     // Extract and apply the shared-cache option.
michael@0:     bool shared = false;
michael@0:     if (aOptions) {
michael@0:       rv = aOptions->GetPropertyAsBool(NS_LITERAL_STRING("shared"), &shared);
michael@0:       if (NS_FAILED(rv) && rv != NS_ERROR_NOT_AVAILABLE) {
michael@0:         return NS_ERROR_INVALID_ARG;
michael@0:       }
michael@0:     }
michael@0:     flags |= shared ? SQLITE_OPEN_SHAREDCACHE : SQLITE_OPEN_PRIVATECACHE;
michael@0:   } else {
michael@0:     // Sometimes, however, it's a special database name.
michael@0:     nsAutoCString keyString;
michael@0:     rv = aDatabaseStore->GetAsACString(keyString);
michael@0:     if (NS_FAILED(rv) || !keyString.EqualsLiteral("memory")) {
michael@0:       return NS_ERROR_INVALID_ARG;
michael@0:     }
michael@0: 
michael@0:     // Just fall through with nullptr storageFile, this will cause the storage
michael@0:     // connection to use a memory DB.
michael@0:   }
michael@0: 
michael@0:   int32_t growthIncrement = -1;
michael@0:   if (aOptions && storageFile) {
michael@0:     rv = aOptions->GetPropertyAsInt32(NS_LITERAL_STRING("growthIncrement"),
michael@0:                                       &growthIncrement);
michael@0:     if (NS_FAILED(rv) && rv != NS_ERROR_NOT_AVAILABLE) {
michael@0:       return NS_ERROR_INVALID_ARG;
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   // Create connection on this thread, but initialize it on its helper thread.
michael@0:   nsRefPtr<Connection> msc = new Connection(this, flags, true);
michael@0:   nsCOMPtr<nsIEventTarget> target = msc->getAsyncExecutionTarget();
michael@0:   MOZ_ASSERT(target, "Cannot initialize a connection that has been closed already");
michael@0: 
michael@0:   nsRefPtr<AsyncInitDatabase> asyncInit =
michael@0:     new AsyncInitDatabase(msc,
michael@0:                           storageFile,
michael@0:                           growthIncrement,
michael@0:                           aCallback);
michael@0:   return target->Dispatch(asyncInit, nsIEventTarget::DISPATCH_NORMAL);
michael@0: }
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: Service::OpenDatabase(nsIFile *aDatabaseFile,
michael@0:                       mozIStorageConnection **_connection)
michael@0: {
michael@0:   NS_ENSURE_ARG(aDatabaseFile);
michael@0: 
michael@0:   // Always ensure that SQLITE_OPEN_CREATE is passed in for compatibility
michael@0:   // reasons.
michael@0:   int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_SHAREDCACHE |
michael@0:               SQLITE_OPEN_CREATE;
michael@0:   nsRefPtr<Connection> msc = new Connection(this, flags, false);
michael@0: 
michael@0:   nsresult rv = msc->initialize(aDatabaseFile);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   msc.forget(_connection);
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: Service::OpenUnsharedDatabase(nsIFile *aDatabaseFile,
michael@0:                               mozIStorageConnection **_connection)
michael@0: {
michael@0:   NS_ENSURE_ARG(aDatabaseFile);
michael@0: 
michael@0:   // Always ensure that SQLITE_OPEN_CREATE is passed in for compatibility
michael@0:   // reasons.
michael@0:   int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_PRIVATECACHE |
michael@0:               SQLITE_OPEN_CREATE;
michael@0:   nsRefPtr<Connection> msc = new Connection(this, flags, false);
michael@0: 
michael@0:   nsresult rv = msc->initialize(aDatabaseFile);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   msc.forget(_connection);
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: Service::OpenDatabaseWithFileURL(nsIFileURL *aFileURL,
michael@0:                                  mozIStorageConnection **_connection)
michael@0: {
michael@0:   NS_ENSURE_ARG(aFileURL);
michael@0: 
michael@0:   // Always ensure that SQLITE_OPEN_CREATE is passed in for compatibility
michael@0:   // reasons.
michael@0:   int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_SHAREDCACHE |
michael@0:               SQLITE_OPEN_CREATE | SQLITE_OPEN_URI;
michael@0:   nsRefPtr<Connection> msc = new Connection(this, flags, false);
michael@0: 
michael@0:   nsresult rv = msc->initialize(aFileURL);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   msc.forget(_connection);
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: Service::BackupDatabaseFile(nsIFile *aDBFile,
michael@0:                             const nsAString &aBackupFileName,
michael@0:                             nsIFile *aBackupParentDirectory,
michael@0:                             nsIFile **backup)
michael@0: {
michael@0:   nsresult rv;
michael@0:   nsCOMPtr<nsIFile> parentDir = aBackupParentDirectory;
michael@0:   if (!parentDir) {
michael@0:     // This argument is optional, and defaults to the same parent directory
michael@0:     // as the current file.
michael@0:     rv = aDBFile->GetParent(getter_AddRefs(parentDir));
michael@0:     NS_ENSURE_SUCCESS(rv, rv);
michael@0:   }
michael@0: 
michael@0:   nsCOMPtr<nsIFile> backupDB;
michael@0:   rv = parentDir->Clone(getter_AddRefs(backupDB));
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = backupDB->Append(aBackupFileName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = backupDB->CreateUnique(nsIFile::NORMAL_FILE_TYPE, 0600);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   nsAutoString fileName;
michael@0:   rv = backupDB->GetLeafName(fileName);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   rv = backupDB->Remove(false);
michael@0:   NS_ENSURE_SUCCESS(rv, rv);
michael@0: 
michael@0:   backupDB.forget(backup);
michael@0: 
michael@0:   return aDBFile->CopyTo(parentDir, fileName);
michael@0: }
michael@0: 
michael@0: ////////////////////////////////////////////////////////////////////////////////
michael@0: //// nsIObserver
michael@0: 
michael@0: NS_IMETHODIMP
michael@0: Service::Observe(nsISupports *, const char *aTopic, const char16_t *)
michael@0: {
michael@0:   if (strcmp(aTopic, "memory-pressure") == 0) {
michael@0:     minimizeMemory();
michael@0:   } else if (strcmp(aTopic, "xpcom-shutdown") == 0) {
michael@0:     shutdown();
michael@0:   } else if (strcmp(aTopic, "xpcom-shutdown-threads") == 0) {
michael@0:     nsCOMPtr<nsIObserverService> os =
michael@0:       mozilla::services::GetObserverService();
michael@0: 
michael@0:     for (size_t i = 0; i < ArrayLength(sObserverTopics); ++i) {
michael@0:       (void)os->RemoveObserver(this, sObserverTopics[i]);
michael@0:     }
michael@0: 
michael@0:     bool anyOpen = false;
michael@0:     do {
michael@0:       nsTArray<nsRefPtr<Connection> > connections;
michael@0:       getConnections(connections);
michael@0:       anyOpen = false;
michael@0:       for (uint32_t i = 0; i < connections.Length(); i++) {
michael@0:         nsRefPtr<Connection> &conn = connections[i];
michael@0:         if (conn->isClosing()) {
michael@0:           anyOpen = true;
michael@0:           break;
michael@0:         }
michael@0:       }
michael@0:       if (anyOpen) {
michael@0:         nsCOMPtr<nsIThread> thread = do_GetCurrentThread();
michael@0:         NS_ProcessNextEvent(thread);
michael@0:       }
michael@0:     } while (anyOpen);
michael@0: 
michael@0:     if (gShutdownChecks == SCM_CRASH) {
michael@0:       nsTArray<nsRefPtr<Connection> > connections;
michael@0:       getConnections(connections);
michael@0:       for (uint32_t i = 0, n = connections.Length(); i < n; i++) {
michael@0:         if (!connections[i]->isClosed()) {
michael@0:           MOZ_CRASH();
michael@0:         }
michael@0:       }
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   return NS_OK;
michael@0: }
michael@0: 
michael@0: } // namespace storage
michael@0: } // namespace mozilla