The Tor Browser / file comparison

comparison: dom/src/storage/DOMStorageDBThread.cpp

dom/src/storage/DOMStorageDBThread.cpp

changeset 0
6474c204b198
equal deleted inserted replaced
-1:000000000000 0:dae88f0fbae1
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* This Source Code Form is subject to the terms of the Mozilla Public
3 * License, v. 2.0. If a copy of the MPL was not distributed with this
4 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6 #include "DOMStorageDBThread.h"
7 #include "DOMStorageCache.h"
8
9 #include "nsIEffectiveTLDService.h"
10 #include "nsDirectoryServiceUtils.h"
11 #include "nsAppDirectoryServiceDefs.h"
12 #include "nsThreadUtils.h"
13 #include "nsProxyRelease.h"
14 #include "mozStorageCID.h"
15 #include "mozStorageHelper.h"
16 #include "mozIStorageService.h"
17 #include "mozIStorageBindingParamsArray.h"
18 #include "mozIStorageBindingParams.h"
19 #include "mozIStorageValueArray.h"
20 #include "mozIStorageFunction.h"
21 #include "nsIObserverService.h"
22 #include "nsIVariant.h"
23 #include "mozilla/IOInterposer.h"
24 #include "mozilla/Services.h"
25
26 // How long we collect write oprerations
27 // before they are flushed to the database
28 // In milliseconds.
29 #define FLUSHING_INTERVAL_MS 5000
30
31 // Write Ahead Log's maximum size is 512KB
32 #define MAX_WAL_SIZE_BYTES 512 * 1024
33
34 namespace mozilla {
35 namespace dom {
36
37 DOMStorageDBBridge::DOMStorageDBBridge()
38 {
39 }
40
41
42 DOMStorageDBThread::DOMStorageDBThread()
43 : mThread(nullptr)
44 , mMonitor("DOMStorageThreadMonitor")
45 , mStopIOThread(false)
46 , mWALModeEnabled(false)
47 , mDBReady(false)
48 , mStatus(NS_OK)
49 , mWorkerStatements(mWorkerConnection)
50 , mReaderStatements(mReaderConnection)
51 , mDirtyEpoch(0)
52 , mFlushImmediately(false)
53 , mPriorityCounter(0)
54 {
55 }
56
57 nsresult
58 DOMStorageDBThread::Init()
59 {
60 nsresult rv;
61
62 // Need to determine location on the main thread, since
63 // NS_GetSpecialDirectory access the atom table that can
64 // be accessed only on the main thread.
65 rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
66 getter_AddRefs(mDatabaseFile));
67 NS_ENSURE_SUCCESS(rv, rv);
68
69 rv = mDatabaseFile->Append(NS_LITERAL_STRING("webappsstore.sqlite"));
70 NS_ENSURE_SUCCESS(rv, rv);
71
72 // Ensure mozIStorageService init on the main thread first.
73 nsCOMPtr<mozIStorageService> service =
74 do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
75 NS_ENSURE_SUCCESS(rv, rv);
76
77 // Need to keep the lock to avoid setting mThread later then
78 // the thread body executes.
79 MonitorAutoLock monitor(mMonitor);
80
81 mThread = PR_CreateThread(PR_USER_THREAD, &DOMStorageDBThread::ThreadFunc, this,
82 PR_PRIORITY_LOW, PR_GLOBAL_THREAD, PR_JOINABLE_THREAD,
83 262144);
84 if (!mThread) {
85 return NS_ERROR_OUT_OF_MEMORY;
86 }
87
88 return NS_OK;
89 }
90
91 nsresult
92 DOMStorageDBThread::Shutdown()
93 {
94 if (!mThread) {
95 return NS_ERROR_NOT_INITIALIZED;
96 }
97
98 Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_SHUTDOWN_DATABASE_MS> timer;
99
100 {
101 MonitorAutoLock monitor(mMonitor);
102
103 // After we stop, no other operations can be accepted
104 mFlushImmediately = true;
105 mStopIOThread = true;
106 monitor.Notify();
107 }
108
109 PR_JoinThread(mThread);
110 mThread = nullptr;
111
112 return mStatus;
113 }
114
115 void
116 DOMStorageDBThread::SyncPreload(DOMStorageCacheBridge* aCache, bool aForceSync)
117 {
118 if (!aForceSync && aCache->LoadedCount()) {
119 // Preload already started for this cache, just wait for it to finish.
120 // LoadWait will exit after LoadDone on the cache has been called.
121 SetHigherPriority();
122 aCache->LoadWait();
123 SetDefaultPriority();
124 return;
125 }
126
127 // Bypass sync load when an update is pending in the queue to write, we would
128 // get incosistent data in the cache. Also don't allow sync main-thread preload
129 // when DB open and init is still pending on the background thread.
130 if (mDBReady && mWALModeEnabled) {
131 bool pendingTasks;
132 {
133 MonitorAutoLock monitor(mMonitor);
134 pendingTasks = mPendingTasks.IsScopeUpdatePending(aCache->Scope()) ||
135 mPendingTasks.IsScopeClearPending(aCache->Scope());
136 }
137
138 if (!pendingTasks) {
139 // WAL is enabled, thus do the load synchronously on the main thread.
140 DBOperation preload(DBOperation::opPreload, aCache);
141 preload.PerformAndFinalize(this);
142 return;
143 }
144 }
145
146 // Need to go asynchronously since WAL is not allowed or scheduled updates
147 // need to be flushed first.
148 // Schedule preload for this cache as the first operation.
149 nsresult rv = InsertDBOp(new DBOperation(DBOperation::opPreloadUrgent, aCache));
150
151 // LoadWait exits after LoadDone of the cache has been called.
152 if (NS_SUCCEEDED(rv)) {
153 aCache->LoadWait();
154 }
155 }
156
157 void
158 DOMStorageDBThread::AsyncFlush()
159 {
160 MonitorAutoLock monitor(mMonitor);
161 mFlushImmediately = true;
162 monitor.Notify();
163 }
164
165 bool
166 DOMStorageDBThread::ShouldPreloadScope(const nsACString& aScope)
167 {
168 MonitorAutoLock monitor(mMonitor);
169 return mScopesHavingData.Contains(aScope);
170 }
171
172 namespace { // anon
173
174 PLDHashOperator
175 GetScopesHavingDataEnum(nsCStringHashKey* aKey, void* aArg)
176 {
177 InfallibleTArray<nsCString>* scopes =
178 static_cast<InfallibleTArray<nsCString>*>(aArg);
179 scopes->AppendElement(aKey->GetKey());
180 return PL_DHASH_NEXT;
181 }
182
183 } // anon
184
185 void
186 DOMStorageDBThread::GetScopesHavingData(InfallibleTArray<nsCString>* aScopes)
187 {
188 MonitorAutoLock monitor(mMonitor);
189 mScopesHavingData.EnumerateEntries(GetScopesHavingDataEnum, aScopes);
190 }
191
192 nsresult
193 DOMStorageDBThread::InsertDBOp(DOMStorageDBThread::DBOperation* aOperation)
194 {
195 MonitorAutoLock monitor(mMonitor);
196
197 // Sentinel to don't forget to delete the operation when we exit early.
198 nsAutoPtr<DOMStorageDBThread::DBOperation> opScope(aOperation);
199
200 if (mStopIOThread) {
201 // Thread use after shutdown demanded.
202 MOZ_ASSERT(false);
203 return NS_ERROR_NOT_INITIALIZED;
204 }
205
206 if (NS_FAILED(mStatus)) {
207 MonitorAutoUnlock unlock(mMonitor);
208 aOperation->Finalize(mStatus);
209 return mStatus;
210 }
211
212 switch (aOperation->Type()) {
213 case DBOperation::opPreload:
214 case DBOperation::opPreloadUrgent:
215 if (mPendingTasks.IsScopeUpdatePending(aOperation->Scope())) {
216 // If there is a pending update operation for the scope first do the flush
217 // before we preload the cache. This may happen in an extremely rare case
218 // when a child process throws away its cache before flush on the parent
219 // has finished. If we would preloaded the cache as a priority operation
220 // before the pending flush, we would have got an inconsistent cache content.
221 mFlushImmediately = true;
222 } else if (mPendingTasks.IsScopeClearPending(aOperation->Scope())) {
223 // The scope is scheduled to be cleared, so just quickly load as empty.
224 // We need to do this to prevent load of the DB data before the scope has
225 // actually been cleared from the database. Preloads are processed
226 // immediately before update and clear operations on the database that
227 // are flushed periodically in batches.
228 MonitorAutoUnlock unlock(mMonitor);
229 aOperation->Finalize(NS_OK);
230 return NS_OK;
231 }
232 // NO BREAK
233
234 case DBOperation::opGetUsage:
235 if (aOperation->Type() == DBOperation::opPreloadUrgent) {
236 SetHigherPriority(); // Dropped back after urgent preload execution
237 mPreloads.InsertElementAt(0, aOperation);
238 } else {
239 mPreloads.AppendElement(aOperation);
240 }
241
242 // DB operation adopted, don't delete it.
243 opScope.forget();
244
245 // Immediately start executing this.
246 monitor.Notify();
247 break;
248
249 default:
250 // Update operations are first collected, coalesced and then flushed
251 // after a short time.
252 mPendingTasks.Add(aOperation);
253
254 // DB operation adopted, don't delete it.
255 opScope.forget();
256
257 ScheduleFlush();
258 break;
259 }
260
261 return NS_OK;
262 }
263
264 void
265 DOMStorageDBThread::SetHigherPriority()
266 {
267 ++mPriorityCounter;
268 PR_SetThreadPriority(mThread, PR_PRIORITY_URGENT);
269 }
270
271 void
272 DOMStorageDBThread::SetDefaultPriority()
273 {
274 if (--mPriorityCounter <= 0) {
275 PR_SetThreadPriority(mThread, PR_PRIORITY_LOW);
276 }
277 }
278
279 void
280 DOMStorageDBThread::ThreadFunc(void* aArg)
281 {
282 PR_SetCurrentThreadName("localStorage DB");
283 mozilla::IOInterposer::RegisterCurrentThread();
284
285 DOMStorageDBThread* thread = static_cast<DOMStorageDBThread*>(aArg);
286 thread->ThreadFunc();
287 mozilla::IOInterposer::UnregisterCurrentThread();
288 }
289
290 void
291 DOMStorageDBThread::ThreadFunc()
292 {
293 nsresult rv = InitDatabase();
294
295 MonitorAutoLock lockMonitor(mMonitor);
296
297 if (NS_FAILED(rv)) {
298 mStatus = rv;
299 mStopIOThread = true;
300 return;
301 }
302
303 while (MOZ_LIKELY(!mStopIOThread || mPreloads.Length() || mPendingTasks.HasTasks())) {
304 if (MOZ_UNLIKELY(TimeUntilFlush() == 0)) {
305 // Flush time is up or flush has been forced, do it now.
306 UnscheduleFlush();
307 if (mPendingTasks.Prepare()) {
308 {
309 MonitorAutoUnlock unlockMonitor(mMonitor);
310 rv = mPendingTasks.Execute(this);
311 }
312
313 if (!mPendingTasks.Finalize(rv)) {
314 mStatus = rv;
315 NS_WARNING("localStorage DB access broken");
316 }
317 }
318 NotifyFlushCompletion();
319 } else if (MOZ_LIKELY(mPreloads.Length())) {
320 nsAutoPtr<DBOperation> op(mPreloads[0]);
321 mPreloads.RemoveElementAt(0);
322 {
323 MonitorAutoUnlock unlockMonitor(mMonitor);
324 op->PerformAndFinalize(this);
325 }
326
327 if (op->Type() == DBOperation::opPreloadUrgent) {
328 SetDefaultPriority(); // urgent preload unscheduled
329 }
330 } else if (MOZ_UNLIKELY(!mStopIOThread)) {
331 lockMonitor.Wait(TimeUntilFlush());
332 }
333 } // thread loop
334
335 mStatus = ShutdownDatabase();
336 }
337
338 extern void
339 ReverseString(const nsCSubstring& aSource, nsCSubstring& aResult);
340
341 namespace { // anon
342
343 class nsReverseStringSQLFunction MOZ_FINAL : public mozIStorageFunction
344 {
345 NS_DECL_ISUPPORTS
346 NS_DECL_MOZISTORAGEFUNCTION
347 };
348
349 NS_IMPL_ISUPPORTS(nsReverseStringSQLFunction, mozIStorageFunction)
350
351 NS_IMETHODIMP
352 nsReverseStringSQLFunction::OnFunctionCall(
353 mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult)
354 {
355 nsresult rv;
356
357 nsAutoCString stringToReverse;
358 rv = aFunctionArguments->GetUTF8String(0, stringToReverse);
359 NS_ENSURE_SUCCESS(rv, rv);
360
361 nsAutoCString result;
362 ReverseString(stringToReverse, result);
363
364 nsCOMPtr<nsIWritableVariant> outVar(do_CreateInstance(
365 NS_VARIANT_CONTRACTID, &rv));
366 NS_ENSURE_SUCCESS(rv, rv);
367
368 rv = outVar->SetAsAUTF8String(result);
369 NS_ENSURE_SUCCESS(rv, rv);
370
371 *aResult = outVar.get();
372 outVar.forget();
373 return NS_OK;
374 }
375
376 } // anon
377
378 nsresult
379 DOMStorageDBThread::OpenDatabaseConnection()
380 {
381 nsresult rv;
382
383 MOZ_ASSERT(!NS_IsMainThread());
384
385 nsCOMPtr<mozIStorageService> service
386 = do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
387 NS_ENSURE_SUCCESS(rv, rv);
388
389 nsCOMPtr<mozIStorageConnection> connection;
390 rv = service->OpenUnsharedDatabase(mDatabaseFile, getter_AddRefs(mWorkerConnection));
391 if (rv == NS_ERROR_FILE_CORRUPTED) {
392 // delete the db and try opening again
393 rv = mDatabaseFile->Remove(false);
394 NS_ENSURE_SUCCESS(rv, rv);
395 rv = service->OpenUnsharedDatabase(mDatabaseFile, getter_AddRefs(mWorkerConnection));
396 }
397 NS_ENSURE_SUCCESS(rv, rv);
398
399 return NS_OK;
400 }
401
402 nsresult
403 DOMStorageDBThread::InitDatabase()
404 {
405 Telemetry::AutoTimer<Telemetry::LOCALDOMSTORAGE_INIT_DATABASE_MS> timer;
406
407 nsresult rv;
408
409 // Here we are on the worker thread. This opens the worker connection.
410 MOZ_ASSERT(!NS_IsMainThread());
411
412 rv = OpenDatabaseConnection();
413 NS_ENSURE_SUCCESS(rv, rv);
414
415 rv = TryJournalMode();
416 NS_ENSURE_SUCCESS(rv, rv);
417
418 // Create a read-only clone
419 (void)mWorkerConnection->Clone(true, getter_AddRefs(mReaderConnection));
420 NS_ENSURE_TRUE(mReaderConnection, NS_ERROR_FAILURE);
421
422 mozStorageTransaction transaction(mWorkerConnection, false);
423
424 // Ensure Gecko 1.9.1 storage table
425 rv = mWorkerConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
426 "CREATE TABLE IF NOT EXISTS webappsstore2 ("
427 "scope TEXT, "
428 "key TEXT, "
429 "value TEXT, "
430 "secure INTEGER, "
431 "owner TEXT)"));
432 NS_ENSURE_SUCCESS(rv, rv);
433
434 rv = mWorkerConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
435 "CREATE UNIQUE INDEX IF NOT EXISTS scope_key_index"
436 " ON webappsstore2(scope, key)"));
437 NS_ENSURE_SUCCESS(rv, rv);
438
439 nsCOMPtr<mozIStorageFunction> function1(new nsReverseStringSQLFunction());
440 NS_ENSURE_TRUE(function1, NS_ERROR_OUT_OF_MEMORY);
441
442 rv = mWorkerConnection->CreateFunction(NS_LITERAL_CSTRING("REVERSESTRING"), 1, function1);
443 NS_ENSURE_SUCCESS(rv, rv);
444
445 bool exists;
446
447 // Check if there is storage of Gecko 1.9.0 and if so, upgrade that storage
448 // to actual webappsstore2 table and drop the obsolete table. First process
449 // this newer table upgrade to priority potential duplicates from older
450 // storage table.
451 rv = mWorkerConnection->TableExists(NS_LITERAL_CSTRING("webappsstore"),
452 &exists);
453 NS_ENSURE_SUCCESS(rv, rv);
454
455 if (exists) {
456 rv = mWorkerConnection->ExecuteSimpleSQL(
457 NS_LITERAL_CSTRING("INSERT OR IGNORE INTO "
458 "webappsstore2(scope, key, value, secure, owner) "
459 "SELECT REVERSESTRING(domain) || '.:', key, value, secure, owner "
460 "FROM webappsstore"));
461 NS_ENSURE_SUCCESS(rv, rv);
462
463 rv = mWorkerConnection->ExecuteSimpleSQL(
464 NS_LITERAL_CSTRING("DROP TABLE webappsstore"));
465 NS_ENSURE_SUCCESS(rv, rv);
466 }
467
468 // Check if there is storage of Gecko 1.8 and if so, upgrade that storage
469 // to actual webappsstore2 table and drop the obsolete table. Potential
470 // duplicates will be ignored.
471 rv = mWorkerConnection->TableExists(NS_LITERAL_CSTRING("moz_webappsstore"),
472 &exists);
473 NS_ENSURE_SUCCESS(rv, rv);
474
475 if (exists) {
476 rv = mWorkerConnection->ExecuteSimpleSQL(
477 NS_LITERAL_CSTRING("INSERT OR IGNORE INTO "
478 "webappsstore2(scope, key, value, secure, owner) "
479 "SELECT REVERSESTRING(domain) || '.:', key, value, secure, domain "
480 "FROM moz_webappsstore"));
481 NS_ENSURE_SUCCESS(rv, rv);
482
483 rv = mWorkerConnection->ExecuteSimpleSQL(
484 NS_LITERAL_CSTRING("DROP TABLE moz_webappsstore"));
485 NS_ENSURE_SUCCESS(rv, rv);
486 }
487
488 rv = transaction.Commit();
489 NS_ENSURE_SUCCESS(rv, rv);
490
491 // Database open and all initiation operation are done. Switching this flag
492 // to true allow main thread to read directly from the database.
493 // If we would allow this sooner, we would have opened a window where main thread
494 // read might operate on a totaly broken and incosistent database.
495 mDBReady = true;
496
497 // List scopes having any stored data
498 nsCOMPtr<mozIStorageStatement> stmt;
499 rv = mWorkerConnection->CreateStatement(NS_LITERAL_CSTRING("SELECT DISTINCT scope FROM webappsstore2"),
500 getter_AddRefs(stmt));
501 NS_ENSURE_SUCCESS(rv, rv);
502 mozStorageStatementScoper scope(stmt);
503
504 while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
505 nsAutoCString foundScope;
506 rv = stmt->GetUTF8String(0, foundScope);
507 NS_ENSURE_SUCCESS(rv, rv);
508
509 MonitorAutoLock monitor(mMonitor);
510 mScopesHavingData.PutEntry(foundScope);
511 }
512
513 return NS_OK;
514 }
515
516 nsresult
517 DOMStorageDBThread::SetJournalMode(bool aIsWal)
518 {
519 nsresult rv;
520
521 nsAutoCString stmtString(
522 MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA journal_mode = ");
523 if (aIsWal) {
524 stmtString.AppendLiteral("wal");
525 } else {
526 stmtString.AppendLiteral("truncate");
527 }
528
529 nsCOMPtr<mozIStorageStatement> stmt;
530 rv = mWorkerConnection->CreateStatement(stmtString, getter_AddRefs(stmt));
531 NS_ENSURE_SUCCESS(rv, rv);
532 mozStorageStatementScoper scope(stmt);
533
534 bool hasResult = false;
535 rv = stmt->ExecuteStep(&hasResult);
536 NS_ENSURE_SUCCESS(rv, rv);
537 if (!hasResult) {
538 return NS_ERROR_FAILURE;
539 }
540
541 nsAutoCString journalMode;
542 rv = stmt->GetUTF8String(0, journalMode);
543 NS_ENSURE_SUCCESS(rv, rv);
544 if ((aIsWal && !journalMode.EqualsLiteral("wal")) ||
545 (!aIsWal && !journalMode.EqualsLiteral("truncate"))) {
546 return NS_ERROR_FAILURE;
547 }
548
549 return NS_OK;
550 }
551
552 nsresult
553 DOMStorageDBThread::TryJournalMode()
554 {
555 nsresult rv;
556
557 rv = SetJournalMode(true);
558 if (NS_FAILED(rv)) {
559 mWALModeEnabled = false;
560
561 rv = SetJournalMode(false);
562 NS_ENSURE_SUCCESS(rv, rv);
563 } else {
564 mWALModeEnabled = true;
565
566 rv = ConfigureWALBehavior();
567 NS_ENSURE_SUCCESS(rv, rv);
568 }
569
570 return NS_OK;
571 }
572
573 nsresult
574 DOMStorageDBThread::ConfigureWALBehavior()
575 {
576 // Get the DB's page size
577 nsCOMPtr<mozIStorageStatement> stmt;
578 nsresult rv = mWorkerConnection->CreateStatement(NS_LITERAL_CSTRING(
579 MOZ_STORAGE_UNIQUIFY_QUERY_STR "PRAGMA page_size"
580 ), getter_AddRefs(stmt));
581 NS_ENSURE_SUCCESS(rv, rv);
582
583 bool hasResult = false;
584 rv = stmt->ExecuteStep(&hasResult);
585 NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
586
587 int32_t pageSize = 0;
588 rv = stmt->GetInt32(0, &pageSize);
589 NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && pageSize > 0, NS_ERROR_UNEXPECTED);
590
591 // Set the threshold for auto-checkpointing the WAL.
592 // We don't want giant logs slowing down reads & shutdown.
593 int32_t thresholdInPages = static_cast<int32_t>(MAX_WAL_SIZE_BYTES / pageSize);
594 nsAutoCString thresholdPragma("PRAGMA wal_autocheckpoint = ");
595 thresholdPragma.AppendInt(thresholdInPages);
596 rv = mWorkerConnection->ExecuteSimpleSQL(thresholdPragma);
597 NS_ENSURE_SUCCESS(rv, rv);
598
599 // Set the maximum WAL log size to reduce footprint on mobile (large empty
600 // WAL files will be truncated)
601 nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
602 // bug 600307: mak recommends setting this to 3 times the auto-checkpoint threshold
603 journalSizePragma.AppendInt(MAX_WAL_SIZE_BYTES * 3);
604 rv = mWorkerConnection->ExecuteSimpleSQL(journalSizePragma);
605 NS_ENSURE_SUCCESS(rv, rv);
606
607 return NS_OK;
608 }
609
610 nsresult
611 DOMStorageDBThread::ShutdownDatabase()
612 {
613 // Has to be called on the worker thread.
614 MOZ_ASSERT(!NS_IsMainThread());
615
616 nsresult rv = mStatus;
617
618 mDBReady = false;
619
620 // Finalize the cached statements.
621 mReaderStatements.FinalizeStatements();
622 mWorkerStatements.FinalizeStatements();
623
624 if (mReaderConnection) {
625 // No need to sync access to mReaderConnection since the main thread
626 // is right now joining this thread, unable to execute any events.
627 mReaderConnection->Close();
628 mReaderConnection = nullptr;
629 }
630
631 if (mWorkerConnection) {
632 rv = mWorkerConnection->Close();
633 mWorkerConnection = nullptr;
634 }
635
636 return rv;
637 }
638
639 void
640 DOMStorageDBThread::ScheduleFlush()
641 {
642 if (mDirtyEpoch) {
643 return; // Already scheduled
644 }
645
646 mDirtyEpoch = PR_IntervalNow() | 1; // Must be non-zero to indicate we are scheduled
647
648 // Wake the monitor from indefinite sleep...
649 mMonitor.Notify();
650 }
651
652 void
653 DOMStorageDBThread::UnscheduleFlush()
654 {
655 // We are just about to do the flush, drop flags
656 mFlushImmediately = false;
657 mDirtyEpoch = 0;
658 }
659
660 PRIntervalTime
661 DOMStorageDBThread::TimeUntilFlush()
662 {
663 if (mFlushImmediately) {
664 return 0; // Do it now regardless the timeout.
665 }
666
667 static_assert(PR_INTERVAL_NO_TIMEOUT != 0,
668 "PR_INTERVAL_NO_TIMEOUT must be non-zero");
669
670 if (!mDirtyEpoch) {
671 return PR_INTERVAL_NO_TIMEOUT; // No pending task...
672 }
673
674 static const PRIntervalTime kMaxAge = PR_MillisecondsToInterval(FLUSHING_INTERVAL_MS);
675
676 PRIntervalTime now = PR_IntervalNow() | 1;
677 PRIntervalTime age = now - mDirtyEpoch;
678 if (age > kMaxAge) {
679 return 0; // It is time.
680 }
681
682 return kMaxAge - age; // Time left, this is used to sleep the monitor
683 }
684
685 void
686 DOMStorageDBThread::NotifyFlushCompletion()
687 {
688 #ifdef DOM_STORAGE_TESTS
689 if (!NS_IsMainThread()) {
690 nsRefPtr<nsRunnableMethod<DOMStorageDBThread, void, false> > event =
691 NS_NewNonOwningRunnableMethod(this, &DOMStorageDBThread::NotifyFlushCompletion);
692 NS_DispatchToMainThread(event);
693 return;
694 }
695
696 nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
697 if (obs) {
698 obs->NotifyObservers(nullptr, "domstorage-test-flushed", nullptr);
699 }
700 #endif
701 }
702
703 // DOMStorageDBThread::DBOperation
704
705 DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
706 DOMStorageCacheBridge* aCache,
707 const nsAString& aKey,
708 const nsAString& aValue)
709 : mType(aType)
710 , mCache(aCache)
711 , mKey(aKey)
712 , mValue(aValue)
713 {
714 MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
715 }
716
717 DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
718 DOMStorageUsageBridge* aUsage)
719 : mType(aType)
720 , mUsage(aUsage)
721 {
722 MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
723 }
724
725 DOMStorageDBThread::DBOperation::DBOperation(const OperationType aType,
726 const nsACString& aScope)
727 : mType(aType)
728 , mCache(nullptr)
729 , mScope(aScope)
730 {
731 MOZ_COUNT_CTOR(DOMStorageDBThread::DBOperation);
732 }
733
734 DOMStorageDBThread::DBOperation::~DBOperation()
735 {
736 MOZ_COUNT_DTOR(DOMStorageDBThread::DBOperation);
737 }
738
739 const nsCString
740 DOMStorageDBThread::DBOperation::Scope()
741 {
742 if (mCache) {
743 return mCache->Scope();
744 }
745
746 return mScope;
747 }
748
749 const nsCString
750 DOMStorageDBThread::DBOperation::Target()
751 {
752 switch (mType) {
753 case opAddItem:
754 case opUpdateItem:
755 case opRemoveItem:
756 return Scope() + NS_LITERAL_CSTRING("|") + NS_ConvertUTF16toUTF8(mKey);
757
758 default:
759 return Scope();
760 }
761 }
762
763 void
764 DOMStorageDBThread::DBOperation::PerformAndFinalize(DOMStorageDBThread* aThread)
765 {
766 Finalize(Perform(aThread));
767 }
768
769 nsresult
770 DOMStorageDBThread::DBOperation::Perform(DOMStorageDBThread* aThread)
771 {
772 nsresult rv;
773
774 switch (mType) {
775 case opPreload:
776 case opPreloadUrgent:
777 {
778 // Already loaded?
779 if (mCache->Loaded()) {
780 break;
781 }
782
783 StatementCache* statements;
784 if (MOZ_UNLIKELY(NS_IsMainThread())) {
785 statements = &aThread->mReaderStatements;
786 } else {
787 statements = &aThread->mWorkerStatements;
788 }
789
790 // OFFSET is an optimization when we have to do a sync load
791 // and cache has already loaded some parts asynchronously.
792 // It skips keys we have already loaded.
793 nsCOMPtr<mozIStorageStatement> stmt = statements->GetCachedStatement(
794 "SELECT key, value FROM webappsstore2 "
795 "WHERE scope = :scope ORDER BY key "
796 "LIMIT -1 OFFSET :offset");
797 NS_ENSURE_STATE(stmt);
798 mozStorageStatementScoper scope(stmt);
799
800 rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
801 mCache->Scope());
802 NS_ENSURE_SUCCESS(rv, rv);
803
804 rv = stmt->BindInt32ByName(NS_LITERAL_CSTRING("offset"),
805 static_cast<int32_t>(mCache->LoadedCount()));
806 NS_ENSURE_SUCCESS(rv, rv);
807
808 bool exists;
809 while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&exists)) && exists) {
810 nsAutoString key;
811 rv = stmt->GetString(0, key);
812 NS_ENSURE_SUCCESS(rv, rv);
813
814 nsAutoString value;
815 rv = stmt->GetString(1, value);
816 NS_ENSURE_SUCCESS(rv, rv);
817
818 if (!mCache->LoadItem(key, value)) {
819 break;
820 }
821 }
822
823 mCache->LoadDone(NS_OK);
824 break;
825 }
826
827 case opGetUsage:
828 {
829 nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
830 "SELECT SUM(LENGTH(key) + LENGTH(value)) FROM webappsstore2"
831 " WHERE scope LIKE :scope"
832 );
833 NS_ENSURE_STATE(stmt);
834
835 mozStorageStatementScoper scope(stmt);
836
837 rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
838 mUsage->Scope() + NS_LITERAL_CSTRING("%"));
839 NS_ENSURE_SUCCESS(rv, rv);
840
841 bool exists;
842 rv = stmt->ExecuteStep(&exists);
843 NS_ENSURE_SUCCESS(rv, rv);
844
845 int64_t usage = 0;
846 if (exists) {
847 rv = stmt->GetInt64(0, &usage);
848 NS_ENSURE_SUCCESS(rv, rv);
849 }
850
851 mUsage->LoadUsage(usage);
852 break;
853 }
854
855 case opAddItem:
856 case opUpdateItem:
857 {
858 MOZ_ASSERT(!NS_IsMainThread());
859
860 nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
861 "INSERT OR REPLACE INTO webappsstore2 (scope, key, value) "
862 "VALUES (:scope, :key, :value) "
863 );
864 NS_ENSURE_STATE(stmt);
865
866 mozStorageStatementScoper scope(stmt);
867
868 rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
869 mCache->Scope());
870 NS_ENSURE_SUCCESS(rv, rv);
871 rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
872 mKey);
873 NS_ENSURE_SUCCESS(rv, rv);
874 rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"),
875 mValue);
876 NS_ENSURE_SUCCESS(rv, rv);
877
878 rv = stmt->Execute();
879 NS_ENSURE_SUCCESS(rv, rv);
880
881 aThread->mScopesHavingData.PutEntry(Scope());
882 break;
883 }
884
885 case opRemoveItem:
886 {
887 MOZ_ASSERT(!NS_IsMainThread());
888
889 nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
890 "DELETE FROM webappsstore2 "
891 "WHERE scope = :scope "
892 "AND key = :key "
893 );
894 NS_ENSURE_STATE(stmt);
895 mozStorageStatementScoper scope(stmt);
896
897 rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
898 mCache->Scope());
899 NS_ENSURE_SUCCESS(rv, rv);
900 rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
901 mKey);
902 NS_ENSURE_SUCCESS(rv, rv);
903
904 rv = stmt->Execute();
905 NS_ENSURE_SUCCESS(rv, rv);
906
907 break;
908 }
909
910 case opClear:
911 {
912 MOZ_ASSERT(!NS_IsMainThread());
913
914 nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
915 "DELETE FROM webappsstore2 "
916 "WHERE scope = :scope"
917 );
918 NS_ENSURE_STATE(stmt);
919 mozStorageStatementScoper scope(stmt);
920
921 rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
922 mCache->Scope());
923 NS_ENSURE_SUCCESS(rv, rv);
924
925 rv = stmt->Execute();
926 NS_ENSURE_SUCCESS(rv, rv);
927
928 aThread->mScopesHavingData.RemoveEntry(Scope());
929 break;
930 }
931
932 case opClearAll:
933 {
934 MOZ_ASSERT(!NS_IsMainThread());
935
936 nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
937 "DELETE FROM webappsstore2"
938 );
939 NS_ENSURE_STATE(stmt);
940 mozStorageStatementScoper scope(stmt);
941
942 rv = stmt->Execute();
943 NS_ENSURE_SUCCESS(rv, rv);
944
945 aThread->mScopesHavingData.Clear();
946 break;
947 }
948
949 case opClearMatchingScope:
950 {
951 MOZ_ASSERT(!NS_IsMainThread());
952
953 nsCOMPtr<mozIStorageStatement> stmt = aThread->mWorkerStatements.GetCachedStatement(
954 "DELETE FROM webappsstore2"
955 " WHERE scope GLOB :scope"
956 );
957 NS_ENSURE_STATE(stmt);
958 mozStorageStatementScoper scope(stmt);
959
960 rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
961 mScope + NS_LITERAL_CSTRING("*"));
962 NS_ENSURE_SUCCESS(rv, rv);
963
964 rv = stmt->Execute();
965 NS_ENSURE_SUCCESS(rv, rv);
966
967 break;
968 }
969
970 default:
971 NS_ERROR("Unknown task type");
972 break;
973 }
974
975 return NS_OK;
976 }
977
978 void
979 DOMStorageDBThread::DBOperation::Finalize(nsresult aRv)
980 {
981 switch (mType) {
982 case opPreloadUrgent:
983 case opPreload:
984 if (NS_FAILED(aRv)) {
985 // When we are here, something failed when loading from the database.
986 // Notify that the storage is loaded to prevent deadlock of the main thread,
987 // even though it is actually empty or incomplete.
988 NS_WARNING("Failed to preload localStorage");
989 }
990
991 mCache->LoadDone(aRv);
992 break;
993
994 case opGetUsage:
995 if (NS_FAILED(aRv)) {
996 mUsage->LoadUsage(0);
997 }
998
999 break;
1000
1001 default:
1002 if (NS_FAILED(aRv)) {
1003 NS_WARNING("localStorage update/clear operation failed,"
1004 " data may not persist or clean up");
1005 }
1006
1007 break;
1008 }
1009 }
1010
1011 // DOMStorageDBThread::PendingOperations
1012
1013 DOMStorageDBThread::PendingOperations::PendingOperations()
1014 : mFlushFailureCount(0)
1015 {
1016 }
1017
1018 bool
1019 DOMStorageDBThread::PendingOperations::HasTasks()
1020 {
1021 return !!mUpdates.Count() || !!mClears.Count();
1022 }
1023
1024 namespace { // anon
1025
1026 PLDHashOperator
1027 ForgetUpdatesForScope(const nsACString& aMapping,
1028 nsAutoPtr<DOMStorageDBThread::DBOperation>& aPendingTask,
1029 void* aArg)
1030 {
1031 DOMStorageDBThread::DBOperation* newOp = static_cast<DOMStorageDBThread::DBOperation*>(aArg);
1032
1033 if (newOp->Type() == DOMStorageDBThread::DBOperation::opClear &&
1034 aPendingTask->Scope() != newOp->Scope()) {
1035 return PL_DHASH_NEXT;
1036 }
1037
1038 if (newOp->Type() == DOMStorageDBThread::DBOperation::opClearMatchingScope &&
1039 !StringBeginsWith(aPendingTask->Scope(), newOp->Scope())) {
1040 return PL_DHASH_NEXT;
1041 }
1042
1043 return PL_DHASH_REMOVE;
1044 }
1045
1046 } // anon
1047
1048 bool
1049 DOMStorageDBThread::PendingOperations::CheckForCoalesceOpportunity(DBOperation* aNewOp,
1050 DBOperation::OperationType aPendingType,
1051 DBOperation::OperationType aNewType)
1052 {
1053 if (aNewOp->Type() != aNewType) {
1054 return false;
1055 }
1056
1057 DOMStorageDBThread::DBOperation* pendingTask;
1058 if (!mUpdates.Get(aNewOp->Target(), &pendingTask)) {
1059 return false;
1060 }
1061
1062 if (pendingTask->Type() != aPendingType) {
1063 return false;
1064 }
1065
1066 return true;
1067 }
1068
1069 void
1070 DOMStorageDBThread::PendingOperations::Add(DOMStorageDBThread::DBOperation* aOperation)
1071 {
1072 // Optimize: when a key to remove has never been written to disk
1073 // just bypass this operation. A kew is new when an operation scheduled
1074 // to write it to the database is of type opAddItem.
1075 if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem, DBOperation::opRemoveItem)) {
1076 mUpdates.Remove(aOperation->Target());
1077 delete aOperation;
1078 return;
1079 }
1080
1081 // Optimize: when changing a key that is new and has never been
1082 // written to disk, keep type of the operation to store it at opAddItem.
1083 // This allows optimization to just forget adding a new key when
1084 // it is removed from the storage before flush.
1085 if (CheckForCoalesceOpportunity(aOperation, DBOperation::opAddItem, DBOperation::opUpdateItem)) {
1086 aOperation->mType = DBOperation::opAddItem;
1087 }
1088
1089 // Optimize: to prevent lose of remove operation on a key when doing
1090 // remove/set/remove on a previously existing key we have to change
1091 // opAddItem to opUpdateItem on the new operation when there is opRemoveItem
1092 // pending for the key.
1093 if (CheckForCoalesceOpportunity(aOperation, DBOperation::opRemoveItem, DBOperation::opAddItem)) {
1094 aOperation->mType = DBOperation::opUpdateItem;
1095 }
1096
1097 switch (aOperation->Type())
1098 {
1099 // Operations on single keys
1100
1101 case DBOperation::opAddItem:
1102 case DBOperation::opUpdateItem:
1103 case DBOperation::opRemoveItem:
1104 // Override any existing operation for the target (=scope+key).
1105 mUpdates.Put(aOperation->Target(), aOperation);
1106 break;
1107
1108 // Clear operations
1109
1110 case DBOperation::opClear:
1111 case DBOperation::opClearMatchingScope:
1112 // Drop all update (insert/remove) operations for equivavelent or matching scope.
1113 // We do this as an optimization as well as a must based on the logic,
1114 // if we would not delete the update tasks, changes would have been stored
1115 // to the database after clear operations have been executed.
1116 mUpdates.Enumerate(ForgetUpdatesForScope, aOperation);
1117 mClears.Put(aOperation->Target(), aOperation);
1118 break;
1119
1120 case DBOperation::opClearAll:
1121 // Drop simply everything, this is a super-operation.
1122 mUpdates.Clear();
1123 mClears.Clear();
1124 mClears.Put(aOperation->Target(), aOperation);
1125 break;
1126
1127 default:
1128 MOZ_ASSERT(false);
1129 break;
1130 }
1131 }
1132
1133 namespace { // anon
1134
1135 PLDHashOperator
1136 CollectTasks(const nsACString& aMapping, nsAutoPtr<DOMStorageDBThread::DBOperation>& aOperation, void* aArg)
1137 {
1138 nsTArray<nsAutoPtr<DOMStorageDBThread::DBOperation> >* tasks =
1139 static_cast<nsTArray<nsAutoPtr<DOMStorageDBThread::DBOperation> >*>(aArg);
1140
1141 tasks->AppendElement(aOperation.forget());
1142 return PL_DHASH_NEXT;
1143 }
1144
1145 } // anon
1146
1147 bool
1148 DOMStorageDBThread::PendingOperations::Prepare()
1149 {
1150 // Called under the lock
1151
1152 // First collect clear operations and then updates, we can
1153 // do this since whenever a clear operation for a scope is
1154 // scheduled, we drop all updates matching that scope. So,
1155 // all scope-related update operations we have here now were
1156 // scheduled after the clear operations.
1157 mClears.Enumerate(CollectTasks, &mExecList);
1158 mClears.Clear();
1159
1160 mUpdates.Enumerate(CollectTasks, &mExecList);
1161 mUpdates.Clear();
1162
1163 return !!mExecList.Length();
1164 }
1165
1166 nsresult
1167 DOMStorageDBThread::PendingOperations::Execute(DOMStorageDBThread* aThread)
1168 {
1169 // Called outside the lock
1170
1171 mozStorageTransaction transaction(aThread->mWorkerConnection, false);
1172
1173 nsresult rv;
1174
1175 for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1176 DOMStorageDBThread::DBOperation* task = mExecList[i];
1177 rv = task->Perform(aThread);
1178 if (NS_FAILED(rv)) {
1179 return rv;
1180 }
1181 }
1182
1183 rv = transaction.Commit();
1184 if (NS_FAILED(rv)) {
1185 return rv;
1186 }
1187
1188 return NS_OK;
1189 }
1190
1191 bool
1192 DOMStorageDBThread::PendingOperations::Finalize(nsresult aRv)
1193 {
1194 // Called under the lock
1195
1196 // The list is kept on a failure to retry it
1197 if (NS_FAILED(aRv)) {
1198 // XXX Followup: we may try to reopen the database and flush these
1199 // pending tasks, however testing showed that even though I/O is actually
1200 // broken some amount of operations is left in sqlite+system buffers and
1201 // seems like successfully flushed to disk.
1202 // Tested by removing a flash card and disconnecting from network while
1203 // using a network drive on Windows system.
1204 NS_WARNING("Flush operation on localStorage database failed");
1205
1206 ++mFlushFailureCount;
1207
1208 return mFlushFailureCount >= 5;
1209 }
1210
1211 mFlushFailureCount = 0;
1212 mExecList.Clear();
1213 return true;
1214 }
1215
1216 namespace { // anon
1217
1218 class FindPendingOperationForScopeData
1219 {
1220 public:
1221 FindPendingOperationForScopeData(const nsACString& aScope) : mScope(aScope), mFound(false) {}
1222 nsCString mScope;
1223 bool mFound;
1224 };
1225
1226 PLDHashOperator
1227 FindPendingClearForScope(const nsACString& aMapping,
1228 DOMStorageDBThread::DBOperation* aPendingOperation,
1229 void* aArg)
1230 {
1231 FindPendingOperationForScopeData* data =
1232 static_cast<FindPendingOperationForScopeData*>(aArg);
1233
1234 if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClearAll) {
1235 data->mFound = true;
1236 return PL_DHASH_STOP;
1237 }
1238
1239 if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClear &&
1240 data->mScope == aPendingOperation->Scope()) {
1241 data->mFound = true;
1242 return PL_DHASH_STOP;
1243 }
1244
1245 if (aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opClearMatchingScope &&
1246 StringBeginsWith(data->mScope, aPendingOperation->Scope())) {
1247 data->mFound = true;
1248 return PL_DHASH_STOP;
1249 }
1250
1251 return PL_DHASH_NEXT;
1252 }
1253
1254 } // anon
1255
1256 bool
1257 DOMStorageDBThread::PendingOperations::IsScopeClearPending(const nsACString& aScope)
1258 {
1259 // Called under the lock
1260
1261 FindPendingOperationForScopeData data(aScope);
1262 mClears.EnumerateRead(FindPendingClearForScope, &data);
1263 if (data.mFound) {
1264 return true;
1265 }
1266
1267 for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1268 DOMStorageDBThread::DBOperation* task = mExecList[i];
1269 FindPendingClearForScope(EmptyCString(), task, &data);
1270
1271 if (data.mFound) {
1272 return true;
1273 }
1274 }
1275
1276 return false;
1277 }
1278
1279 namespace { // anon
1280
1281 PLDHashOperator
1282 FindPendingUpdateForScope(const nsACString& aMapping,
1283 DOMStorageDBThread::DBOperation* aPendingOperation,
1284 void* aArg)
1285 {
1286 FindPendingOperationForScopeData* data =
1287 static_cast<FindPendingOperationForScopeData*>(aArg);
1288
1289 if ((aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opAddItem ||
1290 aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opUpdateItem ||
1291 aPendingOperation->Type() == DOMStorageDBThread::DBOperation::opRemoveItem) &&
1292 data->mScope == aPendingOperation->Scope()) {
1293 data->mFound = true;
1294 return PL_DHASH_STOP;
1295 }
1296
1297 return PL_DHASH_NEXT;
1298 }
1299
1300 } // anon
1301
1302 bool
1303 DOMStorageDBThread::PendingOperations::IsScopeUpdatePending(const nsACString& aScope)
1304 {
1305 // Called under the lock
1306
1307 FindPendingOperationForScopeData data(aScope);
1308 mUpdates.EnumerateRead(FindPendingUpdateForScope, &data);
1309 if (data.mFound) {
1310 return true;
1311 }
1312
1313 for (uint32_t i = 0; i < mExecList.Length(); ++i) {
1314 DOMStorageDBThread::DBOperation* task = mExecList[i];
1315 FindPendingUpdateForScope(EmptyCString(), task, &data);
1316
1317 if (data.mFound) {
1318 return true;
1319 }
1320 }
1321
1322 return false;
1323 }
1324
1325 } // ::dom
1326 } // ::mozilla

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial