toolkit/components/places/PlacesDBUtils.jsm@129ffea94266
toolkit/components/places/PlacesDBUtils.jsm
Sat, 03 Jan 2015 20:18:00 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Sat, 03 Jan 2015 20:18:00 +0100
- branch
- TOR_BUG_3246
- changeset 7
- 129ffea94266
- permissions
- -rw-r--r--
Conditionally enable double key logic according to:
private browsing mode or privacy.thirdparty.isolate preference and
implement in GetCookieStringCommon and FindCookie where it counts...
With some reservations of how to convince FindCookie users to test
condition and pass a nullptr when disabling double key logic.
1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: sw=2 ts=2 sts=2 expandtab filetype=javascript
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 const Cc = Components.classes;
8 const Ci = Components.interfaces;
9 const Cr = Components.results;
10 const Cu = Components.utils;
12 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
13 Cu.import("resource://gre/modules/Services.jsm");
14 Cu.import("resource://gre/modules/PlacesUtils.jsm");
16 this.EXPORTED_SYMBOLS = [ "PlacesDBUtils" ];
18 ////////////////////////////////////////////////////////////////////////////////
19 //// Constants
21 const FINISHED_MAINTENANCE_TOPIC = "places-maintenance-finished";
23 const BYTES_PER_MEBIBYTE = 1048576;
25 ////////////////////////////////////////////////////////////////////////////////
26 //// Smart getters
28 XPCOMUtils.defineLazyGetter(this, "DBConn", function() {
29 return PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection;
30 });
32 ////////////////////////////////////////////////////////////////////////////////
33 //// PlacesDBUtils
35 this.PlacesDBUtils = {
36 /**
37 * Executes a list of maintenance tasks.
38 * Once finished it will pass a array log to the callback attached to tasks.
39 * FINISHED_MAINTENANCE_TOPIC is notified through observer service on finish.
40 *
41 * @param aTasks
42 * Tasks object to execute.
43 */
44 _executeTasks: function PDBU__executeTasks(aTasks)
45 {
46 if (PlacesDBUtils._isShuttingDown) {
47 aTasks.log("- We are shutting down. Will not schedule the tasks.");
48 aTasks.clear();
49 }
51 let task = aTasks.pop();
52 if (task) {
53 task.call(PlacesDBUtils, aTasks);
54 }
55 else {
56 // All tasks have been completed.
57 // Telemetry the time it took for maintenance, if a start time exists.
58 if (aTasks._telemetryStart) {
59 Services.telemetry.getHistogramById("PLACES_IDLE_MAINTENANCE_TIME_MS")
60 .add(Date.now() - aTasks._telemetryStart);
61 aTasks._telemetryStart = 0;
62 }
64 if (aTasks.callback) {
65 let scope = aTasks.scope || Cu.getGlobalForObject(aTasks.callback);
66 aTasks.callback.call(scope, aTasks.messages);
67 }
69 // Notify observers that maintenance finished.
70 Services.prefs.setIntPref("places.database.lastMaintenance", parseInt(Date.now() / 1000));
71 Services.obs.notifyObservers(null, FINISHED_MAINTENANCE_TOPIC, null);
72 }
73 },
75 _isShuttingDown : false,
76 shutdown: function PDBU_shutdown() {
77 PlacesDBUtils._isShuttingDown = true;
78 },
80 /**
81 * Executes integrity check and common maintenance tasks.
82 *
83 * @param [optional] aCallback
84 * Callback to be invoked when done. The callback will get a array
85 * of log messages.
86 * @param [optional] aScope
87 * Scope for the callback.
88 */
89 maintenanceOnIdle: function PDBU_maintenanceOnIdle(aCallback, aScope)
90 {
91 let tasks = new Tasks([
92 this.checkIntegrity
93 , this.checkCoherence
94 , this._refreshUI
95 ]);
96 tasks._telemetryStart = Date.now();
97 tasks.callback = aCallback;
98 tasks.scope = aScope;
99 this._executeTasks(tasks);
100 },
102 /**
103 * Executes integrity check, common and advanced maintenance tasks (like
104 * expiration and vacuum). Will also collect statistics on the database.
105 *
106 * @param [optional] aCallback
107 * Callback to be invoked when done. The callback will get a array
108 * of log messages.
109 * @param [optional] aScope
110 * Scope for the callback.
111 */
112 checkAndFixDatabase: function PDBU_checkAndFixDatabase(aCallback, aScope)
113 {
114 let tasks = new Tasks([
115 this.checkIntegrity
116 , this.checkCoherence
117 , this.expire
118 , this.vacuum
119 , this.stats
120 , this._refreshUI
121 ]);
122 tasks.callback = aCallback;
123 tasks.scope = aScope;
124 this._executeTasks(tasks);
125 },
127 /**
128 * Forces a full refresh of Places views.
129 *
130 * @param [optional] aTasks
131 * Tasks object to execute.
132 */
133 _refreshUI: function PDBU__refreshUI(aTasks)
134 {
135 let tasks = new Tasks(aTasks);
137 // Send batch update notifications to update the UI.
138 PlacesUtils.history.runInBatchMode({
139 runBatched: function (aUserData) {}
140 }, null);
141 PlacesDBUtils._executeTasks(tasks);
142 },
144 _handleError: function PDBU__handleError(aError)
145 {
146 Cu.reportError("Async statement execution returned with '" +
147 aError.result + "', '" + aError.message + "'");
148 },
150 /**
151 * Tries to execute a REINDEX on the database.
152 *
153 * @param [optional] aTasks
154 * Tasks object to execute.
155 */
156 reindex: function PDBU_reindex(aTasks)
157 {
158 let tasks = new Tasks(aTasks);
159 tasks.log("> Reindex");
161 let stmt = DBConn.createAsyncStatement("REINDEX");
162 stmt.executeAsync({
163 handleError: PlacesDBUtils._handleError,
164 handleResult: function () {},
166 handleCompletion: function (aReason)
167 {
168 if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
169 tasks.log("+ The database has been reindexed");
170 }
171 else {
172 tasks.log("- Unable to reindex database");
173 }
175 PlacesDBUtils._executeTasks(tasks);
176 }
177 });
178 stmt.finalize();
179 },
181 /**
182 * Checks integrity but does not try to fix the database through a reindex.
183 *
184 * @param [optional] aTasks
185 * Tasks object to execute.
186 */
187 _checkIntegritySkipReindex: function PDBU__checkIntegritySkipReindex(aTasks)
188 this.checkIntegrity(aTasks, true),
190 /**
191 * Checks integrity and tries to fix the database through a reindex.
192 *
193 * @param [optional] aTasks
194 * Tasks object to execute.
195 * @param [optional] aSkipdReindex
196 * Whether to try to reindex database or not.
197 */
198 checkIntegrity: function PDBU_checkIntegrity(aTasks, aSkipReindex)
199 {
200 let tasks = new Tasks(aTasks);
201 tasks.log("> Integrity check");
203 // Run a integrity check, but stop at the first error.
204 let stmt = DBConn.createAsyncStatement("PRAGMA integrity_check(1)");
205 stmt.executeAsync({
206 handleError: PlacesDBUtils._handleError,
208 _corrupt: false,
209 handleResult: function (aResultSet)
210 {
211 let row = aResultSet.getNextRow();
212 this._corrupt = row.getResultByIndex(0) != "ok";
213 },
215 handleCompletion: function (aReason)
216 {
217 if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
218 if (this._corrupt) {
219 tasks.log("- The database is corrupt");
220 if (aSkipReindex) {
221 tasks.log("- Unable to fix corruption, database will be replaced on next startup");
222 Services.prefs.setBoolPref("places.database.replaceOnStartup", true);
223 tasks.clear();
224 }
225 else {
226 // Try to reindex, this often fixed simple indices corruption.
227 // We insert from the top of the queue, they will run inverse.
228 tasks.push(PlacesDBUtils._checkIntegritySkipReindex);
229 tasks.push(PlacesDBUtils.reindex);
230 }
231 }
232 else {
233 tasks.log("+ The database is sane");
234 }
235 }
236 else {
237 tasks.log("- Unable to check database status");
238 tasks.clear();
239 }
241 PlacesDBUtils._executeTasks(tasks);
242 }
243 });
244 stmt.finalize();
245 },
247 /**
248 * Checks data coherence and tries to fix most common errors.
249 *
250 * @param [optional] aTasks
251 * Tasks object to execute.
252 */
253 checkCoherence: function PDBU_checkCoherence(aTasks)
254 {
255 let tasks = new Tasks(aTasks);
256 tasks.log("> Coherence check");
258 let stmts = PlacesDBUtils._getBoundCoherenceStatements();
259 DBConn.executeAsync(stmts, stmts.length, {
260 handleError: PlacesDBUtils._handleError,
261 handleResult: function () {},
263 handleCompletion: function (aReason)
264 {
265 if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
266 tasks.log("+ The database is coherent");
267 }
268 else {
269 tasks.log("- Unable to check database coherence");
270 tasks.clear();
271 }
273 PlacesDBUtils._executeTasks(tasks);
274 }
275 });
276 stmts.forEach(function (aStmt) aStmt.finalize());
277 },
279 _getBoundCoherenceStatements: function PDBU__getBoundCoherenceStatements()
280 {
281 let cleanupStatements = [];
283 // MOZ_ANNO_ATTRIBUTES
284 // A.1 remove obsolete annotations from moz_annos.
285 // The 'weave0' idiom exploits character ordering (0 follows /) to
286 // efficiently select all annos with a 'weave/' prefix.
287 let deleteObsoleteAnnos = DBConn.createAsyncStatement(
288 "DELETE FROM moz_annos " +
289 "WHERE anno_attribute_id IN ( " +
290 " SELECT id FROM moz_anno_attributes " +
291 " WHERE name BETWEEN 'weave/' AND 'weave0' " +
292 ")");
293 cleanupStatements.push(deleteObsoleteAnnos);
295 // A.2 remove obsolete annotations from moz_items_annos.
296 let deleteObsoleteItemsAnnos = DBConn.createAsyncStatement(
297 "DELETE FROM moz_items_annos " +
298 "WHERE anno_attribute_id IN ( " +
299 " SELECT id FROM moz_anno_attributes " +
300 " WHERE name = 'sync/children' " +
301 " OR name = 'placesInternal/GUID' " +
302 " OR name BETWEEN 'weave/' AND 'weave0' " +
303 ")");
304 cleanupStatements.push(deleteObsoleteItemsAnnos);
306 // A.3 remove unused attributes.
307 let deleteUnusedAnnoAttributes = DBConn.createAsyncStatement(
308 "DELETE FROM moz_anno_attributes WHERE id IN ( " +
309 "SELECT id FROM moz_anno_attributes n " +
310 "WHERE NOT EXISTS " +
311 "(SELECT id FROM moz_annos WHERE anno_attribute_id = n.id LIMIT 1) " +
312 "AND NOT EXISTS " +
313 "(SELECT id FROM moz_items_annos WHERE anno_attribute_id = n.id LIMIT 1) " +
314 ")");
315 cleanupStatements.push(deleteUnusedAnnoAttributes);
317 // MOZ_ANNOS
318 // B.1 remove annos with an invalid attribute
319 let deleteInvalidAttributeAnnos = DBConn.createAsyncStatement(
320 "DELETE FROM moz_annos WHERE id IN ( " +
321 "SELECT id FROM moz_annos a " +
322 "WHERE NOT EXISTS " +
323 "(SELECT id FROM moz_anno_attributes " +
324 "WHERE id = a.anno_attribute_id LIMIT 1) " +
325 ")");
326 cleanupStatements.push(deleteInvalidAttributeAnnos);
328 // B.2 remove orphan annos
329 let deleteOrphanAnnos = DBConn.createAsyncStatement(
330 "DELETE FROM moz_annos WHERE id IN ( " +
331 "SELECT id FROM moz_annos a " +
332 "WHERE NOT EXISTS " +
333 "(SELECT id FROM moz_places WHERE id = a.place_id LIMIT 1) " +
334 ")");
335 cleanupStatements.push(deleteOrphanAnnos);
337 // MOZ_BOOKMARKS_ROOTS
338 // C.1 fix missing Places root
339 // Bug 477739 shows a case where the root could be wrongly removed
340 // due to an endianness issue. We try to fix broken roots here.
341 let selectPlacesRoot = DBConn.createStatement(
342 "SELECT id FROM moz_bookmarks WHERE id = :places_root");
343 selectPlacesRoot.params["places_root"] = PlacesUtils.placesRootId;
344 if (!selectPlacesRoot.executeStep()) {
345 // We are missing the root, try to recreate it.
346 let createPlacesRoot = DBConn.createAsyncStatement(
347 "INSERT INTO moz_bookmarks (id, type, fk, parent, position, title, "
348 + "guid) "
349 + "VALUES (:places_root, 2, NULL, 0, 0, :title, GENERATE_GUID())");
350 createPlacesRoot.params["places_root"] = PlacesUtils.placesRootId;
351 createPlacesRoot.params["title"] = "";
352 cleanupStatements.push(createPlacesRoot);
354 // Now ensure that other roots are children of Places root.
355 let fixPlacesRootChildren = DBConn.createAsyncStatement(
356 "UPDATE moz_bookmarks SET parent = :places_root WHERE id IN " +
357 "(SELECT folder_id FROM moz_bookmarks_roots " +
358 "WHERE folder_id <> :places_root)");
359 fixPlacesRootChildren.params["places_root"] = PlacesUtils.placesRootId;
360 cleanupStatements.push(fixPlacesRootChildren);
361 }
362 selectPlacesRoot.finalize();
364 // C.2 fix roots titles
365 // some alpha version has wrong roots title, and this also fixes them if
366 // locale has changed.
367 let updateRootTitleSql = "UPDATE moz_bookmarks SET title = :title " +
368 "WHERE id = :root_id AND title <> :title";
369 // root
370 let fixPlacesRootTitle = DBConn.createAsyncStatement(updateRootTitleSql);
371 fixPlacesRootTitle.params["root_id"] = PlacesUtils.placesRootId;
372 fixPlacesRootTitle.params["title"] = "";
373 cleanupStatements.push(fixPlacesRootTitle);
374 // bookmarks menu
375 let fixBookmarksMenuTitle = DBConn.createAsyncStatement(updateRootTitleSql);
376 fixBookmarksMenuTitle.params["root_id"] = PlacesUtils.bookmarksMenuFolderId;
377 fixBookmarksMenuTitle.params["title"] =
378 PlacesUtils.getString("BookmarksMenuFolderTitle");
379 cleanupStatements.push(fixBookmarksMenuTitle);
380 // bookmarks toolbar
381 let fixBookmarksToolbarTitle = DBConn.createAsyncStatement(updateRootTitleSql);
382 fixBookmarksToolbarTitle.params["root_id"] = PlacesUtils.toolbarFolderId;
383 fixBookmarksToolbarTitle.params["title"] =
384 PlacesUtils.getString("BookmarksToolbarFolderTitle");
385 cleanupStatements.push(fixBookmarksToolbarTitle);
386 // unsorted bookmarks
387 let fixUnsortedBookmarksTitle = DBConn.createAsyncStatement(updateRootTitleSql);
388 fixUnsortedBookmarksTitle.params["root_id"] = PlacesUtils.unfiledBookmarksFolderId;
389 fixUnsortedBookmarksTitle.params["title"] =
390 PlacesUtils.getString("UnsortedBookmarksFolderTitle");
391 cleanupStatements.push(fixUnsortedBookmarksTitle);
392 // tags
393 let fixTagsRootTitle = DBConn.createAsyncStatement(updateRootTitleSql);
394 fixTagsRootTitle.params["root_id"] = PlacesUtils.tagsFolderId;
395 fixTagsRootTitle.params["title"] =
396 PlacesUtils.getString("TagsFolderTitle");
397 cleanupStatements.push(fixTagsRootTitle);
399 // MOZ_BOOKMARKS
400 // D.1 remove items without a valid place
401 // if fk IS NULL we fix them in D.7
402 let deleteNoPlaceItems = DBConn.createAsyncStatement(
403 "DELETE FROM moz_bookmarks WHERE id NOT IN ( " +
404 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
405 ") AND id IN (" +
406 "SELECT b.id FROM moz_bookmarks b " +
407 "WHERE fk NOT NULL AND b.type = :bookmark_type " +
408 "AND NOT EXISTS (SELECT url FROM moz_places WHERE id = b.fk LIMIT 1) " +
409 ")");
410 deleteNoPlaceItems.params["bookmark_type"] = PlacesUtils.bookmarks.TYPE_BOOKMARK;
411 cleanupStatements.push(deleteNoPlaceItems);
413 // D.2 remove items that are not uri bookmarks from tag containers
414 let deleteBogusTagChildren = DBConn.createAsyncStatement(
415 "DELETE FROM moz_bookmarks WHERE id NOT IN ( " +
416 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
417 ") AND id IN (" +
418 "SELECT b.id FROM moz_bookmarks b " +
419 "WHERE b.parent IN " +
420 "(SELECT id FROM moz_bookmarks WHERE parent = :tags_folder) " +
421 "AND b.type <> :bookmark_type " +
422 ")");
423 deleteBogusTagChildren.params["tags_folder"] = PlacesUtils.tagsFolderId;
424 deleteBogusTagChildren.params["bookmark_type"] = PlacesUtils.bookmarks.TYPE_BOOKMARK;
425 cleanupStatements.push(deleteBogusTagChildren);
427 // D.3 remove empty tags
428 let deleteEmptyTags = DBConn.createAsyncStatement(
429 "DELETE FROM moz_bookmarks WHERE id NOT IN ( " +
430 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
431 ") AND id IN (" +
432 "SELECT b.id FROM moz_bookmarks b " +
433 "WHERE b.id IN " +
434 "(SELECT id FROM moz_bookmarks WHERE parent = :tags_folder) " +
435 "AND NOT EXISTS " +
436 "(SELECT id from moz_bookmarks WHERE parent = b.id LIMIT 1) " +
437 ")");
438 deleteEmptyTags.params["tags_folder"] = PlacesUtils.tagsFolderId;
439 cleanupStatements.push(deleteEmptyTags);
441 // D.4 move orphan items to unsorted folder
442 let fixOrphanItems = DBConn.createAsyncStatement(
443 "UPDATE moz_bookmarks SET parent = :unsorted_folder WHERE id NOT IN ( " +
444 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
445 ") AND id IN (" +
446 "SELECT b.id FROM moz_bookmarks b " +
447 "WHERE b.parent <> 0 " + // exclude Places root
448 "AND NOT EXISTS " +
449 "(SELECT id FROM moz_bookmarks WHERE id = b.parent LIMIT 1) " +
450 ")");
451 fixOrphanItems.params["unsorted_folder"] = PlacesUtils.unfiledBookmarksFolderId;
452 cleanupStatements.push(fixOrphanItems);
454 // D.5 fix wrong keywords
455 let fixInvalidKeywords = DBConn.createAsyncStatement(
456 "UPDATE moz_bookmarks SET keyword_id = NULL WHERE id NOT IN ( " +
457 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
458 ") AND id IN ( " +
459 "SELECT id FROM moz_bookmarks b " +
460 "WHERE keyword_id NOT NULL " +
461 "AND NOT EXISTS " +
462 "(SELECT id FROM moz_keywords WHERE id = b.keyword_id LIMIT 1) " +
463 ")");
464 cleanupStatements.push(fixInvalidKeywords);
466 // D.6 fix wrong item types
467 // Folders and separators should not have an fk.
468 // If they have a valid fk convert them to bookmarks. Later in D.9 we
469 // will move eventual children to unsorted bookmarks.
470 let fixBookmarksAsFolders = DBConn.createAsyncStatement(
471 "UPDATE moz_bookmarks SET type = :bookmark_type WHERE id NOT IN ( " +
472 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
473 ") AND id IN ( " +
474 "SELECT id FROM moz_bookmarks b " +
475 "WHERE type IN (:folder_type, :separator_type) " +
476 "AND fk NOTNULL " +
477 ")");
478 fixBookmarksAsFolders.params["bookmark_type"] = PlacesUtils.bookmarks.TYPE_BOOKMARK;
479 fixBookmarksAsFolders.params["folder_type"] = PlacesUtils.bookmarks.TYPE_FOLDER;
480 fixBookmarksAsFolders.params["separator_type"] = PlacesUtils.bookmarks.TYPE_SEPARATOR;
481 cleanupStatements.push(fixBookmarksAsFolders);
483 // D.7 fix wrong item types
484 // Bookmarks should have an fk, if they don't have any, convert them to
485 // folders.
486 let fixFoldersAsBookmarks = DBConn.createAsyncStatement(
487 "UPDATE moz_bookmarks SET type = :folder_type WHERE id NOT IN ( " +
488 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
489 ") AND id IN ( " +
490 "SELECT id FROM moz_bookmarks b " +
491 "WHERE type = :bookmark_type " +
492 "AND fk IS NULL " +
493 ")");
494 fixFoldersAsBookmarks.params["bookmark_type"] = PlacesUtils.bookmarks.TYPE_BOOKMARK;
495 fixFoldersAsBookmarks.params["folder_type"] = PlacesUtils.bookmarks.TYPE_FOLDER;
496 cleanupStatements.push(fixFoldersAsBookmarks);
498 // D.9 fix wrong parents
499 // Items cannot have separators or other bookmarks
500 // as parent, if they have bad parent move them to unsorted bookmarks.
501 let fixInvalidParents = DBConn.createAsyncStatement(
502 "UPDATE moz_bookmarks SET parent = :unsorted_folder WHERE id NOT IN ( " +
503 "SELECT folder_id FROM moz_bookmarks_roots " + // skip roots
504 ") AND id IN ( " +
505 "SELECT id FROM moz_bookmarks b " +
506 "WHERE EXISTS " +
507 "(SELECT id FROM moz_bookmarks WHERE id = b.parent " +
508 "AND type IN (:bookmark_type, :separator_type) " +
509 "LIMIT 1) " +
510 ")");
511 fixInvalidParents.params["unsorted_folder"] = PlacesUtils.unfiledBookmarksFolderId;
512 fixInvalidParents.params["bookmark_type"] = PlacesUtils.bookmarks.TYPE_BOOKMARK;
513 fixInvalidParents.params["separator_type"] = PlacesUtils.bookmarks.TYPE_SEPARATOR;
514 cleanupStatements.push(fixInvalidParents);
516 // D.10 recalculate positions
517 // This requires multiple related statements.
518 // We can detect a folder with bad position values comparing the sum of
519 // all distinct position values (+1 since position is 0-based) with the
520 // triangular numbers obtained by the number of children (n).
521 // SUM(DISTINCT position + 1) == (n * (n + 1) / 2).
522 cleanupStatements.push(DBConn.createAsyncStatement(
523 "CREATE TEMP TABLE IF NOT EXISTS moz_bm_reindex_temp ( " +
524 " id INTEGER PRIMARY_KEY " +
525 ", parent INTEGER " +
526 ", position INTEGER " +
527 ") "
528 ));
529 cleanupStatements.push(DBConn.createAsyncStatement(
530 "INSERT INTO moz_bm_reindex_temp " +
531 "SELECT id, parent, 0 " +
532 "FROM moz_bookmarks b " +
533 "WHERE parent IN ( " +
534 "SELECT parent " +
535 "FROM moz_bookmarks " +
536 "GROUP BY parent " +
537 "HAVING (SUM(DISTINCT position + 1) - (count(*) * (count(*) + 1) / 2)) <> 0 " +
538 ") " +
539 "ORDER BY parent ASC, position ASC, ROWID ASC "
540 ));
541 cleanupStatements.push(DBConn.createAsyncStatement(
542 "CREATE INDEX IF NOT EXISTS moz_bm_reindex_temp_index " +
543 "ON moz_bm_reindex_temp(parent)"
544 ));
545 cleanupStatements.push(DBConn.createAsyncStatement(
546 "UPDATE moz_bm_reindex_temp SET position = ( " +
547 "ROWID - (SELECT MIN(t.ROWID) FROM moz_bm_reindex_temp t " +
548 "WHERE t.parent = moz_bm_reindex_temp.parent) " +
549 ") "
550 ));
551 cleanupStatements.push(DBConn.createAsyncStatement(
552 "CREATE TEMP TRIGGER IF NOT EXISTS moz_bm_reindex_temp_trigger " +
553 "BEFORE DELETE ON moz_bm_reindex_temp " +
554 "FOR EACH ROW " +
555 "BEGIN " +
556 "UPDATE moz_bookmarks SET position = OLD.position WHERE id = OLD.id; " +
557 "END "
558 ));
559 cleanupStatements.push(DBConn.createAsyncStatement(
560 "DELETE FROM moz_bm_reindex_temp "
561 ));
562 cleanupStatements.push(DBConn.createAsyncStatement(
563 "DROP INDEX moz_bm_reindex_temp_index "
564 ));
565 cleanupStatements.push(DBConn.createAsyncStatement(
566 "DROP TRIGGER moz_bm_reindex_temp_trigger "
567 ));
568 cleanupStatements.push(DBConn.createAsyncStatement(
569 "DROP TABLE moz_bm_reindex_temp "
570 ));
572 // D.12 Fix empty-named tags.
573 // Tags were allowed to have empty names due to a UI bug. Fix them
574 // replacing their title with "(notitle)".
575 let fixEmptyNamedTags = DBConn.createAsyncStatement(
576 "UPDATE moz_bookmarks SET title = :empty_title " +
577 "WHERE length(title) = 0 AND type = :folder_type " +
578 "AND parent = :tags_folder"
579 );
580 fixEmptyNamedTags.params["empty_title"] = "(notitle)";
581 fixEmptyNamedTags.params["folder_type"] = PlacesUtils.bookmarks.TYPE_FOLDER;
582 fixEmptyNamedTags.params["tags_folder"] = PlacesUtils.tagsFolderId;
583 cleanupStatements.push(fixEmptyNamedTags);
585 // MOZ_FAVICONS
586 // E.1 remove orphan icons
587 let deleteOrphanIcons = DBConn.createAsyncStatement(
588 "DELETE FROM moz_favicons WHERE id IN (" +
589 "SELECT id FROM moz_favicons f " +
590 "WHERE NOT EXISTS " +
591 "(SELECT id FROM moz_places WHERE favicon_id = f.id LIMIT 1) " +
592 ")");
593 cleanupStatements.push(deleteOrphanIcons);
595 // MOZ_HISTORYVISITS
596 // F.1 remove orphan visits
597 let deleteOrphanVisits = DBConn.createAsyncStatement(
598 "DELETE FROM moz_historyvisits WHERE id IN (" +
599 "SELECT id FROM moz_historyvisits v " +
600 "WHERE NOT EXISTS " +
601 "(SELECT id FROM moz_places WHERE id = v.place_id LIMIT 1) " +
602 ")");
603 cleanupStatements.push(deleteOrphanVisits);
605 // MOZ_INPUTHISTORY
606 // G.1 remove orphan input history
607 let deleteOrphanInputHistory = DBConn.createAsyncStatement(
608 "DELETE FROM moz_inputhistory WHERE place_id IN (" +
609 "SELECT place_id FROM moz_inputhistory i " +
610 "WHERE NOT EXISTS " +
611 "(SELECT id FROM moz_places WHERE id = i.place_id LIMIT 1) " +
612 ")");
613 cleanupStatements.push(deleteOrphanInputHistory);
615 // MOZ_ITEMS_ANNOS
616 // H.1 remove item annos with an invalid attribute
617 let deleteInvalidAttributeItemsAnnos = DBConn.createAsyncStatement(
618 "DELETE FROM moz_items_annos WHERE id IN ( " +
619 "SELECT id FROM moz_items_annos t " +
620 "WHERE NOT EXISTS " +
621 "(SELECT id FROM moz_anno_attributes " +
622 "WHERE id = t.anno_attribute_id LIMIT 1) " +
623 ")");
624 cleanupStatements.push(deleteInvalidAttributeItemsAnnos);
626 // H.2 remove orphan item annos
627 let deleteOrphanItemsAnnos = DBConn.createAsyncStatement(
628 "DELETE FROM moz_items_annos WHERE id IN ( " +
629 "SELECT id FROM moz_items_annos t " +
630 "WHERE NOT EXISTS " +
631 "(SELECT id FROM moz_bookmarks WHERE id = t.item_id LIMIT 1) " +
632 ")");
633 cleanupStatements.push(deleteOrphanItemsAnnos);
635 // MOZ_KEYWORDS
636 // I.1 remove unused keywords
637 let deleteUnusedKeywords = DBConn.createAsyncStatement(
638 "DELETE FROM moz_keywords WHERE id IN ( " +
639 "SELECT id FROM moz_keywords k " +
640 "WHERE NOT EXISTS " +
641 "(SELECT id FROM moz_bookmarks WHERE keyword_id = k.id LIMIT 1) " +
642 ")");
643 cleanupStatements.push(deleteUnusedKeywords);
645 // MOZ_PLACES
646 // L.1 fix wrong favicon ids
647 let fixInvalidFaviconIds = DBConn.createAsyncStatement(
648 "UPDATE moz_places SET favicon_id = NULL WHERE id IN ( " +
649 "SELECT id FROM moz_places h " +
650 "WHERE favicon_id NOT NULL " +
651 "AND NOT EXISTS " +
652 "(SELECT id FROM moz_favicons WHERE id = h.favicon_id LIMIT 1) " +
653 ")");
654 cleanupStatements.push(fixInvalidFaviconIds);
656 // L.2 recalculate visit_count and last_visit_date
657 let fixVisitStats = DBConn.createAsyncStatement(
658 "UPDATE moz_places " +
659 "SET visit_count = (SELECT count(*) FROM moz_historyvisits " +
660 "WHERE place_id = moz_places.id AND visit_type NOT IN (0,4,7,8)), " +
661 "last_visit_date = (SELECT MAX(visit_date) FROM moz_historyvisits " +
662 "WHERE place_id = moz_places.id) " +
663 "WHERE id IN ( " +
664 "SELECT h.id FROM moz_places h " +
665 "WHERE visit_count <> (SELECT count(*) FROM moz_historyvisits v " +
666 "WHERE v.place_id = h.id AND visit_type NOT IN (0,4,7,8)) " +
667 "OR last_visit_date <> (SELECT MAX(visit_date) FROM moz_historyvisits v " +
668 "WHERE v.place_id = h.id) " +
669 ")");
670 cleanupStatements.push(fixVisitStats);
672 // L.3 recalculate hidden for redirects.
673 let fixRedirectsHidden = DBConn.createAsyncStatement(
674 "UPDATE moz_places " +
675 "SET hidden = 1 " +
676 "WHERE id IN ( " +
677 "SELECT h.id FROM moz_places h " +
678 "JOIN moz_historyvisits src ON src.place_id = h.id " +
679 "JOIN moz_historyvisits dst ON dst.from_visit = src.id AND dst.visit_type IN (5,6) " +
680 "LEFT JOIN moz_bookmarks on fk = h.id AND fk ISNULL " +
681 "GROUP BY src.place_id HAVING count(*) = visit_count " +
682 ")");
683 cleanupStatements.push(fixRedirectsHidden);
685 // MAINTENANCE STATEMENTS SHOULD GO ABOVE THIS POINT!
687 return cleanupStatements;
688 },
690 /**
691 * Tries to vacuum the database.
692 *
693 * @param [optional] aTasks
694 * Tasks object to execute.
695 */
696 vacuum: function PDBU_vacuum(aTasks)
697 {
698 let tasks = new Tasks(aTasks);
699 tasks.log("> Vacuum");
701 let DBFile = Services.dirsvc.get("ProfD", Ci.nsILocalFile);
702 DBFile.append("places.sqlite");
703 tasks.log("Initial database size is " +
704 parseInt(DBFile.fileSize / 1024) + " KiB");
706 let stmt = DBConn.createAsyncStatement("VACUUM");
707 stmt.executeAsync({
708 handleError: PlacesDBUtils._handleError,
709 handleResult: function () {},
711 handleCompletion: function (aReason)
712 {
713 if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
714 tasks.log("+ The database has been vacuumed");
715 let vacuumedDBFile = Services.dirsvc.get("ProfD", Ci.nsILocalFile);
716 vacuumedDBFile.append("places.sqlite");
717 tasks.log("Final database size is " +
718 parseInt(vacuumedDBFile.fileSize / 1024) + " KiB");
719 }
720 else {
721 tasks.log("- Unable to vacuum database");
722 tasks.clear();
723 }
725 PlacesDBUtils._executeTasks(tasks);
726 }
727 });
728 stmt.finalize();
729 },
731 /**
732 * Forces a full expiration on the database.
733 *
734 * @param [optional] aTasks
735 * Tasks object to execute.
736 */
737 expire: function PDBU_expire(aTasks)
738 {
739 let tasks = new Tasks(aTasks);
740 tasks.log("> Orphans expiration");
742 let expiration = Cc["@mozilla.org/places/expiration;1"].
743 getService(Ci.nsIObserver);
745 Services.obs.addObserver(function (aSubject, aTopic, aData) {
746 Services.obs.removeObserver(arguments.callee, aTopic);
747 tasks.log("+ Database cleaned up");
748 PlacesDBUtils._executeTasks(tasks);
749 }, PlacesUtils.TOPIC_EXPIRATION_FINISHED, false);
751 // Force an orphans expiration step.
752 expiration.observe(null, "places-debug-start-expiration", 0);
753 },
755 /**
756 * Collects statistical data on the database.
757 *
758 * @param [optional] aTasks
759 * Tasks object to execute.
760 */
761 stats: function PDBU_stats(aTasks)
762 {
763 let tasks = new Tasks(aTasks);
764 tasks.log("> Statistics");
766 let DBFile = Services.dirsvc.get("ProfD", Ci.nsILocalFile);
767 DBFile.append("places.sqlite");
768 tasks.log("Database size is " + parseInt(DBFile.fileSize / 1024) + " KiB");
770 [ "user_version"
771 , "page_size"
772 , "cache_size"
773 , "journal_mode"
774 , "synchronous"
775 ].forEach(function (aPragma) {
776 let stmt = DBConn.createStatement("PRAGMA " + aPragma);
777 stmt.executeStep();
778 tasks.log(aPragma + " is " + stmt.getString(0));
779 stmt.finalize();
780 });
782 // Get maximum number of unique URIs.
783 try {
784 let limitURIs = Services.prefs.getIntPref(
785 "places.history.expiration.transient_current_max_pages");
786 tasks.log("History can store a maximum of " + limitURIs + " unique pages");
787 } catch(ex) {}
789 let stmt = DBConn.createStatement(
790 "SELECT name FROM sqlite_master WHERE type = :type");
791 stmt.params.type = "table";
792 while (stmt.executeStep()) {
793 let tableName = stmt.getString(0);
794 let countStmt = DBConn.createStatement(
795 "SELECT count(*) FROM " + tableName);
796 countStmt.executeStep();
797 tasks.log("Table " + tableName + " has " + countStmt.getInt32(0) + " records");
798 countStmt.finalize();
799 }
800 stmt.reset();
802 stmt.params.type = "index";
803 while (stmt.executeStep()) {
804 tasks.log("Index " + stmt.getString(0));
805 }
806 stmt.reset();
808 stmt.params.type = "trigger";
809 while (stmt.executeStep()) {
810 tasks.log("Trigger " + stmt.getString(0));
811 }
812 stmt.finalize();
814 PlacesDBUtils._executeTasks(tasks);
815 },
817 /**
818 * Collects telemetry data.
819 *
820 * There are essentially two modes of collection and the mode is
821 * determined by the presence of aHealthReportCallback. If
822 * aHealthReportCallback is not defined (the default) then we are in
823 * "Telemetry" mode. Results will be reported to Telemetry. If we are
824 * in "Health Report" mode only the probes with a true healthreport
825 * flag will be collected and the results will be reported to the
826 * aHealthReportCallback.
827 *
828 * @param [optional] aTasks
829 * Tasks object to execute.
830 * @param [optional] aHealthReportCallback
831 * Function to receive data relevant for Firefox Health Report.
832 */
833 telemetry: function PDBU_telemetry(aTasks, aHealthReportCallback=null)
834 {
835 let tasks = new Tasks(aTasks);
837 let isTelemetry = !aHealthReportCallback;
839 // This will be populated with one integer property for each probe result,
840 // using the histogram name as key.
841 let probeValues = {};
843 // The following array contains an ordered list of entries that are
844 // processed to collect telemetry data. Each entry has these properties:
845 //
846 // histogram: Name of the telemetry histogram to update.
847 // query: This is optional. If present, contains a database command
848 // that will be executed asynchronously, and whose result will
849 // be added to the telemetry histogram.
850 // callback: This is optional. If present, contains a function that must
851 // return the value that will be added to the telemetry
852 // histogram. If a query is also present, its result is passed
853 // as the first argument of the function. If the function
854 // raises an exception, no data is added to the histogram.
855 // healthreport: Boolean indicating whether this probe is relevant
856 // to Firefox Health Report.
857 //
858 // Since all queries are executed in order by the database backend, the
859 // callbacks can also use the result of previous queries stored in the
860 // probeValues object.
861 let probes = [
862 { histogram: "PLACES_PAGES_COUNT",
863 healthreport: true,
864 query: "SELECT count(*) FROM moz_places" },
866 { histogram: "PLACES_BOOKMARKS_COUNT",
867 healthreport: true,
868 query: "SELECT count(*) FROM moz_bookmarks b "
869 + "JOIN moz_bookmarks t ON t.id = b.parent "
870 + "AND t.parent <> :tags_folder "
871 + "WHERE b.type = :type_bookmark " },
873 { histogram: "PLACES_TAGS_COUNT",
874 query: "SELECT count(*) FROM moz_bookmarks "
875 + "WHERE parent = :tags_folder " },
877 { histogram: "PLACES_FOLDERS_COUNT",
878 query: "SELECT count(*) FROM moz_bookmarks "
879 + "WHERE TYPE = :type_folder "
880 + "AND parent NOT IN (0, :places_root, :tags_folder) " },
882 { histogram: "PLACES_KEYWORDS_COUNT",
883 query: "SELECT count(*) FROM moz_keywords " },
885 { histogram: "PLACES_SORTED_BOOKMARKS_PERC",
886 query: "SELECT IFNULL(ROUND(( "
887 + "SELECT count(*) FROM moz_bookmarks b "
888 + "JOIN moz_bookmarks t ON t.id = b.parent "
889 + "AND t.parent <> :tags_folder AND t.parent > :places_root "
890 + "WHERE b.type = :type_bookmark "
891 + ") * 100 / ( "
892 + "SELECT count(*) FROM moz_bookmarks b "
893 + "JOIN moz_bookmarks t ON t.id = b.parent "
894 + "AND t.parent <> :tags_folder "
895 + "WHERE b.type = :type_bookmark "
896 + ")), 0) " },
898 { histogram: "PLACES_TAGGED_BOOKMARKS_PERC",
899 query: "SELECT IFNULL(ROUND(( "
900 + "SELECT count(*) FROM moz_bookmarks b "
901 + "JOIN moz_bookmarks t ON t.id = b.parent "
902 + "AND t.parent = :tags_folder "
903 + ") * 100 / ( "
904 + "SELECT count(*) FROM moz_bookmarks b "
905 + "JOIN moz_bookmarks t ON t.id = b.parent "
906 + "AND t.parent <> :tags_folder "
907 + "WHERE b.type = :type_bookmark "
908 + ")), 0) " },
910 { histogram: "PLACES_DATABASE_FILESIZE_MB",
911 callback: function () {
912 let DBFile = Services.dirsvc.get("ProfD", Ci.nsILocalFile);
913 DBFile.append("places.sqlite");
914 return parseInt(DBFile.fileSize / BYTES_PER_MEBIBYTE);
915 }
916 },
918 { histogram: "PLACES_DATABASE_JOURNALSIZE_MB",
919 callback: function () {
920 let DBFile = Services.dirsvc.get("ProfD", Ci.nsILocalFile);
921 DBFile.append("places.sqlite-wal");
922 return parseInt(DBFile.fileSize / BYTES_PER_MEBIBYTE);
923 }
924 },
926 { histogram: "PLACES_DATABASE_PAGESIZE_B",
927 query: "PRAGMA page_size /* PlacesDBUtils.jsm PAGESIZE_B */" },
929 { histogram: "PLACES_DATABASE_SIZE_PER_PAGE_B",
930 query: "PRAGMA page_count",
931 callback: function (aDbPageCount) {
932 // Note that the database file size would not be meaningful for this
933 // calculation, because the file grows in fixed-size chunks.
934 let dbPageSize = probeValues.PLACES_DATABASE_PAGESIZE_B;
935 let placesPageCount = probeValues.PLACES_PAGES_COUNT;
936 return Math.round((dbPageSize * aDbPageCount) / placesPageCount);
937 }
938 },
940 { histogram: "PLACES_ANNOS_BOOKMARKS_COUNT",
941 query: "SELECT count(*) FROM moz_items_annos" },
943 // LENGTH is not a perfect measure, since it returns the number of bytes
944 // only for BLOBs, the number of chars for anything else. Though it's
945 // the best approximation we have.
946 { histogram: "PLACES_ANNOS_BOOKMARKS_SIZE_KB",
947 query: "SELECT SUM(LENGTH(content))/1024 FROM moz_items_annos" },
949 { histogram: "PLACES_ANNOS_PAGES_COUNT",
950 query: "SELECT count(*) FROM moz_annos" },
952 { histogram: "PLACES_ANNOS_PAGES_SIZE_KB",
953 query: "SELECT SUM(LENGTH(content))/1024 FROM moz_annos" },
954 ];
956 let params = {
957 tags_folder: PlacesUtils.tagsFolderId,
958 type_folder: PlacesUtils.bookmarks.TYPE_FOLDER,
959 type_bookmark: PlacesUtils.bookmarks.TYPE_BOOKMARK,
960 places_root: PlacesUtils.placesRootId
961 };
963 let outstandingProbes = 0;
965 function reportResult(aProbe, aValue) {
966 outstandingProbes--;
968 let value = aValue;
969 try {
970 if ("callback" in aProbe) {
971 value = aProbe.callback(value);
972 }
973 probeValues[aProbe.histogram] = value;
974 Services.telemetry.getHistogramById(aProbe.histogram).add(value);
975 } catch (ex) {
976 Components.utils.reportError("Error adding value " + value +
977 " to histogram " + aProbe.histogram +
978 ": " + ex);
979 }
981 if (!outstandingProbes && aHealthReportCallback) {
982 try {
983 aHealthReportCallback(probeValues);
984 } catch (ex) {
985 Components.utils.reportError(ex);
986 }
987 }
988 }
990 for (let i = 0; i < probes.length; i++) {
991 let probe = probes[i];
993 if (!isTelemetry && !probe.healthreport) {
994 continue;
995 }
997 outstandingProbes++;
999 if (!("query" in probe)) {
1000 reportResult(probe);
1001 continue;
1002 }
1004 let stmt = DBConn.createAsyncStatement(probe.query);
1005 for (param in params) {
1006 if (probe.query.indexOf(":" + param) > 0) {
1007 stmt.params[param] = params[param];
1008 }
1009 }
1011 try {
1012 stmt.executeAsync({
1013 handleError: PlacesDBUtils._handleError,
1014 handleResult: function (aResultSet) {
1015 let row = aResultSet.getNextRow();
1016 reportResult(probe, row.getResultByIndex(0));
1017 },
1018 handleCompletion: function () {}
1019 });
1020 } finally{
1021 stmt.finalize();
1022 }
1023 }
1025 PlacesDBUtils._executeTasks(tasks);
1026 },
1028 /**
1029 * Runs a list of tasks, notifying log messages to the callback.
1030 *
1031 * @param aTasks
1032 * Array of tasks to be executed, in form of pointers to methods in
1033 * this module.
1034 * @param [optional] aCallback
1035 * Callback to be invoked when done. It will receive an array of
1036 * log messages.
1037 */
1038 runTasks: function PDBU_runTasks(aTasks, aCallback) {
1039 let tasks = new Tasks(aTasks);
1040 tasks.callback = aCallback;
1041 PlacesDBUtils._executeTasks(tasks);
1042 }
1043 };
1045 /**
1046 * LIFO tasks stack.
1047 *
1048 * @param [optional] aTasks
1049 * Array of tasks or another Tasks object to clone.
1050 */
1051 function Tasks(aTasks)
1052 {
1053 if (aTasks) {
1054 if (Array.isArray(aTasks)) {
1055 this._list = aTasks.slice(0, aTasks.length);
1056 }
1057 // This supports passing in a Tasks-like object, with a "list" property,
1058 // for compatibility reasons.
1059 else if (typeof(aTasks) == "object" &&
1060 (Tasks instanceof Tasks || "list" in aTasks)) {
1061 this._list = aTasks.list;
1062 this._log = aTasks.messages;
1063 this.callback = aTasks.callback;
1064 this.scope = aTasks.scope;
1065 this._telemetryStart = aTasks._telemetryStart;
1066 }
1067 }
1068 }
1070 Tasks.prototype = {
1071 _list: [],
1072 _log: [],
1073 callback: null,
1074 scope: null,
1075 _telemetryStart: 0,
1077 /**
1078 * Adds a task to the top of the list.
1079 *
1080 * @param aNewElt
1081 * Task to be added.
1082 */
1083 push: function T_push(aNewElt)
1084 {
1085 this._list.unshift(aNewElt);
1086 },
1088 /**
1089 * Returns and consumes next task.
1090 *
1091 * @return next task or undefined if no task is left.
1092 */
1093 pop: function T_pop() this._list.shift(),
1095 /**
1096 * Removes all tasks.
1097 */
1098 clear: function T_clear()
1099 {
1100 this._list.length = 0;
1101 },
1103 /**
1104 * Returns array of tasks ordered from the next to be run to the latest.
1105 */
1106 get list() this._list.slice(0, this._list.length),
1108 /**
1109 * Adds a message to the log.
1110 *
1111 * @param aMsg
1112 * String message to be added.
1113 */
1114 log: function T_log(aMsg)
1115 {
1116 this._log.push(aMsg);
1117 },
1119 /**
1120 * Returns array of log messages ordered from oldest to newest.
1121 */
1122 get messages() this._log.slice(0, this._log.length),
1123 }
