michael@0: /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
michael@0:  * vim: sw=2 ts=2 sts=2 expandtab filetype=javascript
michael@0:  * This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0:  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: this.EXPORTED_SYMBOLS = ["PlacesBackups"];
michael@0: 
michael@0: const Ci = Components.interfaces;
michael@0: const Cu = Components.utils;
michael@0: const Cc = Components.classes;
michael@0: 
michael@0: Cu.import("resource://gre/modules/XPCOMUtils.jsm");
michael@0: Cu.import("resource://gre/modules/Services.jsm");
michael@0: Cu.import("resource://gre/modules/PlacesUtils.jsm");
michael@0: Cu.import("resource://gre/modules/Task.jsm");
michael@0: Cu.import("resource://gre/modules/osfile.jsm");
michael@0: Cu.import("resource://gre/modules/NetUtil.jsm");
michael@0: 
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "BookmarkJSONUtils",
michael@0:   "resource://gre/modules/BookmarkJSONUtils.jsm");
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "Deprecated",
michael@0:   "resource://gre/modules/Deprecated.jsm");
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "OS",
michael@0:   "resource://gre/modules/osfile.jsm");
michael@0: XPCOMUtils.defineLazyModuleGetter(this, "Sqlite",
michael@0:   "resource://gre/modules/Sqlite.jsm");
michael@0: 
michael@0: XPCOMUtils.defineLazyGetter(this, "localFileCtor",
michael@0:   () => Components.Constructor("@mozilla.org/file/local;1",
michael@0:                                "nsILocalFile", "initWithPath"));
michael@0: 
michael@0: XPCOMUtils.defineLazyGetter(this, "filenamesRegex",
michael@0:   () => new RegExp("^bookmarks-([0-9\-]+)(?:_([0-9]+)){0,1}(?:_([a-z0-9=\+\-]{24})){0,1}\.(json|html)", "i")
michael@0: );
michael@0: 
michael@0: /**
michael@0:  * Appends meta-data information to a given filename.
michael@0:  */
michael@0: function appendMetaDataToFilename(aFilename, aMetaData) {
michael@0:   let matches = aFilename.match(filenamesRegex);
michael@0:   return "bookmarks-" + matches[1] +
michael@0:                   "_" + aMetaData.count +
michael@0:                   "_" + aMetaData.hash +
michael@0:                   "." + matches[4];
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Gets the hash from a backup filename.
michael@0:  *
michael@0:  * @return the extracted hash or null.
michael@0:  */
michael@0: function getHashFromFilename(aFilename) {
michael@0:   let matches = aFilename.match(filenamesRegex);
michael@0:   if (matches && matches[3])
michael@0:     return matches[3];
michael@0:   return null;
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Given two filenames, checks if they contain the same date.
michael@0:  */
michael@0: function isFilenameWithSameDate(aSourceName, aTargetName) {
michael@0:   let sourceMatches = aSourceName.match(filenamesRegex);
michael@0:   let targetMatches = aTargetName.match(filenamesRegex);
michael@0: 
michael@0:   return sourceMatches && targetMatches &&
michael@0:          sourceMatches[1] == targetMatches[1] &&
michael@0:          sourceMatches[4] == targetMatches[4];
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Given a filename, searches for another backup with the same date.
michael@0:  *
michael@0:  * @return OS.File path string or null.
michael@0:  */
michael@0: function getBackupFileForSameDate(aFilename) {
michael@0:   return Task.spawn(function* () {
michael@0:     let backupFiles = yield PlacesBackups.getBackupFiles();
michael@0:     for (let backupFile of backupFiles) {
michael@0:       if (isFilenameWithSameDate(OS.Path.basename(backupFile), aFilename))
michael@0:         return backupFile;
michael@0:     }
michael@0:     return null;
michael@0:   });
michael@0: }
michael@0: 
michael@0: this.PlacesBackups = {
michael@0:   /**
michael@0:    * Matches the backup filename:
michael@0:    *  0: file name
michael@0:    *  1: date in form Y-m-d
michael@0:    *  2: bookmarks count
michael@0:    *  3: contents hash
michael@0:    *  4: file extension
michael@0:    */
michael@0:   get filenamesRegex() filenamesRegex,
michael@0: 
michael@0:   get folder() {
michael@0:     Deprecated.warning(
michael@0:       "PlacesBackups.folder is deprecated and will be removed in a future version",
michael@0:       "https://bugzilla.mozilla.org/show_bug.cgi?id=859695");
michael@0:     return this._folder;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * This exists just to avoid spamming deprecate warnings from internal calls
michael@0:    * needed to support deprecated methods themselves.
michael@0:    */
michael@0:   get _folder() {
michael@0:     let bookmarksBackupDir = Services.dirsvc.get("ProfD", Ci.nsILocalFile);
michael@0:     bookmarksBackupDir.append(this.profileRelativeFolderPath);
michael@0:     if (!bookmarksBackupDir.exists()) {
michael@0:       bookmarksBackupDir.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0700", 8));
michael@0:       if (!bookmarksBackupDir.exists())
michael@0:         throw("Unable to create bookmarks backup folder");
michael@0:     }
michael@0:     delete this._folder;
michael@0:     return this._folder = bookmarksBackupDir;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Gets backup folder asynchronously.
michael@0:    * @return {Promise}
michael@0:    * @resolve the folder (the folder string path).
michael@0:    */
michael@0:   getBackupFolder: function PB_getBackupFolder() {
michael@0:     return Task.spawn(function* () {
michael@0:       if (this._backupFolder) {
michael@0:         return this._backupFolder;
michael@0:       }
michael@0:       let profileDir = OS.Constants.Path.profileDir;
michael@0:       let backupsDirPath = OS.Path.join(profileDir, this.profileRelativeFolderPath);
michael@0:       yield OS.File.makeDir(backupsDirPath, { ignoreExisting: true });
michael@0:       return this._backupFolder = backupsDirPath;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   get profileRelativeFolderPath() "bookmarkbackups",
michael@0: 
michael@0:   /**
michael@0:    * Cache current backups in a sorted (by date DESC) array.
michael@0:    */
michael@0:   get entries() {
michael@0:     Deprecated.warning(
michael@0:       "PlacesBackups.entries is deprecated and will be removed in a future version",
michael@0:       "https://bugzilla.mozilla.org/show_bug.cgi?id=859695");
michael@0:     return this._entries;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * This exists just to avoid spamming deprecate warnings from internal calls
michael@0:    * needed to support deprecated methods themselves.
michael@0:    */
michael@0:   get _entries() {
michael@0:     delete this._entries;
michael@0:     this._entries = [];
michael@0:     let files = this._folder.directoryEntries;
michael@0:     while (files.hasMoreElements()) {
michael@0:       let entry = files.getNext().QueryInterface(Ci.nsIFile);
michael@0:       // A valid backup is any file that matches either the localized or
michael@0:       // not-localized filename (bug 445704).
michael@0:       if (!entry.isHidden() && filenamesRegex.test(entry.leafName)) {
michael@0:         // Remove bogus backups in future dates.
michael@0:         if (this.getDateForFile(entry) > new Date()) {
michael@0:           entry.remove(false);
michael@0:           continue;
michael@0:         }
michael@0:         this._entries.push(entry);
michael@0:       }
michael@0:     }
michael@0:     this._entries.sort((a, b) => {
michael@0:       let aDate = this.getDateForFile(a);
michael@0:       let bDate = this.getDateForFile(b);
michael@0:       return aDate < bDate ? 1 : aDate > bDate ? -1 : 0;
michael@0:     });
michael@0:     return this._entries;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Cache current backups in a sorted (by date DESC) array.
michael@0:    * @return {Promise}
michael@0:    * @resolve a sorted array of string paths.
michael@0:    */
michael@0:   getBackupFiles: function PB_getBackupFiles() {
michael@0:     return Task.spawn(function* () {
michael@0:       if (this._backupFiles)
michael@0:         return this._backupFiles;
michael@0: 
michael@0:       this._backupFiles = [];
michael@0: 
michael@0:       let backupFolderPath = yield this.getBackupFolder();
michael@0:       let iterator = new OS.File.DirectoryIterator(backupFolderPath);
michael@0:       yield iterator.forEach(function(aEntry) {
michael@0:         // Since this is a lazy getter and OS.File I/O is serialized, we can
michael@0:         // safely remove .tmp files without risking to remove ongoing backups.
michael@0:         if (aEntry.name.endsWith(".tmp")) {
michael@0:           OS.File.remove(aEntry.path);
michael@0:           return;
michael@0:         }
michael@0: 
michael@0:         if (filenamesRegex.test(aEntry.name)) {
michael@0:           // Remove bogus backups in future dates.
michael@0:           let filePath = aEntry.path;
michael@0:           if (this.getDateForFile(filePath) > new Date()) {
michael@0:             return OS.File.remove(filePath);
michael@0:           } else {
michael@0:             this._backupFiles.push(filePath);
michael@0:           }
michael@0:         }
michael@0:       }.bind(this));
michael@0:       iterator.close();
michael@0: 
michael@0:       this._backupFiles.sort((a, b) => {
michael@0:         let aDate = this.getDateForFile(a);
michael@0:         let bDate = this.getDateForFile(b);
michael@0:         return aDate < bDate ? 1 : aDate > bDate ? -1 : 0;
michael@0:       });
michael@0: 
michael@0:       return this._backupFiles;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Creates a filename for bookmarks backup files.
michael@0:    *
michael@0:    * @param [optional] aDateObj
michael@0:    *                   Date object used to build the filename.
michael@0:    *                   Will use current date if empty.
michael@0:    * @return A bookmarks backup filename.
michael@0:    */
michael@0:   getFilenameForDate: function PB_getFilenameForDate(aDateObj) {
michael@0:     let dateObj = aDateObj || new Date();
michael@0:     // Use YYYY-MM-DD (ISO 8601) as it doesn't contain illegal characters
michael@0:     // and makes the alphabetical order of multiple backup files more useful.
michael@0:     return "bookmarks-" + dateObj.toLocaleFormat("%Y-%m-%d") + ".json";
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Creates a Date object from a backup file.  The date is the backup
michael@0:    * creation date.
michael@0:    *
michael@0:    * @param aBackupFile
michael@0:    *        nsIFile or string path of the backup.
michael@0:    * @return A Date object for the backup's creation time.
michael@0:    */
michael@0:   getDateForFile: function PB_getDateForFile(aBackupFile) {
michael@0:     let filename = (aBackupFile instanceof Ci.nsIFile) ? aBackupFile.leafName
michael@0:                                                        : OS.Path.basename(aBackupFile);
michael@0:     let matches = filename.match(filenamesRegex);
michael@0:     if (!matches)
michael@0:       throw("Invalid backup file name: " + filename);
michael@0:     return new Date(matches[1].replace(/-/g, "/"));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Get the most recent backup file.
michael@0:    *
michael@0:    * @param [optional] aFileExt
michael@0:    *                   Force file extension.  Either "html" or "json".
michael@0:    *                   Will check for both if not defined.
michael@0:    * @returns nsIFile backup file
michael@0:    */
michael@0:   getMostRecent: function PB_getMostRecent(aFileExt) {
michael@0:     Deprecated.warning(
michael@0:       "PlacesBackups.getMostRecent is deprecated and will be removed in a future version",
michael@0:       "https://bugzilla.mozilla.org/show_bug.cgi?id=859695");
michael@0: 
michael@0:     let fileExt = aFileExt || "(json|html)";
michael@0:     for (let i = 0; i < this._entries.length; i++) {
michael@0:       let rx = new RegExp("\." + fileExt + "$");
michael@0:       if (this._entries[i].leafName.match(rx))
michael@0:         return this._entries[i];
michael@0:     }
michael@0:     return null;
michael@0:   },
michael@0: 
michael@0:    /**
michael@0:     * Get the most recent backup file.
michael@0:     *
michael@0:     * @param [optional] aFileExt
michael@0:     *                   Force file extension.  Either "html" or "json".
michael@0:     *                   Will check for both if not defined.
michael@0:     * @return {Promise}
michael@0:     * @result the path to the file.
michael@0:     */
michael@0:    getMostRecentBackup: function PB_getMostRecentBackup(aFileExt) {
michael@0:      return Task.spawn(function* () {
michael@0:        let fileExt = aFileExt || "(json|html)";
michael@0:        let entries = yield this.getBackupFiles();
michael@0:        for (let entry of entries) {
michael@0:          let rx = new RegExp("\." + fileExt + "$");
michael@0:          if (OS.Path.basename(entry).match(rx)) {
michael@0:            return entry;
michael@0:          }
michael@0:        }
michael@0:        return null;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Serializes bookmarks using JSON, and writes to the supplied file.
michael@0:    * Note: any item that should not be backed up must be annotated with
michael@0:    *       "places/excludeFromBackup".
michael@0:    *
michael@0:    * @param aFilePath
michael@0:    *        OS.File path for the "bookmarks.json" file to be created.
michael@0:    * @return {Promise}
michael@0:    * @resolves the number of serialized uri nodes.
michael@0:    * @deprecated passing an nsIFile is deprecated
michael@0:    */
michael@0:   saveBookmarksToJSONFile: function PB_saveBookmarksToJSONFile(aFilePath) {
michael@0:     if (aFilePath instanceof Ci.nsIFile) {
michael@0:       Deprecated.warning("Passing an nsIFile to PlacesBackups.saveBookmarksToJSONFile " +
michael@0:                          "is deprecated. Please use an OS.File path instead.",
michael@0:                          "https://developer.mozilla.org/docs/JavaScript_OS.File");
michael@0:       aFilePath = aFilePath.path;
michael@0:     }
michael@0:     return Task.spawn(function* () {
michael@0:       let { count: nodeCount, hash: hash } =
michael@0:         yield BookmarkJSONUtils.exportToFile(aFilePath);
michael@0: 
michael@0:       let backupFolderPath = yield this.getBackupFolder();
michael@0:       if (OS.Path.dirname(aFilePath) == backupFolderPath) {
michael@0:         // We are creating a backup in the default backups folder,
michael@0:         // so just update the internal cache.
michael@0:         this._entries.unshift(new localFileCtor(aFilePath));
michael@0:         if (!this._backupFiles) {
michael@0:           yield this.getBackupFiles();
michael@0:         }
michael@0:         this._backupFiles.unshift(aFilePath);
michael@0:       } else {
michael@0:         // If we are saving to a folder different than our backups folder, then
michael@0:         // we also want to copy this new backup to it.
michael@0:         // This way we ensure the latest valid backup is the same saved by the
michael@0:         // user.  See bug 424389.
michael@0:         let mostRecentBackupFile = yield this.getMostRecentBackup("json");
michael@0:         if (!mostRecentBackupFile ||
michael@0:             hash != getHashFromFilename(OS.Path.basename(mostRecentBackupFile))) {
michael@0:           let name = this.getFilenameForDate();
michael@0:           let newFilename = appendMetaDataToFilename(name,
michael@0:                                                      { count: nodeCount,
michael@0:                                                        hash: hash });
michael@0:           let newFilePath = OS.Path.join(backupFolderPath, newFilename);
michael@0:           let backupFile = yield getBackupFileForSameDate(name);
michael@0:           if (backupFile) {
michael@0:             // There is already a backup for today, replace it.
michael@0:             yield OS.File.remove(backupFile, { ignoreAbsent: true });
michael@0:             if (!this._backupFiles)
michael@0:               yield this.getBackupFiles();
michael@0:             else
michael@0:               this._backupFiles.shift();
michael@0:             this._backupFiles.unshift(newFilePath);
michael@0:           } else {
michael@0:             // There is no backup for today, add the new one.
michael@0:             this._entries.unshift(new localFileCtor(newFilePath));
michael@0:             if (!this._backupFiles)
michael@0:               yield this.getBackupFiles();
michael@0:             this._backupFiles.unshift(newFilePath);
michael@0:           }
michael@0: 
michael@0:           yield OS.File.copy(aFilePath, newFilePath);
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       return nodeCount;
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Creates a dated backup in <profile>/bookmarkbackups.
michael@0:    * Stores the bookmarks using JSON.
michael@0:    * Note: any item that should not be backed up must be annotated with
michael@0:    *       "places/excludeFromBackup".
michael@0:    *
michael@0:    * @param [optional] int aMaxBackups
michael@0:    *                       The maximum number of backups to keep.  If set to 0
michael@0:    *                       all existing backups are removed and aForceBackup is
michael@0:    *                       ignored, so a new one won't be created.
michael@0:    * @param [optional] bool aForceBackup
michael@0:    *                        Forces creating a backup even if one was already
michael@0:    *                        created that day (overwrites).
michael@0:    * @return {Promise}
michael@0:    */
michael@0:   create: function PB_create(aMaxBackups, aForceBackup) {
michael@0:     let limitBackups = function* () {
michael@0:       let backupFiles = yield this.getBackupFiles();
michael@0:       if (typeof aMaxBackups == "number" && aMaxBackups > -1 &&
michael@0:           backupFiles.length >= aMaxBackups) {
michael@0:         let numberOfBackupsToDelete = backupFiles.length - aMaxBackups;
michael@0:         while (numberOfBackupsToDelete--) {
michael@0:           this._entries.pop();
michael@0:           let oldestBackup = this._backupFiles.pop();
michael@0:           yield OS.File.remove(oldestBackup);
michael@0:         }
michael@0:       }
michael@0:     }.bind(this);
michael@0: 
michael@0:     return Task.spawn(function* () {
michael@0:       if (aMaxBackups === 0) {
michael@0:         // Backups are disabled, delete any existing one and bail out.
michael@0:         yield limitBackups(0);
michael@0:         return;
michael@0:       }
michael@0: 
michael@0:       // Ensure to initialize _backupFiles
michael@0:       if (!this._backupFiles)
michael@0:         yield this.getBackupFiles();
michael@0:       let newBackupFilename = this.getFilenameForDate();
michael@0:       // If we already have a backup for today we should do nothing, unless we
michael@0:       // were required to enforce a new backup.
michael@0:       let backupFile = yield getBackupFileForSameDate(newBackupFilename);
michael@0:       if (backupFile && !aForceBackup)
michael@0:         return;
michael@0: 
michael@0:       if (backupFile) {
michael@0:         // In case there is a backup for today we should recreate it.
michael@0:         this._backupFiles.shift();
michael@0:         this._entries.shift();
michael@0:         yield OS.File.remove(backupFile, { ignoreAbsent: true });
michael@0:       }
michael@0: 
michael@0:       // Now check the hash of the most recent backup, and try to create a new
michael@0:       // backup, if that fails due to hash conflict, just rename the old backup.
michael@0:       let mostRecentBackupFile = yield this.getMostRecentBackup();
michael@0:       let mostRecentHash = mostRecentBackupFile &&
michael@0:                            getHashFromFilename(OS.Path.basename(mostRecentBackupFile));
michael@0: 
michael@0:       // Save bookmarks to a backup file.
michael@0:       let backupFolder = yield this.getBackupFolder();
michael@0:       let newBackupFile = OS.Path.join(backupFolder, newBackupFilename);
michael@0:       let newFilenameWithMetaData;
michael@0:       try {
michael@0:         let { count: nodeCount, hash: hash } =
michael@0:           yield BookmarkJSONUtils.exportToFile(newBackupFile,
michael@0:                                                { failIfHashIs: mostRecentHash });
michael@0:         newFilenameWithMetaData = appendMetaDataToFilename(newBackupFilename,
michael@0:                                                            { count: nodeCount,
michael@0:                                                              hash: hash });
michael@0:       } catch (ex if ex.becauseSameHash) {
michael@0:         // The last backup already contained up-to-date information, just
michael@0:         // rename it as if it was today's backup.
michael@0:         this._backupFiles.shift();
michael@0:         this._entries.shift();
michael@0:         newBackupFile = mostRecentBackupFile;
michael@0:         newFilenameWithMetaData = appendMetaDataToFilename(
michael@0:           newBackupFilename,
michael@0:           { count: this.getBookmarkCountForFile(mostRecentBackupFile),
michael@0:             hash: mostRecentHash });
michael@0:       }
michael@0: 
michael@0:       // Append metadata to the backup filename.
michael@0:       let newBackupFileWithMetadata = OS.Path.join(backupFolder, newFilenameWithMetaData);
michael@0:       yield OS.File.move(newBackupFile, newBackupFileWithMetadata);
michael@0:       this._entries.unshift(new localFileCtor(newBackupFileWithMetadata));
michael@0:       this._backupFiles.unshift(newBackupFileWithMetadata);
michael@0: 
michael@0:       // Limit the number of backups.
michael@0:       yield limitBackups(aMaxBackups);
michael@0:     }.bind(this));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Gets the bookmark count for backup file.
michael@0:    *
michael@0:    * @param aFilePath
michael@0:    *        File path The backup file.
michael@0:    *
michael@0:    * @return the bookmark count or null.
michael@0:    */
michael@0:   getBookmarkCountForFile: function PB_getBookmarkCountForFile(aFilePath) {
michael@0:     let count = null;
michael@0:     let filename = OS.Path.basename(aFilePath);
michael@0:     let matches = filename.match(filenamesRegex);
michael@0:     if (matches && matches[2])
michael@0:       count = matches[2];
michael@0:     return count;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Gets a bookmarks tree representation usable to create backups in different
michael@0:    * file formats.  The root or the tree is PlacesUtils.placesRootId.
michael@0:    * Items annotated with PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO and all of their
michael@0:    * descendants are excluded.
michael@0:    *
michael@0:    * @return an object representing a tree with the places root as its root.
michael@0:    *         Each bookmark is represented by an object having these properties:
michael@0:    *         * id: the item id (make this not enumerable after bug 824502)
michael@0:    *         * title: the title
michael@0:    *         * guid: unique id
michael@0:    *         * parent: item id of the parent folder, not enumerable
michael@0:    *         * index: the position in the parent
michael@0:    *         * dateAdded: microseconds from the epoch
michael@0:    *         * lastModified: microseconds from the epoch
michael@0:    *         * type: type of the originating node as defined in PlacesUtils 
michael@0:    *         The following properties exist only for a subset of bookmarks:
michael@0:    *         * annos: array of annotations
michael@0:    *         * uri: url
michael@0:    *         * iconuri: favicon's url
michael@0:    *         * keyword: associated keyword
michael@0:    *         * charset: last known charset
michael@0:    *         * tags: csv string of tags
michael@0:    *         * root: string describing whether this represents a root
michael@0:    *         * children: array of child items in a folder
michael@0:    */
michael@0:   getBookmarksTree: function () {
michael@0:     return Task.spawn(function* () {
michael@0:       let dbFilePath = OS.Path.join(OS.Constants.Path.profileDir, "places.sqlite");
michael@0:       let conn = yield Sqlite.openConnection({ path: dbFilePath,
michael@0:                                                sharedMemoryCache: false });
michael@0:       let rows = [];
michael@0:       try {
michael@0:         rows = yield conn.execute(
michael@0:           "SELECT b.id, h.url, IFNULL(b.title, '') AS title, b.parent, " +
michael@0:                  "b.position AS [index], b.type, b.dateAdded, b.lastModified, " +
michael@0:                  "b.guid, f.url AS iconuri, " +
michael@0:                  "( SELECT GROUP_CONCAT(t.title, ',') " +
michael@0:                    "FROM moz_bookmarks b2 " +
michael@0:                    "JOIN moz_bookmarks t ON t.id = +b2.parent AND t.parent = :tags_folder " +
michael@0:                    "WHERE b2.fk = h.id " +
michael@0:                  ") AS tags, " +
michael@0:                  "EXISTS (SELECT 1 FROM moz_items_annos WHERE item_id = b.id LIMIT 1) AS has_annos, " +
michael@0:                  "( SELECT a.content FROM moz_annos a " +
michael@0:                    "JOIN moz_anno_attributes n ON a.anno_attribute_id = n.id " +
michael@0:                    "WHERE place_id = h.id AND n.name = :charset_anno " +
michael@0:                  ") AS charset " +
michael@0:           "FROM moz_bookmarks b " +
michael@0:           "LEFT JOIN moz_bookmarks p ON p.id = b.parent " +
michael@0:           "LEFT JOIN moz_places h ON h.id = b.fk " +
michael@0:           "LEFT JOIN moz_favicons f ON f.id = h.favicon_id " +
michael@0:           "WHERE b.id <> :tags_folder AND b.parent <> :tags_folder AND p.parent <> :tags_folder " +
michael@0:           "ORDER BY b.parent, b.position",
michael@0:           { tags_folder: PlacesUtils.tagsFolderId,
michael@0:             charset_anno: PlacesUtils.CHARSET_ANNO });
michael@0:       } catch(e) {
michael@0:         Cu.reportError("Unable to query the database " + e);
michael@0:       } finally {
michael@0:         yield conn.close();
michael@0:       }
michael@0: 
michael@0:       let startTime = Date.now();
michael@0:       // Create a Map for lookup and recursive building of the tree.
michael@0:       let itemsMap = new Map();
michael@0:       for (let row of rows) {
michael@0:         let id = row.getResultByName("id");
michael@0:         try {
michael@0:           let bookmark = sqliteRowToBookmarkObject(row);
michael@0:           if (itemsMap.has(id)) {
michael@0:             // Since children may be added before parents, we should merge with
michael@0:             // the existing object.
michael@0:             let original = itemsMap.get(id);
michael@0:             for (let prop of Object.getOwnPropertyNames(bookmark)) {
michael@0:               original[prop] = bookmark[prop];
michael@0:             }
michael@0:             bookmark = original;
michael@0:           }
michael@0:           else {
michael@0:             itemsMap.set(id, bookmark);
michael@0:           }
michael@0: 
michael@0:           // Append bookmark to its parent.
michael@0:           if (!itemsMap.has(bookmark.parent))
michael@0:             itemsMap.set(bookmark.parent, {});
michael@0:           let parent = itemsMap.get(bookmark.parent);
michael@0:           if (!("children" in parent))
michael@0:             parent.children = [];
michael@0:           parent.children.push(bookmark);
michael@0:         } catch (e) {
michael@0:           Cu.reportError("Error while reading node " + id + " " + e);
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       // Handle excluded items, by removing entire subtrees pointed by them.
michael@0:       function removeFromMap(id) {
michael@0:         // Could have been removed by a previous call, since we can't
michael@0:         // predict order of items in EXCLUDE_FROM_BACKUP_ANNO.
michael@0:         if (itemsMap.has(id)) {
michael@0:           let excludedItem = itemsMap.get(id);
michael@0:           if (excludedItem.children) {
michael@0:             for (let child of excludedItem.children) {
michael@0:               removeFromMap(child.id);
michael@0:             }
michael@0:           }
michael@0:           // Remove the excluded item from its parent's children...
michael@0:           let parentItem = itemsMap.get(excludedItem.parent);
michael@0:           parentItem.children = parentItem.children.filter(aChild => aChild.id != id);
michael@0:           // ...then remove it from the map.
michael@0:           itemsMap.delete(id);
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       for (let id of PlacesUtils.annotations.getItemsWithAnnotation(
michael@0:                        PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO)) {
michael@0:         removeFromMap(id);
michael@0:       }
michael@0: 
michael@0:       // Report the time taken to build the tree. This doesn't take into
michael@0:       // account the time spent in the query since that's off the main-thread.
michael@0:       try {
michael@0:         Services.telemetry
michael@0:                 .getHistogramById("PLACES_BACKUPS_BOOKMARKSTREE_MS")
michael@0:                 .add(Date.now() - startTime);
michael@0:       } catch (ex) {
michael@0:         Components.utils.reportError("Unable to report telemetry.");
michael@0:       }
michael@0: 
michael@0:       return [itemsMap.get(PlacesUtils.placesRootId), itemsMap.size];
michael@0:     });
michael@0:   }
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Helper function to convert a Sqlite.jsm row to a bookmark object
michael@0:  * representation.
michael@0:  *
michael@0:  * @param aRow The Sqlite.jsm result row.
michael@0:  */
michael@0: function sqliteRowToBookmarkObject(aRow) {
michael@0:   let bookmark = {};
michael@0:   for (let p of [ "id" ,"guid", "title", "index", "dateAdded", "lastModified" ]) {
michael@0:     bookmark[p] = aRow.getResultByName(p);
michael@0:   }
michael@0:   Object.defineProperty(bookmark, "parent",
michael@0:                         { value: aRow.getResultByName("parent") });
michael@0: 
michael@0:   let type = aRow.getResultByName("type");
michael@0: 
michael@0:   // Add annotations.
michael@0:   if (aRow.getResultByName("has_annos")) {
michael@0:     try {
michael@0:       bookmark.annos = PlacesUtils.getAnnotationsForItem(bookmark.id);
michael@0:     } catch (e) {
michael@0:       Cu.reportError("Unexpected error while reading annotations " + e);
michael@0:     }
michael@0:   }
michael@0: 
michael@0:   switch (type) {
michael@0:     case Ci.nsINavBookmarksService.TYPE_BOOKMARK:
michael@0:       // TODO: What about shortcuts?
michael@0:       bookmark.type = PlacesUtils.TYPE_X_MOZ_PLACE;
michael@0:       // This will throw if we try to serialize an invalid url and the node will
michael@0:       // just be skipped.
michael@0:       bookmark.uri = NetUtil.newURI(aRow.getResultByName("url")).spec;
michael@0:       // Keywords are cached, so this should be decently fast.
michael@0:       let keyword = PlacesUtils.bookmarks.getKeywordForBookmark(bookmark.id);
michael@0:       if (keyword)
michael@0:         bookmark.keyword = keyword;
michael@0:       let charset = aRow.getResultByName("charset");
michael@0:       if (charset)
michael@0:         bookmark.charset = charset;
michael@0:       let tags = aRow.getResultByName("tags");
michael@0:       if (tags)
michael@0:         bookmark.tags = tags;
michael@0:       let iconuri = aRow.getResultByName("iconuri");
michael@0:       if (iconuri)
michael@0:         bookmark.iconuri = iconuri;
michael@0:       break;
michael@0:     case Ci.nsINavBookmarksService.TYPE_FOLDER:
michael@0:       bookmark.type = PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER;
michael@0: 
michael@0:       // Mark root folders.
michael@0:       if (bookmark.id == PlacesUtils.placesRootId)
michael@0:         bookmark.root = "placesRoot";
michael@0:       else if (bookmark.id == PlacesUtils.bookmarksMenuFolderId)
michael@0:         bookmark.root = "bookmarksMenuFolder";
michael@0:       else if (bookmark.id == PlacesUtils.unfiledBookmarksFolderId)
michael@0:         bookmark.root = "unfiledBookmarksFolder";
michael@0:       else if (bookmark.id == PlacesUtils.toolbarFolderId)
michael@0:         bookmark.root = "toolbarFolder";
michael@0:       break;
michael@0:     case Ci.nsINavBookmarksService.TYPE_SEPARATOR:
michael@0:       bookmark.type = PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR;
michael@0:       break;
michael@0:     default:
michael@0:       Cu.reportError("Unexpected bookmark type");
michael@0:       break;
michael@0:   }
michael@0:   return bookmark;
michael@0: }