michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this file, michael@0: * You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: this.EXPORTED_SYMBOLS = [ "BookmarkJSONUtils" ]; michael@0: michael@0: const Ci = Components.interfaces; michael@0: const Cc = Components.classes; michael@0: const Cu = Components.utils; michael@0: const Cr = Components.results; michael@0: michael@0: Cu.import("resource://gre/modules/XPCOMUtils.jsm"); michael@0: Cu.import("resource://gre/modules/Services.jsm"); michael@0: Cu.import("resource://gre/modules/NetUtil.jsm"); michael@0: Cu.import("resource://gre/modules/osfile.jsm"); michael@0: Cu.import("resource://gre/modules/PlacesUtils.jsm"); michael@0: Cu.import("resource://gre/modules/Promise.jsm"); michael@0: Cu.import("resource://gre/modules/Task.jsm"); michael@0: michael@0: XPCOMUtils.defineLazyModuleGetter(this, "PlacesBackups", michael@0: "resource://gre/modules/PlacesBackups.jsm"); michael@0: XPCOMUtils.defineLazyModuleGetter(this, "Deprecated", michael@0: "resource://gre/modules/Deprecated.jsm"); michael@0: michael@0: XPCOMUtils.defineLazyGetter(this, "gTextDecoder", () => new TextDecoder()); michael@0: XPCOMUtils.defineLazyGetter(this, "gTextEncoder", () => new TextEncoder()); michael@0: michael@0: /** michael@0: * Generates an hash for the given string. michael@0: * michael@0: * @note The generated hash is returned in base64 form. Mind the fact base64 michael@0: * is case-sensitive if you are going to reuse this code. michael@0: */ michael@0: function generateHash(aString) { michael@0: let cryptoHash = Cc["@mozilla.org/security/hash;1"] michael@0: .createInstance(Ci.nsICryptoHash); michael@0: cryptoHash.init(Ci.nsICryptoHash.MD5); michael@0: let stringStream = Cc["@mozilla.org/io/string-input-stream;1"] michael@0: .createInstance(Ci.nsIStringInputStream); michael@0: stringStream.data = aString; michael@0: cryptoHash.updateFromStream(stringStream, -1); michael@0: // base64 allows the '/' char, but we can't use it for filenames. michael@0: return cryptoHash.finish(true).replace("/", "-", "g"); michael@0: } michael@0: michael@0: this.BookmarkJSONUtils = Object.freeze({ michael@0: /** michael@0: * Import bookmarks from a url. michael@0: * michael@0: * @param aSpec michael@0: * url of the bookmark data. michael@0: * @param aReplace michael@0: * Boolean if true, replace existing bookmarks, else merge. michael@0: * michael@0: * @return {Promise} michael@0: * @resolves When the new bookmarks have been created. michael@0: * @rejects JavaScript exception. michael@0: */ michael@0: importFromURL: function BJU_importFromURL(aSpec, aReplace) { michael@0: return Task.spawn(function* () { michael@0: notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN); michael@0: try { michael@0: let importer = new BookmarkImporter(aReplace); michael@0: yield importer.importFromURL(aSpec); michael@0: michael@0: notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS); michael@0: } catch(ex) { michael@0: Cu.reportError("Failed to restore bookmarks from " + aSpec + ": " + ex); michael@0: notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED); michael@0: } michael@0: }); michael@0: }, michael@0: michael@0: /** michael@0: * Restores bookmarks and tags from a JSON file. michael@0: * @note any item annotated with "places/excludeFromBackup" won't be removed michael@0: * before executing the restore. michael@0: * michael@0: * @param aFilePath michael@0: * OS.File path string of bookmarks in JSON format to be restored. michael@0: * @param aReplace michael@0: * Boolean if true, replace existing bookmarks, else merge. michael@0: * michael@0: * @return {Promise} michael@0: * @resolves When the new bookmarks have been created. michael@0: * @rejects JavaScript exception. michael@0: * @deprecated passing an nsIFile is deprecated michael@0: */ michael@0: importFromFile: function BJU_importFromFile(aFilePath, aReplace) { michael@0: if (aFilePath instanceof Ci.nsIFile) { michael@0: Deprecated.warning("Passing an nsIFile to BookmarksJSONUtils.importFromFile " + michael@0: "is deprecated. Please use an OS.File path string instead.", michael@0: "https://developer.mozilla.org/docs/JavaScript_OS.File"); michael@0: aFilePath = aFilePath.path; michael@0: } michael@0: michael@0: return Task.spawn(function* () { michael@0: notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN); michael@0: try { michael@0: if (!(yield OS.File.exists(aFilePath))) michael@0: throw new Error("Cannot restore from nonexisting json file"); michael@0: michael@0: let importer = new BookmarkImporter(aReplace); michael@0: yield importer.importFromURL(OS.Path.toFileURI(aFilePath)); michael@0: michael@0: notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS); michael@0: } catch(ex) { michael@0: Cu.reportError("Failed to restore bookmarks from " + aFilePath + ": " + ex); michael@0: notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED); michael@0: throw ex; michael@0: } michael@0: }); michael@0: }, michael@0: michael@0: /** michael@0: * Serializes bookmarks using JSON, and writes to the supplied file path. michael@0: * michael@0: * @param aFilePath michael@0: * OS.File path string for the "bookmarks.json" file to be created. michael@0: * @param [optional] aOptions michael@0: * Object containing options for the export: michael@0: * - failIfHashIs: if the generated file would have the same hash michael@0: * defined here, will reject with ex.becauseSameHash michael@0: * @return {Promise} michael@0: * @resolves once the file has been created, to an object with the michael@0: * following properties: michael@0: * - count: number of exported bookmarks michael@0: * - hash: file hash for contents comparison michael@0: * @rejects JavaScript exception. michael@0: * @deprecated passing an nsIFile is deprecated michael@0: */ michael@0: exportToFile: function BJU_exportToFile(aFilePath, aOptions={}) { michael@0: if (aFilePath instanceof Ci.nsIFile) { michael@0: Deprecated.warning("Passing an nsIFile to BookmarksJSONUtils.exportToFile " + michael@0: "is deprecated. Please use an OS.File path string instead.", michael@0: "https://developer.mozilla.org/docs/JavaScript_OS.File"); michael@0: aFilePath = aFilePath.path; michael@0: } michael@0: return Task.spawn(function* () { michael@0: let [bookmarks, count] = yield PlacesBackups.getBookmarksTree(); michael@0: let startTime = Date.now(); michael@0: let jsonString = JSON.stringify(bookmarks); michael@0: // Report the time taken to convert the tree to JSON. michael@0: try { michael@0: Services.telemetry michael@0: .getHistogramById("PLACES_BACKUPS_TOJSON_MS") michael@0: .add(Date.now() - startTime); michael@0: } catch (ex) { michael@0: Components.utils.reportError("Unable to report telemetry."); michael@0: } michael@0: michael@0: startTime = Date.now(); michael@0: let hash = generateHash(jsonString); michael@0: // Report the time taken to generate the hash. michael@0: try { michael@0: Services.telemetry michael@0: .getHistogramById("PLACES_BACKUPS_HASHING_MS") michael@0: .add(Date.now() - startTime); michael@0: } catch (ex) { michael@0: Components.utils.reportError("Unable to report telemetry."); michael@0: } michael@0: michael@0: if (hash === aOptions.failIfHashIs) { michael@0: let e = new Error("Hash conflict"); michael@0: e.becauseSameHash = true; michael@0: throw e; michael@0: } michael@0: michael@0: // Do not write to the tmp folder, otherwise if it has a different michael@0: // filesystem writeAtomic will fail. Eventual dangling .tmp files should michael@0: // be cleaned up by the caller. michael@0: yield OS.File.writeAtomic(aFilePath, jsonString, michael@0: { tmpPath: OS.Path.join(aFilePath + ".tmp") }); michael@0: return { count: count, hash: hash }; michael@0: }); michael@0: } michael@0: }); michael@0: michael@0: function BookmarkImporter(aReplace) { michael@0: this._replace = aReplace; michael@0: } michael@0: BookmarkImporter.prototype = { michael@0: /** michael@0: * Import bookmarks from a url. michael@0: * michael@0: * @param aSpec michael@0: * url of the bookmark data. michael@0: * michael@0: * @return {Promise} michael@0: * @resolves When the new bookmarks have been created. michael@0: * @rejects JavaScript exception. michael@0: */ michael@0: importFromURL: function BI_importFromURL(aSpec) { michael@0: let deferred = Promise.defer(); michael@0: michael@0: let streamObserver = { michael@0: onStreamComplete: function (aLoader, aContext, aStatus, aLength, michael@0: aResult) { michael@0: let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]. michael@0: createInstance(Ci.nsIScriptableUnicodeConverter); michael@0: converter.charset = "UTF-8"; michael@0: michael@0: try { michael@0: let jsonString = converter.convertFromByteArray(aResult, michael@0: aResult.length); michael@0: deferred.resolve(this.importFromJSON(jsonString)); michael@0: } catch (ex) { michael@0: Cu.reportError("Failed to import from URL: " + ex); michael@0: deferred.reject(ex); michael@0: throw ex; michael@0: } michael@0: }.bind(this) michael@0: }; michael@0: michael@0: try { michael@0: let channel = Services.io.newChannelFromURI(NetUtil.newURI(aSpec)); michael@0: let streamLoader = Cc["@mozilla.org/network/stream-loader;1"]. michael@0: createInstance(Ci.nsIStreamLoader); michael@0: michael@0: streamLoader.init(streamObserver); michael@0: channel.asyncOpen(streamLoader, channel); michael@0: } catch (ex) { michael@0: deferred.reject(ex); michael@0: } michael@0: michael@0: return deferred.promise; michael@0: }, michael@0: michael@0: /** michael@0: * Import bookmarks from a JSON string. michael@0: * michael@0: * @param aString michael@0: * JSON string of serialized bookmark data. michael@0: */ michael@0: importFromJSON: function BI_importFromJSON(aString) { michael@0: let deferred = Promise.defer(); michael@0: let nodes = michael@0: PlacesUtils.unwrapNodes(aString, PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER); michael@0: michael@0: if (nodes.length == 0 || !nodes[0].children || michael@0: nodes[0].children.length == 0) { michael@0: deferred.resolve(); // Nothing to restore michael@0: } else { michael@0: // Ensure tag folder gets processed last michael@0: nodes[0].children.sort(function sortRoots(aNode, bNode) { michael@0: return (aNode.root && aNode.root == "tagsFolder") ? 1 : michael@0: (bNode.root && bNode.root == "tagsFolder") ? -1 : 0; michael@0: }); michael@0: michael@0: let batch = { michael@0: nodes: nodes[0].children, michael@0: runBatched: function runBatched() { michael@0: if (this._replace) { michael@0: // Get roots excluded from the backup, we will not remove them michael@0: // before restoring. michael@0: let excludeItems = PlacesUtils.annotations.getItemsWithAnnotation( michael@0: PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO); michael@0: // Delete existing children of the root node, excepting: michael@0: // 1. special folders: delete the child nodes michael@0: // 2. tags folder: untag via the tagging api michael@0: let root = PlacesUtils.getFolderContents(PlacesUtils.placesRootId, michael@0: false, false).root; michael@0: let childIds = []; michael@0: for (let i = 0; i < root.childCount; i++) { michael@0: let childId = root.getChild(i).itemId; michael@0: if (excludeItems.indexOf(childId) == -1 && michael@0: childId != PlacesUtils.tagsFolderId) { michael@0: childIds.push(childId); michael@0: } michael@0: } michael@0: root.containerOpen = false; michael@0: michael@0: for (let i = 0; i < childIds.length; i++) { michael@0: let rootItemId = childIds[i]; michael@0: if (PlacesUtils.isRootItem(rootItemId)) { michael@0: PlacesUtils.bookmarks.removeFolderChildren(rootItemId); michael@0: } else { michael@0: PlacesUtils.bookmarks.removeItem(rootItemId); michael@0: } michael@0: } michael@0: } michael@0: michael@0: let searchIds = []; michael@0: let folderIdMap = []; michael@0: michael@0: for (let node of batch.nodes) { michael@0: if (!node.children || node.children.length == 0) michael@0: continue; // Nothing to restore for this root michael@0: michael@0: if (node.root) { michael@0: let container = PlacesUtils.placesRootId; // Default to places root michael@0: switch (node.root) { michael@0: case "bookmarksMenuFolder": michael@0: container = PlacesUtils.bookmarksMenuFolderId; michael@0: break; michael@0: case "tagsFolder": michael@0: container = PlacesUtils.tagsFolderId; michael@0: break; michael@0: case "unfiledBookmarksFolder": michael@0: container = PlacesUtils.unfiledBookmarksFolderId; michael@0: break; michael@0: case "toolbarFolder": michael@0: container = PlacesUtils.toolbarFolderId; michael@0: break; michael@0: } michael@0: michael@0: // Insert the data into the db michael@0: for (let child of node.children) { michael@0: let index = child.index; michael@0: let [folders, searches] = michael@0: this.importJSONNode(child, container, index, 0); michael@0: for (let i = 0; i < folders.length; i++) { michael@0: if (folders[i]) michael@0: folderIdMap[i] = folders[i]; michael@0: } michael@0: searchIds = searchIds.concat(searches); michael@0: } michael@0: } else { michael@0: this.importJSONNode( michael@0: node, PlacesUtils.placesRootId, node.index, 0); michael@0: } michael@0: } michael@0: michael@0: // Fixup imported place: uris that contain folders michael@0: searchIds.forEach(function(aId) { michael@0: let oldURI = PlacesUtils.bookmarks.getBookmarkURI(aId); michael@0: let uri = fixupQuery(oldURI, folderIdMap); michael@0: if (!uri.equals(oldURI)) { michael@0: PlacesUtils.bookmarks.changeBookmarkURI(aId, uri); michael@0: } michael@0: }); michael@0: michael@0: deferred.resolve(); michael@0: }.bind(this) michael@0: }; michael@0: michael@0: PlacesUtils.bookmarks.runInBatchMode(batch, null); michael@0: } michael@0: return deferred.promise; michael@0: }, michael@0: michael@0: /** michael@0: * Takes a JSON-serialized node and inserts it into the db. michael@0: * michael@0: * @param aData michael@0: * The unwrapped data blob of dropped or pasted data. michael@0: * @param aContainer michael@0: * The container the data was dropped or pasted into michael@0: * @param aIndex michael@0: * The index within the container the item was dropped or pasted at michael@0: * @return an array containing of maps of old folder ids to new folder ids, michael@0: * and an array of saved search ids that need to be fixed up. michael@0: * eg: [[[oldFolder1, newFolder1]], [search1]] michael@0: */ michael@0: importJSONNode: function BI_importJSONNode(aData, aContainer, aIndex, michael@0: aGrandParentId) { michael@0: let folderIdMap = []; michael@0: let searchIds = []; michael@0: let id = -1; michael@0: switch (aData.type) { michael@0: case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: michael@0: if (aContainer == PlacesUtils.tagsFolderId) { michael@0: // Node is a tag michael@0: if (aData.children) { michael@0: aData.children.forEach(function(aChild) { michael@0: try { michael@0: PlacesUtils.tagging.tagURI( michael@0: NetUtil.newURI(aChild.uri), [aData.title]); michael@0: } catch (ex) { michael@0: // Invalid tag child, skip it michael@0: } michael@0: }); michael@0: return [folderIdMap, searchIds]; michael@0: } michael@0: } else if (aData.annos && michael@0: aData.annos.some(anno => anno.name == PlacesUtils.LMANNO_FEEDURI)) { michael@0: // Node is a livemark michael@0: let feedURI = null; michael@0: let siteURI = null; michael@0: aData.annos = aData.annos.filter(function(aAnno) { michael@0: switch (aAnno.name) { michael@0: case PlacesUtils.LMANNO_FEEDURI: michael@0: feedURI = NetUtil.newURI(aAnno.value); michael@0: return false; michael@0: case PlacesUtils.LMANNO_SITEURI: michael@0: siteURI = NetUtil.newURI(aAnno.value); michael@0: return false; michael@0: default: michael@0: return true; michael@0: } michael@0: }); michael@0: michael@0: if (feedURI) { michael@0: PlacesUtils.livemarks.addLivemark({ michael@0: title: aData.title, michael@0: feedURI: feedURI, michael@0: parentId: aContainer, michael@0: index: aIndex, michael@0: lastModified: aData.lastModified, michael@0: siteURI: siteURI michael@0: }).then(function (aLivemark) { michael@0: let id = aLivemark.id; michael@0: if (aData.dateAdded) michael@0: PlacesUtils.bookmarks.setItemDateAdded(id, aData.dateAdded); michael@0: if (aData.annos && aData.annos.length) michael@0: PlacesUtils.setAnnotationsForItem(id, aData.annos); michael@0: }, Cu.reportError); michael@0: } michael@0: } else { michael@0: id = PlacesUtils.bookmarks.createFolder( michael@0: aContainer, aData.title, aIndex); michael@0: folderIdMap[aData.id] = id; michael@0: // Process children michael@0: if (aData.children) { michael@0: for (let i = 0; i < aData.children.length; i++) { michael@0: let child = aData.children[i]; michael@0: let [folders, searches] = michael@0: this.importJSONNode(child, id, i, aContainer); michael@0: for (let j = 0; j < folders.length; j++) { michael@0: if (folders[j]) michael@0: folderIdMap[j] = folders[j]; michael@0: } michael@0: searchIds = searchIds.concat(searches); michael@0: } michael@0: } michael@0: } michael@0: break; michael@0: case PlacesUtils.TYPE_X_MOZ_PLACE: michael@0: id = PlacesUtils.bookmarks.insertBookmark( michael@0: aContainer, NetUtil.newURI(aData.uri), aIndex, aData.title); michael@0: if (aData.keyword) michael@0: PlacesUtils.bookmarks.setKeywordForBookmark(id, aData.keyword); michael@0: if (aData.tags) { michael@0: // TODO (bug 967196) the tagging service should trim by itself. michael@0: let tags = aData.tags.split(",").map(tag => tag.trim()); michael@0: if (tags.length) michael@0: PlacesUtils.tagging.tagURI(NetUtil.newURI(aData.uri), tags); michael@0: } michael@0: if (aData.charset) { michael@0: PlacesUtils.annotations.setPageAnnotation( michael@0: NetUtil.newURI(aData.uri), PlacesUtils.CHARSET_ANNO, aData.charset, michael@0: 0, Ci.nsIAnnotationService.EXPIRE_NEVER); michael@0: } michael@0: if (aData.uri.substr(0, 6) == "place:") michael@0: searchIds.push(id); michael@0: if (aData.icon) { michael@0: try { michael@0: // Create a fake faviconURI to use (FIXME: bug 523932) michael@0: let faviconURI = NetUtil.newURI("fake-favicon-uri:" + aData.uri); michael@0: PlacesUtils.favicons.replaceFaviconDataFromDataURL( michael@0: faviconURI, aData.icon, 0); michael@0: PlacesUtils.favicons.setAndFetchFaviconForPage( michael@0: NetUtil.newURI(aData.uri), faviconURI, false, michael@0: PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE); michael@0: } catch (ex) { michael@0: Components.utils.reportError("Failed to import favicon data:" + ex); michael@0: } michael@0: } michael@0: if (aData.iconUri) { michael@0: try { michael@0: PlacesUtils.favicons.setAndFetchFaviconForPage( michael@0: NetUtil.newURI(aData.uri), NetUtil.newURI(aData.iconUri), false, michael@0: PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE); michael@0: } catch (ex) { michael@0: Components.utils.reportError("Failed to import favicon URI:" + ex); michael@0: } michael@0: } michael@0: break; michael@0: case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: michael@0: id = PlacesUtils.bookmarks.insertSeparator(aContainer, aIndex); michael@0: break; michael@0: default: michael@0: // Unknown node type michael@0: } michael@0: michael@0: // Set generic properties, valid for all nodes michael@0: if (id != -1 && aContainer != PlacesUtils.tagsFolderId && michael@0: aGrandParentId != PlacesUtils.tagsFolderId) { michael@0: if (aData.dateAdded) michael@0: PlacesUtils.bookmarks.setItemDateAdded(id, aData.dateAdded); michael@0: if (aData.lastModified) michael@0: PlacesUtils.bookmarks.setItemLastModified(id, aData.lastModified); michael@0: if (aData.annos && aData.annos.length) michael@0: PlacesUtils.setAnnotationsForItem(id, aData.annos); michael@0: } michael@0: michael@0: return [folderIdMap, searchIds]; michael@0: } michael@0: } michael@0: michael@0: function notifyObservers(topic) { michael@0: Services.obs.notifyObservers(null, topic, "json"); michael@0: } michael@0: michael@0: /** michael@0: * Replaces imported folder ids with their local counterparts in a place: URI. michael@0: * michael@0: * @param aURI michael@0: * A place: URI with folder ids. michael@0: * @param aFolderIdMap michael@0: * An array mapping old folder id to new folder ids. michael@0: * @returns the fixed up URI if all matched. If some matched, it returns michael@0: * the URI with only the matching folders included. If none matched michael@0: * it returns the input URI unchanged. michael@0: */ michael@0: function fixupQuery(aQueryURI, aFolderIdMap) { michael@0: let convert = function(str, p1, offset, s) { michael@0: return "folder=" + aFolderIdMap[p1]; michael@0: } michael@0: let stringURI = aQueryURI.spec.replace(/folder=([0-9]+)/g, convert); michael@0: michael@0: return NetUtil.newURI(stringURI); michael@0: }