1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/services/sync/modules/record.js Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,629 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +this.EXPORTED_SYMBOLS = [
1.9 + "WBORecord",
1.10 + "RecordManager",
1.11 + "CryptoWrapper",
1.12 + "CollectionKeyManager",
1.13 + "Collection",
1.14 +];
1.15 +
1.16 +const Cc = Components.classes;
1.17 +const Ci = Components.interfaces;
1.18 +const Cr = Components.results;
1.19 +const Cu = Components.utils;
1.20 +
1.21 +const CRYPTO_COLLECTION = "crypto";
1.22 +const KEYS_WBO = "keys";
1.23 +
1.24 +Cu.import("resource://gre/modules/Log.jsm");
1.25 +Cu.import("resource://services-sync/constants.js");
1.26 +Cu.import("resource://services-sync/keys.js");
1.27 +Cu.import("resource://services-sync/resource.js");
1.28 +Cu.import("resource://services-sync/util.js");
1.29 +
1.30 +this.WBORecord = function WBORecord(collection, id) {
1.31 + this.data = {};
1.32 + this.payload = {};
1.33 + this.collection = collection; // Optional.
1.34 + this.id = id; // Optional.
1.35 +}
1.36 +WBORecord.prototype = {
1.37 + _logName: "Sync.Record.WBO",
1.38 +
1.39 + get sortindex() {
1.40 + if (this.data.sortindex)
1.41 + return this.data.sortindex;
1.42 + return 0;
1.43 + },
1.44 +
1.45 + // Get thyself from your URI, then deserialize.
1.46 + // Set thine 'response' field.
1.47 + fetch: function fetch(resource) {
1.48 + if (!resource instanceof Resource) {
1.49 + throw new Error("First argument must be a Resource instance.");
1.50 + }
1.51 +
1.52 + let r = resource.get();
1.53 + if (r.success) {
1.54 + this.deserialize(r); // Warning! Muffles exceptions!
1.55 + }
1.56 + this.response = r;
1.57 + return this;
1.58 + },
1.59 +
1.60 + upload: function upload(resource) {
1.61 + if (!resource instanceof Resource) {
1.62 + throw new Error("First argument must be a Resource instance.");
1.63 + }
1.64 +
1.65 + return resource.put(this);
1.66 + },
1.67 +
1.68 + // Take a base URI string, with trailing slash, and return the URI of this
1.69 + // WBO based on collection and ID.
1.70 + uri: function(base) {
1.71 + if (this.collection && this.id) {
1.72 + let url = Utils.makeURI(base + this.collection + "/" + this.id);
1.73 + url.QueryInterface(Ci.nsIURL);
1.74 + return url;
1.75 + }
1.76 + return null;
1.77 + },
1.78 +
1.79 + deserialize: function deserialize(json) {
1.80 + this.data = json.constructor.toString() == String ? JSON.parse(json) : json;
1.81 +
1.82 + try {
1.83 + // The payload is likely to be JSON, but if not, keep it as a string
1.84 + this.payload = JSON.parse(this.payload);
1.85 + } catch(ex) {}
1.86 + },
1.87 +
1.88 + toJSON: function toJSON() {
1.89 + // Copy fields from data to be stringified, making sure payload is a string
1.90 + let obj = {};
1.91 + for (let [key, val] in Iterator(this.data))
1.92 + obj[key] = key == "payload" ? JSON.stringify(val) : val;
1.93 + if (this.ttl)
1.94 + obj.ttl = this.ttl;
1.95 + return obj;
1.96 + },
1.97 +
1.98 + toString: function toString() {
1.99 + return "{ " +
1.100 + "id: " + this.id + " " +
1.101 + "index: " + this.sortindex + " " +
1.102 + "modified: " + this.modified + " " +
1.103 + "ttl: " + this.ttl + " " +
1.104 + "payload: " + JSON.stringify(this.payload) +
1.105 + " }";
1.106 + }
1.107 +};
1.108 +
1.109 +Utils.deferGetSet(WBORecord, "data", ["id", "modified", "sortindex", "payload"]);
1.110 +
1.111 +/**
1.112 + * An interface and caching layer for records.
1.113 + */
1.114 +this.RecordManager = function RecordManager(service) {
1.115 + this.service = service;
1.116 +
1.117 + this._log = Log.repository.getLogger(this._logName);
1.118 + this._records = {};
1.119 +}
1.120 +RecordManager.prototype = {
1.121 + _recordType: WBORecord,
1.122 + _logName: "Sync.RecordManager",
1.123 +
1.124 + import: function RecordMgr_import(url) {
1.125 + this._log.trace("Importing record: " + (url.spec ? url.spec : url));
1.126 + try {
1.127 + // Clear out the last response with empty object if GET fails
1.128 + this.response = {};
1.129 + this.response = this.service.resource(url).get();
1.130 +
1.131 + // Don't parse and save the record on failure
1.132 + if (!this.response.success)
1.133 + return null;
1.134 +
1.135 + let record = new this._recordType(url);
1.136 + record.deserialize(this.response);
1.137 +
1.138 + return this.set(url, record);
1.139 + } catch(ex) {
1.140 + this._log.debug("Failed to import record: " + Utils.exceptionStr(ex));
1.141 + return null;
1.142 + }
1.143 + },
1.144 +
1.145 + get: function RecordMgr_get(url) {
1.146 + // Use a url string as the key to the hash
1.147 + let spec = url.spec ? url.spec : url;
1.148 + if (spec in this._records)
1.149 + return this._records[spec];
1.150 + return this.import(url);
1.151 + },
1.152 +
1.153 + set: function RecordMgr_set(url, record) {
1.154 + let spec = url.spec ? url.spec : url;
1.155 + return this._records[spec] = record;
1.156 + },
1.157 +
1.158 + contains: function RecordMgr_contains(url) {
1.159 + if ((url.spec || url) in this._records)
1.160 + return true;
1.161 + return false;
1.162 + },
1.163 +
1.164 + clearCache: function recordMgr_clearCache() {
1.165 + this._records = {};
1.166 + },
1.167 +
1.168 + del: function RecordMgr_del(url) {
1.169 + delete this._records[url];
1.170 + }
1.171 +};
1.172 +
1.173 +this.CryptoWrapper = function CryptoWrapper(collection, id) {
1.174 + this.cleartext = {};
1.175 + WBORecord.call(this, collection, id);
1.176 + this.ciphertext = null;
1.177 + this.id = id;
1.178 +}
1.179 +CryptoWrapper.prototype = {
1.180 + __proto__: WBORecord.prototype,
1.181 + _logName: "Sync.Record.CryptoWrapper",
1.182 +
1.183 + ciphertextHMAC: function ciphertextHMAC(keyBundle) {
1.184 + let hasher = keyBundle.sha256HMACHasher;
1.185 + if (!hasher) {
1.186 + throw "Cannot compute HMAC without an HMAC key.";
1.187 + }
1.188 +
1.189 + return Utils.bytesAsHex(Utils.digestUTF8(this.ciphertext, hasher));
1.190 + },
1.191 +
1.192 + /*
1.193 + * Don't directly use the sync key. Instead, grab a key for this
1.194 + * collection, which is decrypted with the sync key.
1.195 + *
1.196 + * Cache those keys; invalidate the cache if the time on the keys collection
1.197 + * changes, or other auth events occur.
1.198 + *
1.199 + * Optional key bundle overrides the collection key lookup.
1.200 + */
1.201 + encrypt: function encrypt(keyBundle) {
1.202 + if (!keyBundle) {
1.203 + throw new Error("A key bundle must be supplied to encrypt.");
1.204 + }
1.205 +
1.206 + this.IV = Svc.Crypto.generateRandomIV();
1.207 + this.ciphertext = Svc.Crypto.encrypt(JSON.stringify(this.cleartext),
1.208 + keyBundle.encryptionKeyB64, this.IV);
1.209 + this.hmac = this.ciphertextHMAC(keyBundle);
1.210 + this.cleartext = null;
1.211 + },
1.212 +
1.213 + // Optional key bundle.
1.214 + decrypt: function decrypt(keyBundle) {
1.215 + if (!this.ciphertext) {
1.216 + throw "No ciphertext: nothing to decrypt?";
1.217 + }
1.218 +
1.219 + if (!keyBundle) {
1.220 + throw new Error("A key bundle must be supplied to decrypt.");
1.221 + }
1.222 +
1.223 + // Authenticate the encrypted blob with the expected HMAC
1.224 + let computedHMAC = this.ciphertextHMAC(keyBundle);
1.225 +
1.226 + if (computedHMAC != this.hmac) {
1.227 + Utils.throwHMACMismatch(this.hmac, computedHMAC);
1.228 + }
1.229 +
1.230 + // Handle invalid data here. Elsewhere we assume that cleartext is an object.
1.231 + let cleartext = Svc.Crypto.decrypt(this.ciphertext,
1.232 + keyBundle.encryptionKeyB64, this.IV);
1.233 + let json_result = JSON.parse(cleartext);
1.234 +
1.235 + if (json_result && (json_result instanceof Object)) {
1.236 + this.cleartext = json_result;
1.237 + this.ciphertext = null;
1.238 + } else {
1.239 + throw "Decryption failed: result is <" + json_result + ">, not an object.";
1.240 + }
1.241 +
1.242 + // Verify that the encrypted id matches the requested record's id.
1.243 + if (this.cleartext.id != this.id)
1.244 + throw "Record id mismatch: " + this.cleartext.id + " != " + this.id;
1.245 +
1.246 + return this.cleartext;
1.247 + },
1.248 +
1.249 + toString: function toString() {
1.250 + let payload = this.deleted ? "DELETED" : JSON.stringify(this.cleartext);
1.251 +
1.252 + return "{ " +
1.253 + "id: " + this.id + " " +
1.254 + "index: " + this.sortindex + " " +
1.255 + "modified: " + this.modified + " " +
1.256 + "ttl: " + this.ttl + " " +
1.257 + "payload: " + payload + " " +
1.258 + "collection: " + (this.collection || "undefined") +
1.259 + " }";
1.260 + },
1.261 +
1.262 + // The custom setter below masks the parent's getter, so explicitly call it :(
1.263 + get id() WBORecord.prototype.__lookupGetter__("id").call(this),
1.264 +
1.265 + // Keep both plaintext and encrypted versions of the id to verify integrity
1.266 + set id(val) {
1.267 + WBORecord.prototype.__lookupSetter__("id").call(this, val);
1.268 + return this.cleartext.id = val;
1.269 + },
1.270 +};
1.271 +
1.272 +Utils.deferGetSet(CryptoWrapper, "payload", ["ciphertext", "IV", "hmac"]);
1.273 +Utils.deferGetSet(CryptoWrapper, "cleartext", "deleted");
1.274 +
1.275 +
1.276 +/**
1.277 + * Keeps track of mappings between collection names ('tabs') and KeyBundles.
1.278 + *
1.279 + * You can update this thing simply by giving it /info/collections. It'll
1.280 + * use the last modified time to bring itself up to date.
1.281 + */
1.282 +this.CollectionKeyManager = function CollectionKeyManager() {
1.283 + this.lastModified = 0;
1.284 + this._collections = {};
1.285 + this._default = null;
1.286 +
1.287 + this._log = Log.repository.getLogger("Sync.CollectionKeyManager");
1.288 +}
1.289 +
1.290 +// TODO: persist this locally as an Identity. Bug 610913.
1.291 +// Note that the last modified time needs to be preserved.
1.292 +CollectionKeyManager.prototype = {
1.293 +
1.294 + // Return information about old vs new keys:
1.295 + // * same: true if two collections are equal
1.296 + // * changed: an array of collection names that changed.
1.297 + _compareKeyBundleCollections: function _compareKeyBundleCollections(m1, m2) {
1.298 + let changed = [];
1.299 +
1.300 + function process(m1, m2) {
1.301 + for (let k1 in m1) {
1.302 + let v1 = m1[k1];
1.303 + let v2 = m2[k1];
1.304 + if (!(v1 && v2 && v1.equals(v2)))
1.305 + changed.push(k1);
1.306 + }
1.307 + }
1.308 +
1.309 + // Diffs both ways.
1.310 + process(m1, m2);
1.311 + process(m2, m1);
1.312 +
1.313 + // Return a sorted, unique array.
1.314 + changed.sort();
1.315 + let last;
1.316 + changed = [x for each (x in changed) if ((x != last) && (last = x))];
1.317 + return {same: changed.length == 0,
1.318 + changed: changed};
1.319 + },
1.320 +
1.321 + get isClear() {
1.322 + return !this._default;
1.323 + },
1.324 +
1.325 + clear: function clear() {
1.326 + this._log.info("Clearing collection keys...");
1.327 + this.lastModified = 0;
1.328 + this._collections = {};
1.329 + this._default = null;
1.330 + },
1.331 +
1.332 + keyForCollection: function(collection) {
1.333 + if (collection && this._collections[collection])
1.334 + return this._collections[collection];
1.335 +
1.336 + return this._default;
1.337 + },
1.338 +
1.339 + /**
1.340 + * If `collections` (an array of strings) is provided, iterate
1.341 + * over it and generate random keys for each collection.
1.342 + * Create a WBO for the given data.
1.343 + */
1.344 + _makeWBO: function(collections, defaultBundle) {
1.345 + let wbo = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
1.346 + let c = {};
1.347 + for (let k in collections) {
1.348 + c[k] = collections[k].keyPairB64;
1.349 + }
1.350 + wbo.cleartext = {
1.351 + "default": defaultBundle ? defaultBundle.keyPairB64 : null,
1.352 + "collections": c,
1.353 + "collection": CRYPTO_COLLECTION,
1.354 + "id": KEYS_WBO
1.355 + };
1.356 + return wbo;
1.357 + },
1.358 +
1.359 + /**
1.360 + * Create a WBO for the current keys.
1.361 + */
1.362 + asWBO: function(collection, id)
1.363 + this._makeWBO(this._collections, this._default),
1.364 +
1.365 + /**
1.366 + * Compute a new default key, and new keys for any specified collections.
1.367 + */
1.368 + newKeys: function(collections) {
1.369 + let newDefaultKey = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
1.370 + newDefaultKey.generateRandom();
1.371 +
1.372 + let newColls = {};
1.373 + if (collections) {
1.374 + collections.forEach(function (c) {
1.375 + let b = new BulkKeyBundle(c);
1.376 + b.generateRandom();
1.377 + newColls[c] = b;
1.378 + });
1.379 + }
1.380 + return [newDefaultKey, newColls];
1.381 + },
1.382 +
1.383 + /**
1.384 + * Generates new keys, but does not replace our local copy. Use this to
1.385 + * verify an upload before storing.
1.386 + */
1.387 + generateNewKeysWBO: function(collections) {
1.388 + let newDefaultKey, newColls;
1.389 + [newDefaultKey, newColls] = this.newKeys(collections);
1.390 +
1.391 + return this._makeWBO(newColls, newDefaultKey);
1.392 + },
1.393 +
1.394 + // Take the fetched info/collections WBO, checking the change
1.395 + // time of the crypto collection.
1.396 + updateNeeded: function(info_collections) {
1.397 +
1.398 + this._log.info("Testing for updateNeeded. Last modified: " + this.lastModified);
1.399 +
1.400 + // No local record of modification time? Need an update.
1.401 + if (!this.lastModified)
1.402 + return true;
1.403 +
1.404 + // No keys on the server? We need an update, though our
1.405 + // update handling will be a little more drastic...
1.406 + if (!(CRYPTO_COLLECTION in info_collections))
1.407 + return true;
1.408 +
1.409 + // Otherwise, we need an update if our modification time is stale.
1.410 + return (info_collections[CRYPTO_COLLECTION] > this.lastModified);
1.411 + },
1.412 +
1.413 + //
1.414 + // Set our keys and modified time to the values fetched from the server.
1.415 + // Returns one of three values:
1.416 + //
1.417 + // * If the default key was modified, return true.
1.418 + // * If the default key was not modified, but per-collection keys were,
1.419 + // return an array of such.
1.420 + // * Otherwise, return false -- we were up-to-date.
1.421 + //
1.422 + setContents: function setContents(payload, modified) {
1.423 +
1.424 + if (!modified)
1.425 + throw "No modified time provided to setContents.";
1.426 +
1.427 + let self = this;
1.428 +
1.429 + this._log.info("Setting collection keys contents. Our last modified: " +
1.430 + this.lastModified + ", input modified: " + modified + ".");
1.431 +
1.432 + if (!payload)
1.433 + throw "No payload in CollectionKeyManager.setContents().";
1.434 +
1.435 + if (!payload.default) {
1.436 + this._log.warn("No downloaded default key: this should not occur.");
1.437 + this._log.warn("Not clearing local keys.");
1.438 + throw "No default key in CollectionKeyManager.setContents(). Cannot proceed.";
1.439 + }
1.440 +
1.441 + // Process the incoming default key.
1.442 + let b = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
1.443 + b.keyPairB64 = payload.default;
1.444 + let newDefault = b;
1.445 +
1.446 + // Process the incoming collections.
1.447 + let newCollections = {};
1.448 + if ("collections" in payload) {
1.449 + this._log.info("Processing downloaded per-collection keys.");
1.450 + let colls = payload.collections;
1.451 + for (let k in colls) {
1.452 + let v = colls[k];
1.453 + if (v) {
1.454 + let keyObj = new BulkKeyBundle(k);
1.455 + keyObj.keyPairB64 = v;
1.456 + if (keyObj) {
1.457 + newCollections[k] = keyObj;
1.458 + }
1.459 + }
1.460 + }
1.461 + }
1.462 +
1.463 + // Check to see if these are already our keys.
1.464 + let sameDefault = (this._default && this._default.equals(newDefault));
1.465 + let collComparison = this._compareKeyBundleCollections(newCollections, this._collections);
1.466 + let sameColls = collComparison.same;
1.467 +
1.468 + if (sameDefault && sameColls) {
1.469 + self._log.info("New keys are the same as our old keys! Bumped local modified time.");
1.470 + self.lastModified = modified;
1.471 + return false;
1.472 + }
1.473 +
1.474 + // Make sure things are nice and tidy before we set.
1.475 + this.clear();
1.476 +
1.477 + this._log.info("Saving downloaded keys.");
1.478 + this._default = newDefault;
1.479 + this._collections = newCollections;
1.480 +
1.481 + // Always trust the server.
1.482 + self._log.info("Bumping last modified to " + modified);
1.483 + self.lastModified = modified;
1.484 +
1.485 + return sameDefault ? collComparison.changed : true;
1.486 + },
1.487 +
1.488 + updateContents: function updateContents(syncKeyBundle, storage_keys) {
1.489 + let log = this._log;
1.490 + log.info("Updating collection keys...");
1.491 +
1.492 + // storage_keys is a WBO, fetched from storage/crypto/keys.
1.493 + // Its payload is the default key, and a map of collections to keys.
1.494 + // We lazily compute the key objects from the strings we're given.
1.495 +
1.496 + let payload;
1.497 + try {
1.498 + payload = storage_keys.decrypt(syncKeyBundle);
1.499 + } catch (ex) {
1.500 + log.warn("Got exception \"" + ex + "\" decrypting storage keys with sync key.");
1.501 + log.info("Aborting updateContents. Rethrowing.");
1.502 + throw ex;
1.503 + }
1.504 +
1.505 + let r = this.setContents(payload, storage_keys.modified);
1.506 + log.info("Collection keys updated.");
1.507 + return r;
1.508 + }
1.509 +}
1.510 +
1.511 +this.Collection = function Collection(uri, recordObj, service) {
1.512 + if (!service) {
1.513 + throw new Error("Collection constructor requires a service.");
1.514 + }
1.515 +
1.516 + Resource.call(this, uri);
1.517 +
1.518 + // This is a bit hacky, but gets the job done.
1.519 + let res = service.resource(uri);
1.520 + this.authenticator = res.authenticator;
1.521 +
1.522 + this._recordObj = recordObj;
1.523 + this._service = service;
1.524 +
1.525 + this._full = false;
1.526 + this._ids = null;
1.527 + this._limit = 0;
1.528 + this._older = 0;
1.529 + this._newer = 0;
1.530 + this._data = [];
1.531 +}
1.532 +Collection.prototype = {
1.533 + __proto__: Resource.prototype,
1.534 + _logName: "Sync.Collection",
1.535 +
1.536 + _rebuildURL: function Coll__rebuildURL() {
1.537 + // XXX should consider what happens if it's not a URL...
1.538 + this.uri.QueryInterface(Ci.nsIURL);
1.539 +
1.540 + let args = [];
1.541 + if (this.older)
1.542 + args.push('older=' + this.older);
1.543 + else if (this.newer) {
1.544 + args.push('newer=' + this.newer);
1.545 + }
1.546 + if (this.full)
1.547 + args.push('full=1');
1.548 + if (this.sort)
1.549 + args.push('sort=' + this.sort);
1.550 + if (this.ids != null)
1.551 + args.push("ids=" + this.ids);
1.552 + if (this.limit > 0 && this.limit != Infinity)
1.553 + args.push("limit=" + this.limit);
1.554 +
1.555 + this.uri.query = (args.length > 0)? '?' + args.join('&') : '';
1.556 + },
1.557 +
1.558 + // get full items
1.559 + get full() { return this._full; },
1.560 + set full(value) {
1.561 + this._full = value;
1.562 + this._rebuildURL();
1.563 + },
1.564 +
1.565 + // Apply the action to a certain set of ids
1.566 + get ids() this._ids,
1.567 + set ids(value) {
1.568 + this._ids = value;
1.569 + this._rebuildURL();
1.570 + },
1.571 +
1.572 + // Limit how many records to get
1.573 + get limit() this._limit,
1.574 + set limit(value) {
1.575 + this._limit = value;
1.576 + this._rebuildURL();
1.577 + },
1.578 +
1.579 + // get only items modified before some date
1.580 + get older() { return this._older; },
1.581 + set older(value) {
1.582 + this._older = value;
1.583 + this._rebuildURL();
1.584 + },
1.585 +
1.586 + // get only items modified since some date
1.587 + get newer() { return this._newer; },
1.588 + set newer(value) {
1.589 + this._newer = value;
1.590 + this._rebuildURL();
1.591 + },
1.592 +
1.593 + // get items sorted by some criteria. valid values:
1.594 + // oldest (oldest first)
1.595 + // newest (newest first)
1.596 + // index
1.597 + get sort() { return this._sort; },
1.598 + set sort(value) {
1.599 + this._sort = value;
1.600 + this._rebuildURL();
1.601 + },
1.602 +
1.603 + pushData: function Coll_pushData(data) {
1.604 + this._data.push(data);
1.605 + },
1.606 +
1.607 + clearRecords: function Coll_clearRecords() {
1.608 + this._data = [];
1.609 + },
1.610 +
1.611 + set recordHandler(onRecord) {
1.612 + // Save this because onProgress is called with this as the ChannelListener
1.613 + let coll = this;
1.614 +
1.615 + // Switch to newline separated records for incremental parsing
1.616 + coll.setHeader("Accept", "application/newlines");
1.617 +
1.618 + this._onProgress = function() {
1.619 + let newline;
1.620 + while ((newline = this._data.indexOf("\n")) > 0) {
1.621 + // Split the json record from the rest of the data
1.622 + let json = this._data.slice(0, newline);
1.623 + this._data = this._data.slice(newline + 1);
1.624 +
1.625 + // Deserialize a record from json and give it to the callback
1.626 + let record = new coll._recordObj();
1.627 + record.deserialize(json);
1.628 + onRecord(record);
1.629 + }
1.630 + };
1.631 + },
1.632 +};
