services/sync/modules/record.js@b8a032363ba2
services/sync/modules/record.js
Thu, 22 Jan 2015 13:21:57 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Thu, 22 Jan 2015 13:21:57 +0100
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
- permissions
- -rw-r--r--
Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6
1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
5 this.EXPORTED_SYMBOLS = [
6 "WBORecord",
7 "RecordManager",
8 "CryptoWrapper",
9 "CollectionKeyManager",
10 "Collection",
11 ];
13 const Cc = Components.classes;
14 const Ci = Components.interfaces;
15 const Cr = Components.results;
16 const Cu = Components.utils;
18 const CRYPTO_COLLECTION = "crypto";
19 const KEYS_WBO = "keys";
21 Cu.import("resource://gre/modules/Log.jsm");
22 Cu.import("resource://services-sync/constants.js");
23 Cu.import("resource://services-sync/keys.js");
24 Cu.import("resource://services-sync/resource.js");
25 Cu.import("resource://services-sync/util.js");
27 this.WBORecord = function WBORecord(collection, id) {
28 this.data = {};
29 this.payload = {};
30 this.collection = collection; // Optional.
31 this.id = id; // Optional.
32 }
33 WBORecord.prototype = {
34 _logName: "Sync.Record.WBO",
36 get sortindex() {
37 if (this.data.sortindex)
38 return this.data.sortindex;
39 return 0;
40 },
42 // Get thyself from your URI, then deserialize.
43 // Set thine 'response' field.
44 fetch: function fetch(resource) {
45 if (!resource instanceof Resource) {
46 throw new Error("First argument must be a Resource instance.");
47 }
49 let r = resource.get();
50 if (r.success) {
51 this.deserialize(r); // Warning! Muffles exceptions!
52 }
53 this.response = r;
54 return this;
55 },
57 upload: function upload(resource) {
58 if (!resource instanceof Resource) {
59 throw new Error("First argument must be a Resource instance.");
60 }
62 return resource.put(this);
63 },
65 // Take a base URI string, with trailing slash, and return the URI of this
66 // WBO based on collection and ID.
67 uri: function(base) {
68 if (this.collection && this.id) {
69 let url = Utils.makeURI(base + this.collection + "/" + this.id);
70 url.QueryInterface(Ci.nsIURL);
71 return url;
72 }
73 return null;
74 },
76 deserialize: function deserialize(json) {
77 this.data = json.constructor.toString() == String ? JSON.parse(json) : json;
79 try {
80 // The payload is likely to be JSON, but if not, keep it as a string
81 this.payload = JSON.parse(this.payload);
82 } catch(ex) {}
83 },
85 toJSON: function toJSON() {
86 // Copy fields from data to be stringified, making sure payload is a string
87 let obj = {};
88 for (let [key, val] in Iterator(this.data))
89 obj[key] = key == "payload" ? JSON.stringify(val) : val;
90 if (this.ttl)
91 obj.ttl = this.ttl;
92 return obj;
93 },
95 toString: function toString() {
96 return "{ " +
97 "id: " + this.id + " " +
98 "index: " + this.sortindex + " " +
99 "modified: " + this.modified + " " +
100 "ttl: " + this.ttl + " " +
101 "payload: " + JSON.stringify(this.payload) +
102 " }";
103 }
104 };
106 Utils.deferGetSet(WBORecord, "data", ["id", "modified", "sortindex", "payload"]);
108 /**
109 * An interface and caching layer for records.
110 */
111 this.RecordManager = function RecordManager(service) {
112 this.service = service;
114 this._log = Log.repository.getLogger(this._logName);
115 this._records = {};
116 }
117 RecordManager.prototype = {
118 _recordType: WBORecord,
119 _logName: "Sync.RecordManager",
121 import: function RecordMgr_import(url) {
122 this._log.trace("Importing record: " + (url.spec ? url.spec : url));
123 try {
124 // Clear out the last response with empty object if GET fails
125 this.response = {};
126 this.response = this.service.resource(url).get();
128 // Don't parse and save the record on failure
129 if (!this.response.success)
130 return null;
132 let record = new this._recordType(url);
133 record.deserialize(this.response);
135 return this.set(url, record);
136 } catch(ex) {
137 this._log.debug("Failed to import record: " + Utils.exceptionStr(ex));
138 return null;
139 }
140 },
142 get: function RecordMgr_get(url) {
143 // Use a url string as the key to the hash
144 let spec = url.spec ? url.spec : url;
145 if (spec in this._records)
146 return this._records[spec];
147 return this.import(url);
148 },
150 set: function RecordMgr_set(url, record) {
151 let spec = url.spec ? url.spec : url;
152 return this._records[spec] = record;
153 },
155 contains: function RecordMgr_contains(url) {
156 if ((url.spec || url) in this._records)
157 return true;
158 return false;
159 },
161 clearCache: function recordMgr_clearCache() {
162 this._records = {};
163 },
165 del: function RecordMgr_del(url) {
166 delete this._records[url];
167 }
168 };
170 this.CryptoWrapper = function CryptoWrapper(collection, id) {
171 this.cleartext = {};
172 WBORecord.call(this, collection, id);
173 this.ciphertext = null;
174 this.id = id;
175 }
176 CryptoWrapper.prototype = {
177 __proto__: WBORecord.prototype,
178 _logName: "Sync.Record.CryptoWrapper",
180 ciphertextHMAC: function ciphertextHMAC(keyBundle) {
181 let hasher = keyBundle.sha256HMACHasher;
182 if (!hasher) {
183 throw "Cannot compute HMAC without an HMAC key.";
184 }
186 return Utils.bytesAsHex(Utils.digestUTF8(this.ciphertext, hasher));
187 },
189 /*
190 * Don't directly use the sync key. Instead, grab a key for this
191 * collection, which is decrypted with the sync key.
192 *
193 * Cache those keys; invalidate the cache if the time on the keys collection
194 * changes, or other auth events occur.
195 *
196 * Optional key bundle overrides the collection key lookup.
197 */
198 encrypt: function encrypt(keyBundle) {
199 if (!keyBundle) {
200 throw new Error("A key bundle must be supplied to encrypt.");
201 }
203 this.IV = Svc.Crypto.generateRandomIV();
204 this.ciphertext = Svc.Crypto.encrypt(JSON.stringify(this.cleartext),
205 keyBundle.encryptionKeyB64, this.IV);
206 this.hmac = this.ciphertextHMAC(keyBundle);
207 this.cleartext = null;
208 },
210 // Optional key bundle.
211 decrypt: function decrypt(keyBundle) {
212 if (!this.ciphertext) {
213 throw "No ciphertext: nothing to decrypt?";
214 }
216 if (!keyBundle) {
217 throw new Error("A key bundle must be supplied to decrypt.");
218 }
220 // Authenticate the encrypted blob with the expected HMAC
221 let computedHMAC = this.ciphertextHMAC(keyBundle);
223 if (computedHMAC != this.hmac) {
224 Utils.throwHMACMismatch(this.hmac, computedHMAC);
225 }
227 // Handle invalid data here. Elsewhere we assume that cleartext is an object.
228 let cleartext = Svc.Crypto.decrypt(this.ciphertext,
229 keyBundle.encryptionKeyB64, this.IV);
230 let json_result = JSON.parse(cleartext);
232 if (json_result && (json_result instanceof Object)) {
233 this.cleartext = json_result;
234 this.ciphertext = null;
235 } else {
236 throw "Decryption failed: result is <" + json_result + ">, not an object.";
237 }
239 // Verify that the encrypted id matches the requested record's id.
240 if (this.cleartext.id != this.id)
241 throw "Record id mismatch: " + this.cleartext.id + " != " + this.id;
243 return this.cleartext;
244 },
246 toString: function toString() {
247 let payload = this.deleted ? "DELETED" : JSON.stringify(this.cleartext);
249 return "{ " +
250 "id: " + this.id + " " +
251 "index: " + this.sortindex + " " +
252 "modified: " + this.modified + " " +
253 "ttl: " + this.ttl + " " +
254 "payload: " + payload + " " +
255 "collection: " + (this.collection || "undefined") +
256 " }";
257 },
259 // The custom setter below masks the parent's getter, so explicitly call it :(
260 get id() WBORecord.prototype.__lookupGetter__("id").call(this),
262 // Keep both plaintext and encrypted versions of the id to verify integrity
263 set id(val) {
264 WBORecord.prototype.__lookupSetter__("id").call(this, val);
265 return this.cleartext.id = val;
266 },
267 };
269 Utils.deferGetSet(CryptoWrapper, "payload", ["ciphertext", "IV", "hmac"]);
270 Utils.deferGetSet(CryptoWrapper, "cleartext", "deleted");
273 /**
274 * Keeps track of mappings between collection names ('tabs') and KeyBundles.
275 *
276 * You can update this thing simply by giving it /info/collections. It'll
277 * use the last modified time to bring itself up to date.
278 */
279 this.CollectionKeyManager = function CollectionKeyManager() {
280 this.lastModified = 0;
281 this._collections = {};
282 this._default = null;
284 this._log = Log.repository.getLogger("Sync.CollectionKeyManager");
285 }
287 // TODO: persist this locally as an Identity. Bug 610913.
288 // Note that the last modified time needs to be preserved.
289 CollectionKeyManager.prototype = {
291 // Return information about old vs new keys:
292 // * same: true if two collections are equal
293 // * changed: an array of collection names that changed.
294 _compareKeyBundleCollections: function _compareKeyBundleCollections(m1, m2) {
295 let changed = [];
297 function process(m1, m2) {
298 for (let k1 in m1) {
299 let v1 = m1[k1];
300 let v2 = m2[k1];
301 if (!(v1 && v2 && v1.equals(v2)))
302 changed.push(k1);
303 }
304 }
306 // Diffs both ways.
307 process(m1, m2);
308 process(m2, m1);
310 // Return a sorted, unique array.
311 changed.sort();
312 let last;
313 changed = [x for each (x in changed) if ((x != last) && (last = x))];
314 return {same: changed.length == 0,
315 changed: changed};
316 },
318 get isClear() {
319 return !this._default;
320 },
322 clear: function clear() {
323 this._log.info("Clearing collection keys...");
324 this.lastModified = 0;
325 this._collections = {};
326 this._default = null;
327 },
329 keyForCollection: function(collection) {
330 if (collection && this._collections[collection])
331 return this._collections[collection];
333 return this._default;
334 },
336 /**
337 * If `collections` (an array of strings) is provided, iterate
338 * over it and generate random keys for each collection.
339 * Create a WBO for the given data.
340 */
341 _makeWBO: function(collections, defaultBundle) {
342 let wbo = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
343 let c = {};
344 for (let k in collections) {
345 c[k] = collections[k].keyPairB64;
346 }
347 wbo.cleartext = {
348 "default": defaultBundle ? defaultBundle.keyPairB64 : null,
349 "collections": c,
350 "collection": CRYPTO_COLLECTION,
351 "id": KEYS_WBO
352 };
353 return wbo;
354 },
356 /**
357 * Create a WBO for the current keys.
358 */
359 asWBO: function(collection, id)
360 this._makeWBO(this._collections, this._default),
362 /**
363 * Compute a new default key, and new keys for any specified collections.
364 */
365 newKeys: function(collections) {
366 let newDefaultKey = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
367 newDefaultKey.generateRandom();
369 let newColls = {};
370 if (collections) {
371 collections.forEach(function (c) {
372 let b = new BulkKeyBundle(c);
373 b.generateRandom();
374 newColls[c] = b;
375 });
376 }
377 return [newDefaultKey, newColls];
378 },
380 /**
381 * Generates new keys, but does not replace our local copy. Use this to
382 * verify an upload before storing.
383 */
384 generateNewKeysWBO: function(collections) {
385 let newDefaultKey, newColls;
386 [newDefaultKey, newColls] = this.newKeys(collections);
388 return this._makeWBO(newColls, newDefaultKey);
389 },
391 // Take the fetched info/collections WBO, checking the change
392 // time of the crypto collection.
393 updateNeeded: function(info_collections) {
395 this._log.info("Testing for updateNeeded. Last modified: " + this.lastModified);
397 // No local record of modification time? Need an update.
398 if (!this.lastModified)
399 return true;
401 // No keys on the server? We need an update, though our
402 // update handling will be a little more drastic...
403 if (!(CRYPTO_COLLECTION in info_collections))
404 return true;
406 // Otherwise, we need an update if our modification time is stale.
407 return (info_collections[CRYPTO_COLLECTION] > this.lastModified);
408 },
410 //
411 // Set our keys and modified time to the values fetched from the server.
412 // Returns one of three values:
413 //
414 // * If the default key was modified, return true.
415 // * If the default key was not modified, but per-collection keys were,
416 // return an array of such.
417 // * Otherwise, return false -- we were up-to-date.
418 //
419 setContents: function setContents(payload, modified) {
421 if (!modified)
422 throw "No modified time provided to setContents.";
424 let self = this;
426 this._log.info("Setting collection keys contents. Our last modified: " +
427 this.lastModified + ", input modified: " + modified + ".");
429 if (!payload)
430 throw "No payload in CollectionKeyManager.setContents().";
432 if (!payload.default) {
433 this._log.warn("No downloaded default key: this should not occur.");
434 this._log.warn("Not clearing local keys.");
435 throw "No default key in CollectionKeyManager.setContents(). Cannot proceed.";
436 }
438 // Process the incoming default key.
439 let b = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
440 b.keyPairB64 = payload.default;
441 let newDefault = b;
443 // Process the incoming collections.
444 let newCollections = {};
445 if ("collections" in payload) {
446 this._log.info("Processing downloaded per-collection keys.");
447 let colls = payload.collections;
448 for (let k in colls) {
449 let v = colls[k];
450 if (v) {
451 let keyObj = new BulkKeyBundle(k);
452 keyObj.keyPairB64 = v;
453 if (keyObj) {
454 newCollections[k] = keyObj;
455 }
456 }
457 }
458 }
460 // Check to see if these are already our keys.
461 let sameDefault = (this._default && this._default.equals(newDefault));
462 let collComparison = this._compareKeyBundleCollections(newCollections, this._collections);
463 let sameColls = collComparison.same;
465 if (sameDefault && sameColls) {
466 self._log.info("New keys are the same as our old keys! Bumped local modified time.");
467 self.lastModified = modified;
468 return false;
469 }
471 // Make sure things are nice and tidy before we set.
472 this.clear();
474 this._log.info("Saving downloaded keys.");
475 this._default = newDefault;
476 this._collections = newCollections;
478 // Always trust the server.
479 self._log.info("Bumping last modified to " + modified);
480 self.lastModified = modified;
482 return sameDefault ? collComparison.changed : true;
483 },
485 updateContents: function updateContents(syncKeyBundle, storage_keys) {
486 let log = this._log;
487 log.info("Updating collection keys...");
489 // storage_keys is a WBO, fetched from storage/crypto/keys.
490 // Its payload is the default key, and a map of collections to keys.
491 // We lazily compute the key objects from the strings we're given.
493 let payload;
494 try {
495 payload = storage_keys.decrypt(syncKeyBundle);
496 } catch (ex) {
497 log.warn("Got exception \"" + ex + "\" decrypting storage keys with sync key.");
498 log.info("Aborting updateContents. Rethrowing.");
499 throw ex;
500 }
502 let r = this.setContents(payload, storage_keys.modified);
503 log.info("Collection keys updated.");
504 return r;
505 }
506 }
508 this.Collection = function Collection(uri, recordObj, service) {
509 if (!service) {
510 throw new Error("Collection constructor requires a service.");
511 }
513 Resource.call(this, uri);
515 // This is a bit hacky, but gets the job done.
516 let res = service.resource(uri);
517 this.authenticator = res.authenticator;
519 this._recordObj = recordObj;
520 this._service = service;
522 this._full = false;
523 this._ids = null;
524 this._limit = 0;
525 this._older = 0;
526 this._newer = 0;
527 this._data = [];
528 }
529 Collection.prototype = {
530 __proto__: Resource.prototype,
531 _logName: "Sync.Collection",
533 _rebuildURL: function Coll__rebuildURL() {
534 // XXX should consider what happens if it's not a URL...
535 this.uri.QueryInterface(Ci.nsIURL);
537 let args = [];
538 if (this.older)
539 args.push('older=' + this.older);
540 else if (this.newer) {
541 args.push('newer=' + this.newer);
542 }
543 if (this.full)
544 args.push('full=1');
545 if (this.sort)
546 args.push('sort=' + this.sort);
547 if (this.ids != null)
548 args.push("ids=" + this.ids);
549 if (this.limit > 0 && this.limit != Infinity)
550 args.push("limit=" + this.limit);
552 this.uri.query = (args.length > 0)? '?' + args.join('&') : '';
553 },
555 // get full items
556 get full() { return this._full; },
557 set full(value) {
558 this._full = value;
559 this._rebuildURL();
560 },
562 // Apply the action to a certain set of ids
563 get ids() this._ids,
564 set ids(value) {
565 this._ids = value;
566 this._rebuildURL();
567 },
569 // Limit how many records to get
570 get limit() this._limit,
571 set limit(value) {
572 this._limit = value;
573 this._rebuildURL();
574 },
576 // get only items modified before some date
577 get older() { return this._older; },
578 set older(value) {
579 this._older = value;
580 this._rebuildURL();
581 },
583 // get only items modified since some date
584 get newer() { return this._newer; },
585 set newer(value) {
586 this._newer = value;
587 this._rebuildURL();
588 },
590 // get items sorted by some criteria. valid values:
591 // oldest (oldest first)
592 // newest (newest first)
593 // index
594 get sort() { return this._sort; },
595 set sort(value) {
596 this._sort = value;
597 this._rebuildURL();
598 },
600 pushData: function Coll_pushData(data) {
601 this._data.push(data);
602 },
604 clearRecords: function Coll_clearRecords() {
605 this._data = [];
606 },
608 set recordHandler(onRecord) {
609 // Save this because onProgress is called with this as the ChannelListener
610 let coll = this;
612 // Switch to newline separated records for incremental parsing
613 coll.setHeader("Accept", "application/newlines");
615 this._onProgress = function() {
616 let newline;
617 while ((newline = this._data.indexOf("\n")) > 0) {
618 // Split the json record from the rest of the data
619 let json = this._data.slice(0, newline);
620 this._data = this._data.slice(newline + 1);
622 // Deserialize a record from json and give it to the callback
623 let record = new coll._recordObj();
624 record.deserialize(json);
625 onRecord(record);
626 }
627 };
628 },
629 };
