The Tor Browser / file comparison

comparison: services/common/modules-testing/storageserver.js

services/common/modules-testing/storageserver.js

branch
TOR_BUG_9701
changeset 15
b8a032363ba2
equal deleted inserted replaced
-1:000000000000 0:4c84dbc4d0b3
1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
3 * You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5 /**
6 * This file contains an implementation of the Storage Server in JavaScript.
7 *
8 * The server should not be used for any production purposes.
9 */
10
11 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
12
13 this.EXPORTED_SYMBOLS = [
14 "ServerBSO",
15 "StorageServerCallback",
16 "StorageServerCollection",
17 "StorageServer",
18 "storageServerForUsers",
19 ];
20
21 Cu.import("resource://testing-common/httpd.js");
22 Cu.import("resource://services-common/async.js");
23 Cu.import("resource://gre/modules/Log.jsm");
24 Cu.import("resource://services-common/utils.js");
25
26 const STORAGE_HTTP_LOGGER = "Services.Common.Test.Server";
27 const STORAGE_API_VERSION = "2.0";
28
29 // Use the same method that record.js does, which mirrors the server.
30 function new_timestamp() {
31 return Math.round(Date.now());
32 }
33
34 function isInteger(s) {
35 let re = /^[0-9]+$/;
36 return re.test(s);
37 }
38
39 function writeHttpBody(response, body) {
40 if (!body) {
41 return;
42 }
43
44 response.bodyOutputStream.write(body, body.length);
45 }
46
47 function sendMozSvcError(request, response, code) {
48 response.setStatusLine(request.httpVersion, 400, "Bad Request");
49 response.setHeader("Content-Type", "text/plain", false);
50 response.bodyOutputStream.write(code, code.length);
51 }
52
53 /**
54 * Represent a BSO on the server.
55 *
56 * A BSO is constructed from an ID, content, and a modified time.
57 *
58 * @param id
59 * (string) ID of the BSO being created.
60 * @param payload
61 * (strong|object) Payload for the BSO. Should ideally be a string. If
62 * an object is passed, it will be fed into JSON.stringify and that
63 * output will be set as the payload.
64 * @param modified
65 * (number) Milliseconds since UNIX epoch that the BSO was last
66 * modified. If not defined or null, the current time will be used.
67 */
68 this.ServerBSO = function ServerBSO(id, payload, modified) {
69 if (!id) {
70 throw new Error("No ID for ServerBSO!");
71 }
72
73 if (!id.match(/^[a-zA-Z0-9_-]{1,64}$/)) {
74 throw new Error("BSO ID is invalid: " + id);
75 }
76
77 this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
78
79 this.id = id;
80 if (!payload) {
81 return;
82 }
83
84 CommonUtils.ensureMillisecondsTimestamp(modified);
85
86 if (typeof payload == "object") {
87 payload = JSON.stringify(payload);
88 }
89
90 this.payload = payload;
91 this.modified = modified || new_timestamp();
92 }
93 ServerBSO.prototype = {
94 FIELDS: [
95 "id",
96 "modified",
97 "payload",
98 "ttl",
99 "sortindex",
100 ],
101
102 toJSON: function toJSON() {
103 let obj = {};
104
105 for each (let key in this.FIELDS) {
106 if (this[key] !== undefined) {
107 obj[key] = this[key];
108 }
109 }
110
111 return obj;
112 },
113
114 delete: function delete_() {
115 this.deleted = true;
116
117 delete this.payload;
118 delete this.modified;
119 },
120
121 /**
122 * Handler for GET requests for this BSO.
123 */
124 getHandler: function getHandler(request, response) {
125 let code = 200;
126 let status = "OK";
127 let body;
128
129 function sendResponse() {
130 response.setStatusLine(request.httpVersion, code, status);
131 writeHttpBody(response, body);
132 }
133
134 if (request.hasHeader("x-if-modified-since")) {
135 let headerModified = parseInt(request.getHeader("x-if-modified-since"),
136 10);
137 CommonUtils.ensureMillisecondsTimestamp(headerModified);
138
139 if (headerModified >= this.modified) {
140 code = 304;
141 status = "Not Modified";
142
143 sendResponse();
144 return;
145 }
146 } else if (request.hasHeader("x-if-unmodified-since")) {
147 let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
148 10);
149 let serverModified = this.modified;
150
151 if (serverModified > requestModified) {
152 code = 412;
153 status = "Precondition Failed";
154 sendResponse();
155 return;
156 }
157 }
158
159 if (!this.deleted) {
160 body = JSON.stringify(this.toJSON());
161 response.setHeader("Content-Type", "application/json", false);
162 response.setHeader("X-Last-Modified", "" + this.modified, false);
163 } else {
164 code = 404;
165 status = "Not Found";
166 }
167
168 sendResponse();
169 },
170
171 /**
172 * Handler for PUT requests for this BSO.
173 */
174 putHandler: function putHandler(request, response) {
175 if (request.hasHeader("Content-Type")) {
176 let ct = request.getHeader("Content-Type");
177 if (ct != "application/json") {
178 throw HTTP_415;
179 }
180 }
181
182 let input = CommonUtils.readBytesFromInputStream(request.bodyInputStream);
183 let parsed;
184 try {
185 parsed = JSON.parse(input);
186 } catch (ex) {
187 return sendMozSvcError(request, response, "8");
188 }
189
190 if (typeof(parsed) != "object") {
191 return sendMozSvcError(request, response, "8");
192 }
193
194 // Don't update if a conditional request fails preconditions.
195 if (request.hasHeader("x-if-unmodified-since")) {
196 let reqModified = parseInt(request.getHeader("x-if-unmodified-since"));
197
198 if (reqModified < this.modified) {
199 response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
200 return;
201 }
202 }
203
204 let code, status;
205 if (this.payload) {
206 code = 204;
207 status = "No Content";
208 } else {
209 code = 201;
210 status = "Created";
211 }
212
213 // Alert when we see unrecognized fields.
214 for (let [key, value] in Iterator(parsed)) {
215 switch (key) {
216 case "payload":
217 if (typeof(value) != "string") {
218 sendMozSvcError(request, response, "8");
219 return true;
220 }
221
222 this.payload = value;
223 break;
224
225 case "ttl":
226 if (!isInteger(value)) {
227 sendMozSvcError(request, response, "8");
228 return true;
229 }
230 this.ttl = parseInt(value, 10);
231 break;
232
233 case "sortindex":
234 if (!isInteger(value) || value.length > 9) {
235 sendMozSvcError(request, response, "8");
236 return true;
237 }
238 this.sortindex = parseInt(value, 10);
239 break;
240
241 case "id":
242 break;
243
244 default:
245 this._log.warn("Unexpected field in BSO record: " + key);
246 sendMozSvcError(request, response, "8");
247 return true;
248 }
249 }
250
251 this.modified = request.timestamp;
252 this.deleted = false;
253 response.setHeader("X-Last-Modified", "" + this.modified, false);
254
255 response.setStatusLine(request.httpVersion, code, status);
256 },
257 };
258
259 /**
260 * Represent a collection on the server.
261 *
262 * The '_bsos' attribute is a mapping of id -> ServerBSO objects.
263 *
264 * Note that if you want these records to be accessible individually,
265 * you need to register their handlers with the server separately, or use a
266 * containing HTTP server that will do so on your behalf.
267 *
268 * @param bsos
269 * An object mapping BSO IDs to ServerBSOs.
270 * @param acceptNew
271 * If true, POSTs to this collection URI will result in new BSOs being
272 * created and wired in on the fly.
273 * @param timestamp
274 * An optional timestamp value to initialize the modified time of the
275 * collection. This should be in the format returned by new_timestamp().
276 */
277 this.StorageServerCollection =
278 function StorageServerCollection(bsos, acceptNew, timestamp=new_timestamp()) {
279 this._bsos = bsos || {};
280 this.acceptNew = acceptNew || false;
281
282 /*
283 * Track modified timestamp.
284 * We can't just use the timestamps of contained BSOs: an empty collection
285 * has a modified time.
286 */
287 CommonUtils.ensureMillisecondsTimestamp(timestamp);
288 this._timestamp = timestamp;
289
290 this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
291 }
292 StorageServerCollection.prototype = {
293 BATCH_MAX_COUNT: 100, // # of records.
294 BATCH_MAX_SIZE: 1024 * 1024, // # bytes.
295
296 _timestamp: null,
297
298 get timestamp() {
299 return this._timestamp;
300 },
301
302 set timestamp(timestamp) {
303 CommonUtils.ensureMillisecondsTimestamp(timestamp);
304 this._timestamp = timestamp;
305 },
306
307 get totalPayloadSize() {
308 let size = 0;
309 for each (let bso in this.bsos()) {
310 size += bso.payload.length;
311 }
312
313 return size;
314 },
315
316 /**
317 * Convenience accessor for our BSO keys.
318 * Excludes deleted items, of course.
319 *
320 * @param filter
321 * A predicate function (applied to the ID and BSO) which dictates
322 * whether to include the BSO's ID in the output.
323 *
324 * @return an array of IDs.
325 */
326 keys: function keys(filter) {
327 return [id for ([id, bso] in Iterator(this._bsos))
328 if (!bso.deleted && (!filter || filter(id, bso)))];
329 },
330
331 /**
332 * Convenience method to get an array of BSOs.
333 * Optionally provide a filter function.
334 *
335 * @param filter
336 * A predicate function, applied to the BSO, which dictates whether to
337 * include the BSO in the output.
338 *
339 * @return an array of ServerBSOs.
340 */
341 bsos: function bsos(filter) {
342 let os = [bso for ([id, bso] in Iterator(this._bsos))
343 if (!bso.deleted)];
344
345 if (!filter) {
346 return os;
347 }
348
349 return os.filter(filter);
350 },
351
352 /**
353 * Obtain a BSO by ID.
354 */
355 bso: function bso(id) {
356 return this._bsos[id];
357 },
358
359 /**
360 * Obtain the payload of a specific BSO.
361 *
362 * Raises if the specified BSO does not exist.
363 */
364 payload: function payload(id) {
365 return this.bso(id).payload;
366 },
367
368 /**
369 * Insert the provided BSO under its ID.
370 *
371 * @return the provided BSO.
372 */
373 insertBSO: function insertBSO(bso) {
374 return this._bsos[bso.id] = bso;
375 },
376
377 /**
378 * Insert the provided payload as part of a new ServerBSO with the provided
379 * ID.
380 *
381 * @param id
382 * The GUID for the BSO.
383 * @param payload
384 * The payload, as provided to the ServerBSO constructor.
385 * @param modified
386 * An optional modified time for the ServerBSO. If not specified, the
387 * current time will be used.
388 *
389 * @return the inserted BSO.
390 */
391 insert: function insert(id, payload, modified) {
392 return this.insertBSO(new ServerBSO(id, payload, modified));
393 },
394
395 /**
396 * Removes an object entirely from the collection.
397 *
398 * @param id
399 * (string) ID to remove.
400 */
401 remove: function remove(id) {
402 delete this._bsos[id];
403 },
404
405 _inResultSet: function _inResultSet(bso, options) {
406 if (!bso.payload) {
407 return false;
408 }
409
410 if (options.ids) {
411 if (options.ids.indexOf(bso.id) == -1) {
412 return false;
413 }
414 }
415
416 if (options.newer) {
417 if (bso.modified <= options.newer) {
418 return false;
419 }
420 }
421
422 if (options.older) {
423 if (bso.modified >= options.older) {
424 return false;
425 }
426 }
427
428 return true;
429 },
430
431 count: function count(options) {
432 options = options || {};
433 let c = 0;
434 for (let [id, bso] in Iterator(this._bsos)) {
435 if (bso.modified && this._inResultSet(bso, options)) {
436 c++;
437 }
438 }
439 return c;
440 },
441
442 get: function get(options) {
443 let data = [];
444 for each (let bso in this._bsos) {
445 if (!bso.modified) {
446 continue;
447 }
448
449 if (!this._inResultSet(bso, options)) {
450 continue;
451 }
452
453 data.push(bso);
454 }
455
456 if (options.sort) {
457 if (options.sort == "oldest") {
458 data.sort(function sortOldest(a, b) {
459 if (a.modified == b.modified) {
460 return 0;
461 }
462
463 return a.modified < b.modified ? -1 : 1;
464 });
465 } else if (options.sort == "newest") {
466 data.sort(function sortNewest(a, b) {
467 if (a.modified == b.modified) {
468 return 0;
469 }
470
471 return a.modified > b.modified ? -1 : 1;
472 });
473 } else if (options.sort == "index") {
474 data.sort(function sortIndex(a, b) {
475 if (a.sortindex == b.sortindex) {
476 return 0;
477 }
478
479 if (a.sortindex !== undefined && b.sortindex == undefined) {
480 return 1;
481 }
482
483 if (a.sortindex === undefined && b.sortindex !== undefined) {
484 return -1;
485 }
486
487 return a.sortindex > b.sortindex ? -1 : 1;
488 });
489 }
490 }
491
492 if (options.limit) {
493 data = data.slice(0, options.limit);
494 }
495
496 return data;
497 },
498
499 post: function post(input, timestamp) {
500 let success = [];
501 let failed = {};
502 let count = 0;
503 let size = 0;
504
505 // This will count records where we have an existing ServerBSO
506 // registered with us as successful and all other records as failed.
507 for each (let record in input) {
508 count += 1;
509 if (count > this.BATCH_MAX_COUNT) {
510 failed[record.id] = "Max record count exceeded.";
511 continue;
512 }
513
514 if (typeof(record.payload) != "string") {
515 failed[record.id] = "Payload is not a string!";
516 continue;
517 }
518
519 size += record.payload.length;
520 if (size > this.BATCH_MAX_SIZE) {
521 failed[record.id] = "Payload max size exceeded!";
522 continue;
523 }
524
525 if (record.sortindex) {
526 if (!isInteger(record.sortindex)) {
527 failed[record.id] = "sortindex is not an integer.";
528 continue;
529 }
530
531 if (record.sortindex.length > 9) {
532 failed[record.id] = "sortindex is too long.";
533 continue;
534 }
535 }
536
537 if ("ttl" in record) {
538 if (!isInteger(record.ttl)) {
539 failed[record.id] = "ttl is not an integer.";
540 continue;
541 }
542 }
543
544 try {
545 let bso = this.bso(record.id);
546 if (!bso && this.acceptNew) {
547 this._log.debug("Creating BSO " + JSON.stringify(record.id) +
548 " on the fly.");
549 bso = new ServerBSO(record.id);
550 this.insertBSO(bso);
551 }
552 if (bso) {
553 bso.payload = record.payload;
554 bso.modified = timestamp;
555 bso.deleted = false;
556 success.push(record.id);
557
558 if (record.sortindex) {
559 bso.sortindex = parseInt(record.sortindex, 10);
560 }
561
562 } else {
563 failed[record.id] = "no bso configured";
564 }
565 } catch (ex) {
566 this._log.info("Exception when processing BSO: " +
567 CommonUtils.exceptionStr(ex));
568 failed[record.id] = "Exception when processing.";
569 }
570 }
571 return {success: success, failed: failed};
572 },
573
574 delete: function delete_(options) {
575 options = options || {};
576
577 // Protocol 2.0 only allows the "ids" query string argument.
578 let keys = Object.keys(options).filter(function(k) {
579 return k != "ids";
580 });
581 if (keys.length) {
582 this._log.warn("Invalid query string parameter to collection delete: " +
583 keys.join(", "));
584 throw new Error("Malformed client request.");
585 }
586
587 if (options.ids && options.ids.length > this.BATCH_MAX_COUNT) {
588 throw HTTP_400;
589 }
590
591 let deleted = [];
592 for (let [id, bso] in Iterator(this._bsos)) {
593 if (this._inResultSet(bso, options)) {
594 this._log.debug("Deleting " + JSON.stringify(bso));
595 deleted.push(bso.id);
596 bso.delete();
597 }
598 }
599 return deleted;
600 },
601
602 parseOptions: function parseOptions(request) {
603 let options = {};
604
605 for each (let chunk in request.queryString.split("&")) {
606 if (!chunk) {
607 continue;
608 }
609 chunk = chunk.split("=");
610 let key = decodeURIComponent(chunk[0]);
611 if (chunk.length == 1) {
612 options[key] = "";
613 } else {
614 options[key] = decodeURIComponent(chunk[1]);
615 }
616 }
617
618 if (options.ids) {
619 options.ids = options.ids.split(",");
620 }
621
622 if (options.newer) {
623 if (!isInteger(options.newer)) {
624 throw HTTP_400;
625 }
626
627 CommonUtils.ensureMillisecondsTimestamp(options.newer);
628 options.newer = parseInt(options.newer, 10);
629 }
630
631 if (options.older) {
632 if (!isInteger(options.older)) {
633 throw HTTP_400;
634 }
635
636 CommonUtils.ensureMillisecondsTimestamp(options.older);
637 options.older = parseInt(options.older, 10);
638 }
639
640 if (options.limit) {
641 if (!isInteger(options.limit)) {
642 throw HTTP_400;
643 }
644
645 options.limit = parseInt(options.limit, 10);
646 }
647
648 return options;
649 },
650
651 getHandler: function getHandler(request, response) {
652 let options = this.parseOptions(request);
653 let data = this.get(options);
654
655 if (request.hasHeader("x-if-modified-since")) {
656 let requestModified = parseInt(request.getHeader("x-if-modified-since"),
657 10);
658 let newestBSO = 0;
659 for each (let bso in data) {
660 if (bso.modified > newestBSO) {
661 newestBSO = bso.modified;
662 }
663 }
664
665 if (requestModified >= newestBSO) {
666 response.setHeader("X-Last-Modified", "" + newestBSO);
667 response.setStatusLine(request.httpVersion, 304, "Not Modified");
668 return;
669 }
670 } else if (request.hasHeader("x-if-unmodified-since")) {
671 let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
672 10);
673 let serverModified = this.timestamp;
674
675 if (serverModified > requestModified) {
676 response.setHeader("X-Last-Modified", "" + serverModified);
677 response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
678 return;
679 }
680 }
681
682 if (options.full) {
683 data = data.map(function map(bso) {
684 return bso.toJSON();
685 });
686 } else {
687 data = data.map(function map(bso) {
688 return bso.id;
689 });
690 }
691
692 // application/json is default media type.
693 let newlines = false;
694 if (request.hasHeader("accept")) {
695 let accept = request.getHeader("accept");
696 if (accept == "application/newlines") {
697 newlines = true;
698 } else if (accept != "application/json") {
699 throw HTTP_406;
700 }
701 }
702
703 let body;
704 if (newlines) {
705 response.setHeader("Content-Type", "application/newlines", false);
706 let normalized = data.map(function map(d) {
707 return JSON.stringify(d);
708 });
709
710 body = normalized.join("\n") + "\n";
711 } else {
712 response.setHeader("Content-Type", "application/json", false);
713 body = JSON.stringify({items: data});
714 }
715
716 this._log.info("Records: " + data.length);
717 response.setHeader("X-Num-Records", "" + data.length, false);
718 response.setHeader("X-Last-Modified", "" + this.timestamp, false);
719 response.setStatusLine(request.httpVersion, 200, "OK");
720 response.bodyOutputStream.write(body, body.length);
721 },
722
723 postHandler: function postHandler(request, response) {
724 let options = this.parseOptions(request);
725
726 if (!request.hasHeader("content-type")) {
727 this._log.info("No Content-Type request header!");
728 throw HTTP_400;
729 }
730
731 let inputStream = request.bodyInputStream;
732 let inputBody = CommonUtils.readBytesFromInputStream(inputStream);
733 let input = [];
734
735 let inputMediaType = request.getHeader("content-type");
736 if (inputMediaType == "application/json") {
737 try {
738 input = JSON.parse(inputBody);
739 } catch (ex) {
740 this._log.info("JSON parse error on input body!");
741 throw HTTP_400;
742 }
743
744 if (!Array.isArray(input)) {
745 this._log.info("Input JSON type not an array!");
746 return sendMozSvcError(request, response, "8");
747 }
748 } else if (inputMediaType == "application/newlines") {
749 for each (let line in inputBody.split("\n")) {
750 let record;
751 try {
752 record = JSON.parse(line);
753 } catch (ex) {
754 this._log.info("JSON parse error on line!");
755 return sendMozSvcError(request, response, "8");
756 }
757
758 input.push(record);
759 }
760 } else {
761 this._log.info("Unknown media type: " + inputMediaType);
762 throw HTTP_415;
763 }
764
765 if (this._ensureUnmodifiedSince(request, response)) {
766 return;
767 }
768
769 let res = this.post(input, request.timestamp);
770 let body = JSON.stringify(res);
771 response.setHeader("Content-Type", "application/json", false);
772 this.timestamp = request.timestamp;
773 response.setHeader("X-Last-Modified", "" + this.timestamp, false);
774
775 response.setStatusLine(request.httpVersion, "200", "OK");
776 response.bodyOutputStream.write(body, body.length);
777 },
778
779 deleteHandler: function deleteHandler(request, response) {
780 this._log.debug("Invoking StorageServerCollection.DELETE.");
781
782 let options = this.parseOptions(request);
783
784 if (this._ensureUnmodifiedSince(request, response)) {
785 return;
786 }
787
788 let deleted = this.delete(options);
789 response.deleted = deleted;
790 this.timestamp = request.timestamp;
791
792 response.setStatusLine(request.httpVersion, 204, "No Content");
793 },
794
795 handler: function handler() {
796 let self = this;
797
798 return function(request, response) {
799 switch(request.method) {
800 case "GET":
801 return self.getHandler(request, response);
802
803 case "POST":
804 return self.postHandler(request, response);
805
806 case "DELETE":
807 return self.deleteHandler(request, response);
808
809 }
810
811 request.setHeader("Allow", "GET,POST,DELETE");
812 response.setStatusLine(request.httpVersion, 405, "Method Not Allowed");
813 };
814 },
815
816 _ensureUnmodifiedSince: function _ensureUnmodifiedSince(request, response) {
817 if (!request.hasHeader("x-if-unmodified-since")) {
818 return false;
819 }
820
821 let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
822 10);
823 let serverModified = this.timestamp;
824
825 this._log.debug("Request modified time: " + requestModified +
826 "; Server modified time: " + serverModified);
827 if (serverModified <= requestModified) {
828 return false;
829 }
830
831 this._log.info("Conditional request rejected because client time older " +
832 "than collection timestamp.");
833 response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
834 return true;
835 },
836 };
837
838
839 //===========================================================================//
840 // httpd.js-based Storage server. //
841 //===========================================================================//
842
843 /**
844 * In general, the preferred way of using StorageServer is to directly
845 * introspect it. Callbacks are available for operations which are hard to
846 * verify through introspection, such as deletions.
847 *
848 * One of the goals of this server is to provide enough hooks for test code to
849 * find out what it needs without monkeypatching. Use this object as your
850 * prototype, and override as appropriate.
851 */
852 this.StorageServerCallback = {
853 onCollectionDeleted: function onCollectionDeleted(user, collection) {},
854 onItemDeleted: function onItemDeleted(user, collection, bsoID) {},
855
856 /**
857 * Called at the top of every request.
858 *
859 * Allows the test to inspect the request. Hooks should be careful not to
860 * modify or change state of the request or they may impact future processing.
861 */
862 onRequest: function onRequest(request) {},
863 };
864
865 /**
866 * Construct a new test Storage server. Takes a callback object (e.g.,
867 * StorageServerCallback) as input.
868 */
869 this.StorageServer = function StorageServer(callback) {
870 this.callback = callback || {__proto__: StorageServerCallback};
871 this.server = new HttpServer();
872 this.started = false;
873 this.users = {};
874 this.requestCount = 0;
875 this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
876
877 // Install our own default handler. This allows us to mess around with the
878 // whole URL space.
879 let handler = this.server._handler;
880 handler._handleDefault = this.handleDefault.bind(this, handler);
881 }
882 StorageServer.prototype = {
883 DEFAULT_QUOTA: 1024 * 1024, // # bytes.
884
885 server: null, // HttpServer.
886 users: null, // Map of username => {collections, password}.
887
888 /**
889 * If true, the server will allow any arbitrary user to be used.
890 *
891 * No authentication will be performed. Whatever user is detected from the
892 * URL or auth headers will be created (if needed) and used.
893 */
894 allowAllUsers: false,
895
896 /**
897 * Start the StorageServer's underlying HTTP server.
898 *
899 * @param port
900 * The numeric port on which to start. A falsy value implies to
901 * select any available port.
902 * @param cb
903 * A callback function (of no arguments) which is invoked after
904 * startup.
905 */
906 start: function start(port, cb) {
907 if (this.started) {
908 this._log.warn("Warning: server already started on " + this.port);
909 return;
910 }
911 if (!port) {
912 port = -1;
913 }
914 this.port = port;
915
916 try {
917 this.server.start(this.port);
918 this.port = this.server.identity.primaryPort;
919 this.started = true;
920 if (cb) {
921 cb();
922 }
923 } catch (ex) {
924 _("==========================================");
925 _("Got exception starting Storage HTTP server on port " + this.port);
926 _("Error: " + CommonUtils.exceptionStr(ex));
927 _("Is there a process already listening on port " + this.port + "?");
928 _("==========================================");
929 do_throw(ex);
930 }
931 },
932
933 /**
934 * Start the server synchronously.
935 *
936 * @param port
937 * The numeric port on which to start. The default is to choose
938 * any available port.
939 */
940 startSynchronous: function startSynchronous(port=-1) {
941 let cb = Async.makeSpinningCallback();
942 this.start(port, cb);
943 cb.wait();
944 },
945
946 /**
947 * Stop the StorageServer's HTTP server.
948 *
949 * @param cb
950 * A callback function. Invoked after the server has been stopped.
951 *
952 */
953 stop: function stop(cb) {
954 if (!this.started) {
955 this._log.warn("StorageServer: Warning: server not running. Can't stop " +
956 "me now!");
957 return;
958 }
959
960 this.server.stop(cb);
961 this.started = false;
962 },
963
964 serverTime: function serverTime() {
965 return new_timestamp();
966 },
967
968 /**
969 * Create a new user, complete with an empty set of collections.
970 *
971 * @param username
972 * The username to use. An Error will be thrown if a user by that name
973 * already exists.
974 * @param password
975 * A password string.
976 *
977 * @return a user object, as would be returned by server.user(username).
978 */
979 registerUser: function registerUser(username, password) {
980 if (username in this.users) {
981 throw new Error("User already exists.");
982 }
983
984 if (!isFinite(parseInt(username))) {
985 throw new Error("Usernames must be numeric: " + username);
986 }
987
988 this._log.info("Registering new user with server: " + username);
989 this.users[username] = {
990 password: password,
991 collections: {},
992 quota: this.DEFAULT_QUOTA,
993 };
994 return this.user(username);
995 },
996
997 userExists: function userExists(username) {
998 return username in this.users;
999 },
1000
1001 getCollection: function getCollection(username, collection) {
1002 return this.users[username].collections[collection];
1003 },
1004
1005 _insertCollection: function _insertCollection(collections, collection, bsos) {
1006 let coll = new StorageServerCollection(bsos, true);
1007 coll.collectionHandler = coll.handler();
1008 collections[collection] = coll;
1009 return coll;
1010 },
1011
1012 createCollection: function createCollection(username, collection, bsos) {
1013 if (!(username in this.users)) {
1014 throw new Error("Unknown user.");
1015 }
1016 let collections = this.users[username].collections;
1017 if (collection in collections) {
1018 throw new Error("Collection already exists.");
1019 }
1020 return this._insertCollection(collections, collection, bsos);
1021 },
1022
1023 deleteCollection: function deleteCollection(username, collection) {
1024 if (!(username in this.users)) {
1025 throw new Error("Unknown user.");
1026 }
1027 delete this.users[username].collections[collection];
1028 },
1029
1030 /**
1031 * Accept a map like the following:
1032 * {
1033 * meta: {global: {version: 1, ...}},
1034 * crypto: {"keys": {}, foo: {bar: 2}},
1035 * bookmarks: {}
1036 * }
1037 * to cause collections and BSOs to be created.
1038 * If a collection already exists, no error is raised.
1039 * If a BSO already exists, it will be updated to the new contents.
1040 */
1041 createContents: function createContents(username, collections) {
1042 if (!(username in this.users)) {
1043 throw new Error("Unknown user.");
1044 }
1045 let userCollections = this.users[username].collections;
1046 for (let [id, contents] in Iterator(collections)) {
1047 let coll = userCollections[id] ||
1048 this._insertCollection(userCollections, id);
1049 for (let [bsoID, payload] in Iterator(contents)) {
1050 coll.insert(bsoID, payload);
1051 }
1052 }
1053 },
1054
1055 /**
1056 * Insert a BSO in an existing collection.
1057 */
1058 insertBSO: function insertBSO(username, collection, bso) {
1059 if (!(username in this.users)) {
1060 throw new Error("Unknown user.");
1061 }
1062 let userCollections = this.users[username].collections;
1063 if (!(collection in userCollections)) {
1064 throw new Error("Unknown collection.");
1065 }
1066 userCollections[collection].insertBSO(bso);
1067 return bso;
1068 },
1069
1070 /**
1071 * Delete all of the collections for the named user.
1072 *
1073 * @param username
1074 * The name of the affected user.
1075 */
1076 deleteCollections: function deleteCollections(username) {
1077 if (!(username in this.users)) {
1078 throw new Error("Unknown user.");
1079 }
1080 let userCollections = this.users[username].collections;
1081 for each (let [name, coll] in Iterator(userCollections)) {
1082 this._log.trace("Bulk deleting " + name + " for " + username + "...");
1083 coll.delete({});
1084 }
1085 this.users[username].collections = {};
1086 },
1087
1088 getQuota: function getQuota(username) {
1089 if (!(username in this.users)) {
1090 throw new Error("Unknown user.");
1091 }
1092
1093 return this.users[username].quota;
1094 },
1095
1096 /**
1097 * Obtain the newest timestamp of all collections for a user.
1098 */
1099 newestCollectionTimestamp: function newestCollectionTimestamp(username) {
1100 let collections = this.users[username].collections;
1101 let newest = 0;
1102 for each (let collection in collections) {
1103 if (collection.timestamp > newest) {
1104 newest = collection.timestamp;
1105 }
1106 }
1107
1108 return newest;
1109 },
1110
1111 /**
1112 * Compute the object that is returned for an info/collections request.
1113 */
1114 infoCollections: function infoCollections(username) {
1115 let responseObject = {};
1116 let colls = this.users[username].collections;
1117 for (let coll in colls) {
1118 responseObject[coll] = colls[coll].timestamp;
1119 }
1120 this._log.trace("StorageServer: info/collections returning " +
1121 JSON.stringify(responseObject));
1122 return responseObject;
1123 },
1124
1125 infoCounts: function infoCounts(username) {
1126 let data = {};
1127 let collections = this.users[username].collections;
1128 for (let [k, v] in Iterator(collections)) {
1129 let count = v.count();
1130 if (!count) {
1131 continue;
1132 }
1133
1134 data[k] = count;
1135 }
1136
1137 return data;
1138 },
1139
1140 infoUsage: function infoUsage(username) {
1141 let data = {};
1142 let collections = this.users[username].collections;
1143 for (let [k, v] in Iterator(collections)) {
1144 data[k] = v.totalPayloadSize;
1145 }
1146
1147 return data;
1148 },
1149
1150 infoQuota: function infoQuota(username) {
1151 let total = 0;
1152 for each (let value in this.infoUsage(username)) {
1153 total += value;
1154 }
1155
1156 return {
1157 quota: this.getQuota(username),
1158 usage: total
1159 };
1160 },
1161
1162 /**
1163 * Simple accessor to allow collective binding and abbreviation of a bunch of
1164 * methods. Yay!
1165 * Use like this:
1166 *
1167 * let u = server.user("john");
1168 * u.collection("bookmarks").bso("abcdefg").payload; // Etc.
1169 *
1170 * @return a proxy for the user data stored in this server.
1171 */
1172 user: function user(username) {
1173 let collection = this.getCollection.bind(this, username);
1174 let createCollection = this.createCollection.bind(this, username);
1175 let createContents = this.createContents.bind(this, username);
1176 let modified = function (collectionName) {
1177 return collection(collectionName).timestamp;
1178 }
1179 let deleteCollections = this.deleteCollections.bind(this, username);
1180 let quota = this.getQuota.bind(this, username);
1181 return {
1182 collection: collection,
1183 createCollection: createCollection,
1184 createContents: createContents,
1185 deleteCollections: deleteCollections,
1186 modified: modified,
1187 quota: quota,
1188 };
1189 },
1190
1191 _pruneExpired: function _pruneExpired() {
1192 let now = Date.now();
1193
1194 for each (let user in this.users) {
1195 for each (let collection in user.collections) {
1196 for each (let bso in collection.bsos()) {
1197 // ttl === 0 is a special case, so we can't simply !ttl.
1198 if (typeof(bso.ttl) != "number") {
1199 continue;
1200 }
1201
1202 let ttlDate = bso.modified + (bso.ttl * 1000);
1203 if (ttlDate < now) {
1204 this._log.info("Deleting BSO because TTL expired: " + bso.id);
1205 bso.delete();
1206 }
1207 }
1208 }
1209 }
1210 },
1211
1212 /*
1213 * Regular expressions for splitting up Storage request paths.
1214 * Storage URLs are of the form:
1215 * /$apipath/$version/$userid/$further
1216 * where $further is usually:
1217 * storage/$collection/$bso
1218 * or
1219 * storage/$collection
1220 * or
1221 * info/$op
1222 *
1223 * We assume for the sake of simplicity that $apipath is empty.
1224 *
1225 * N.B., we don't follow any kind of username spec here, because as far as I
1226 * can tell there isn't one. See Bug 689671. Instead we follow the Python
1227 * server code.
1228 *
1229 * Path: [all, version, first, rest]
1230 * Storage: [all, collection?, id?]
1231 */
1232 pathRE: /^\/([0-9]+(?:\.[0-9]+)?)(?:\/([0-9]+)\/([^\/]+)(?:\/(.+))?)?$/,
1233 storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/,
1234
1235 defaultHeaders: {},
1236
1237 /**
1238 * HTTP response utility.
1239 */
1240 respond: function respond(req, resp, code, status, body, headers, timestamp) {
1241 this._log.info("Response: " + code + " " + status);
1242 resp.setStatusLine(req.httpVersion, code, status);
1243 for each (let [header, value] in Iterator(headers || this.defaultHeaders)) {
1244 resp.setHeader(header, value, false);
1245 }
1246
1247 if (timestamp) {
1248 resp.setHeader("X-Timestamp", "" + timestamp, false);
1249 }
1250
1251 if (body) {
1252 resp.bodyOutputStream.write(body, body.length);
1253 }
1254 },
1255
1256 /**
1257 * This is invoked by the HttpServer. `this` is bound to the StorageServer;
1258 * `handler` is the HttpServer's handler.
1259 *
1260 * TODO: need to use the correct Storage API response codes and errors here.
1261 */
1262 handleDefault: function handleDefault(handler, req, resp) {
1263 this.requestCount++;
1264 let timestamp = new_timestamp();
1265 try {
1266 this._handleDefault(handler, req, resp, timestamp);
1267 } catch (e) {
1268 if (e instanceof HttpError) {
1269 this.respond(req, resp, e.code, e.description, "", {}, timestamp);
1270 } else {
1271 this._log.warn(CommonUtils.exceptionStr(e));
1272 throw e;
1273 }
1274 }
1275 },
1276
1277 _handleDefault: function _handleDefault(handler, req, resp, timestamp) {
1278 let path = req.path;
1279 if (req.queryString.length) {
1280 path += "?" + req.queryString;
1281 }
1282
1283 this._log.debug("StorageServer: Handling request: " + req.method + " " +
1284 path);
1285
1286 if (this.callback.onRequest) {
1287 this.callback.onRequest(req);
1288 }
1289
1290 // Prune expired records for all users at top of request. This is the
1291 // easiest way to process TTLs since all requests go through here.
1292 this._pruneExpired();
1293
1294 req.timestamp = timestamp;
1295 resp.setHeader("X-Timestamp", "" + timestamp, false);
1296
1297 let parts = this.pathRE.exec(req.path);
1298 if (!parts) {
1299 this._log.debug("StorageServer: Unexpected request: bad URL " + req.path);
1300 throw HTTP_404;
1301 }
1302
1303 let [all, version, userPath, first, rest] = parts;
1304 if (version != STORAGE_API_VERSION) {
1305 this._log.debug("StorageServer: Unknown version.");
1306 throw HTTP_404;
1307 }
1308
1309 let username;
1310
1311 // By default, the server requires users to be authenticated. When a
1312 // request arrives, the user must have been previously configured and
1313 // the request must have authentication. In "allow all users" mode, we
1314 // take the username from the URL, create the user on the fly, and don't
1315 // perform any authentication.
1316 if (!this.allowAllUsers) {
1317 // Enforce authentication.
1318 if (!req.hasHeader("authorization")) {
1319 this.respond(req, resp, 401, "Authorization Required", "{}", {
1320 "WWW-Authenticate": 'Basic realm="secret"'
1321 });
1322 return;
1323 }
1324
1325 let ensureUserExists = function ensureUserExists(username) {
1326 if (this.userExists(username)) {
1327 return;
1328 }
1329
1330 this._log.info("StorageServer: Unknown user: " + username);
1331 throw HTTP_401;
1332 }.bind(this);
1333
1334 let auth = req.getHeader("authorization");
1335 this._log.debug("Authorization: " + auth);
1336
1337 if (auth.indexOf("Basic ") == 0) {
1338 let decoded = CommonUtils.safeAtoB(auth.substr(6));
1339 this._log.debug("Decoded Basic Auth: " + decoded);
1340 let [user, password] = decoded.split(":", 2);
1341
1342 if (!password) {
1343 this._log.debug("Malformed HTTP Basic Authorization header: " + auth);
1344 throw HTTP_400;
1345 }
1346
1347 this._log.debug("Got HTTP Basic auth for user: " + user);
1348 ensureUserExists(user);
1349 username = user;
1350
1351 if (this.users[user].password != password) {
1352 this._log.debug("StorageServer: Provided password is not correct.");
1353 throw HTTP_401;
1354 }
1355 // TODO support token auth.
1356 } else {
1357 this._log.debug("Unsupported HTTP authorization type: " + auth);
1358 throw HTTP_500;
1359 }
1360 // All users mode.
1361 } else {
1362 // Auto create user with dummy password.
1363 if (!this.userExists(userPath)) {
1364 this.registerUser(userPath, "DUMMY-PASSWORD-*&%#");
1365 }
1366
1367 username = userPath;
1368 }
1369
1370 // Hand off to the appropriate handler for this path component.
1371 if (first in this.toplevelHandlers) {
1372 let handler = this.toplevelHandlers[first];
1373 try {
1374 return handler.call(this, handler, req, resp, version, username, rest);
1375 } catch (ex) {
1376 this._log.warn("Got exception during request: " +
1377 CommonUtils.exceptionStr(ex));
1378 throw ex;
1379 }
1380 }
1381 this._log.debug("StorageServer: Unknown top-level " + first);
1382 throw HTTP_404;
1383 },
1384
1385 /**
1386 * Collection of the handler methods we use for top-level path components.
1387 */
1388 toplevelHandlers: {
1389 "storage": function handleStorage(handler, req, resp, version, username,
1390 rest) {
1391 let respond = this.respond.bind(this, req, resp);
1392 if (!rest || !rest.length) {
1393 this._log.debug("StorageServer: top-level storage " +
1394 req.method + " request.");
1395
1396 if (req.method != "DELETE") {
1397 respond(405, "Method Not Allowed", null, {"Allow": "DELETE"});
1398 return;
1399 }
1400
1401 this.user(username).deleteCollections();
1402
1403 respond(204, "No Content");
1404 return;
1405 }
1406
1407 let match = this.storageRE.exec(rest);
1408 if (!match) {
1409 this._log.warn("StorageServer: Unknown storage operation " + rest);
1410 throw HTTP_404;
1411 }
1412 let [all, collection, bsoID] = match;
1413 let coll = this.getCollection(username, collection);
1414 let collectionExisted = !!coll;
1415
1416 switch (req.method) {
1417 case "GET":
1418 // Tried to GET on a collection that doesn't exist.
1419 if (!coll) {
1420 respond(404, "Not Found");
1421 return;
1422 }
1423
1424 // No BSO URL parameter goes to collection handler.
1425 if (!bsoID) {
1426 return coll.collectionHandler(req, resp);
1427 }
1428
1429 // Handle non-existent BSO.
1430 let bso = coll.bso(bsoID);
1431 if (!bso) {
1432 respond(404, "Not Found");
1433 return;
1434 }
1435
1436 // Proxy to BSO handler.
1437 return bso.getHandler(req, resp);
1438
1439 case "DELETE":
1440 // Collection doesn't exist.
1441 if (!coll) {
1442 respond(404, "Not Found");
1443 return;
1444 }
1445
1446 // Deleting a specific BSO.
1447 if (bsoID) {
1448 let bso = coll.bso(bsoID);
1449
1450 // BSO does not exist on the server. Nothing to do.
1451 if (!bso) {
1452 respond(404, "Not Found");
1453 return;
1454 }
1455
1456 if (req.hasHeader("x-if-unmodified-since")) {
1457 let modified = parseInt(req.getHeader("x-if-unmodified-since"));
1458 CommonUtils.ensureMillisecondsTimestamp(modified);
1459
1460 if (bso.modified > modified) {
1461 respond(412, "Precondition Failed");
1462 return;
1463 }
1464 }
1465
1466 bso.delete();
1467 coll.timestamp = req.timestamp;
1468 this.callback.onItemDeleted(username, collection, bsoID);
1469 respond(204, "No Content");
1470 return;
1471 }
1472
1473 // Proxy to collection handler.
1474 coll.collectionHandler(req, resp);
1475
1476 // Spot if this is a DELETE for some IDs, and don't blow away the
1477 // whole collection!
1478 //
1479 // We already handled deleting the BSOs by invoking the deleted
1480 // collection's handler. However, in the case of
1481 //
1482 // DELETE storage/foobar
1483 //
1484 // we also need to remove foobar from the collections map. This
1485 // clause tries to differentiate the above request from
1486 //
1487 // DELETE storage/foobar?ids=foo,baz
1488 //
1489 // and do the right thing.
1490 // TODO: less hacky method.
1491 if (-1 == req.queryString.indexOf("ids=")) {
1492 // When you delete the entire collection, we drop it.
1493 this._log.debug("Deleting entire collection.");
1494 delete this.users[username].collections[collection];
1495 this.callback.onCollectionDeleted(username, collection);
1496 }
1497
1498 // Notify of item deletion.
1499 let deleted = resp.deleted || [];
1500 for (let i = 0; i < deleted.length; ++i) {
1501 this.callback.onItemDeleted(username, collection, deleted[i]);
1502 }
1503 return;
1504
1505 case "POST":
1506 case "PUT":
1507 // Auto-create collection if it doesn't exist.
1508 if (!coll) {
1509 coll = this.createCollection(username, collection);
1510 }
1511
1512 try {
1513 if (bsoID) {
1514 let bso = coll.bso(bsoID);
1515 if (!bso) {
1516 this._log.trace("StorageServer: creating BSO " + collection +
1517 "/" + bsoID);
1518 try {
1519 bso = coll.insert(bsoID);
1520 } catch (ex) {
1521 return sendMozSvcError(req, resp, "8");
1522 }
1523 }
1524
1525 bso.putHandler(req, resp);
1526
1527 coll.timestamp = req.timestamp;
1528 return resp;
1529 }
1530
1531 return coll.collectionHandler(req, resp);
1532 } catch (ex) {
1533 if (ex instanceof HttpError) {
1534 if (!collectionExisted) {
1535 this.deleteCollection(username, collection);
1536 }
1537 }
1538
1539 throw ex;
1540 }
1541
1542 default:
1543 throw new Error("Request method " + req.method + " not implemented.");
1544 }
1545 },
1546
1547 "info": function handleInfo(handler, req, resp, version, username, rest) {
1548 switch (rest) {
1549 case "collections":
1550 return this.handleInfoCollections(req, resp, username);
1551
1552 case "collection_counts":
1553 return this.handleInfoCounts(req, resp, username);
1554
1555 case "collection_usage":
1556 return this.handleInfoUsage(req, resp, username);
1557
1558 case "quota":
1559 return this.handleInfoQuota(req, resp, username);
1560
1561 default:
1562 this._log.warn("StorageServer: Unknown info operation " + rest);
1563 throw HTTP_404;
1564 }
1565 }
1566 },
1567
1568 handleInfoConditional: function handleInfoConditional(request, response,
1569 user) {
1570 if (!request.hasHeader("x-if-modified-since")) {
1571 return false;
1572 }
1573
1574 let requestModified = request.getHeader("x-if-modified-since");
1575 requestModified = parseInt(requestModified, 10);
1576
1577 let serverModified = this.newestCollectionTimestamp(user);
1578
1579 this._log.info("Server mtime: " + serverModified + "; Client modified: " +
1580 requestModified);
1581 if (serverModified > requestModified) {
1582 return false;
1583 }
1584
1585 this.respond(request, response, 304, "Not Modified", null, {
1586 "X-Last-Modified": "" + serverModified
1587 });
1588
1589 return true;
1590 },
1591
1592 handleInfoCollections: function handleInfoCollections(request, response,
1593 user) {
1594 if (this.handleInfoConditional(request, response, user)) {
1595 return;
1596 }
1597
1598 let info = this.infoCollections(user);
1599 let body = JSON.stringify(info);
1600 this.respond(request, response, 200, "OK", body, {
1601 "Content-Type": "application/json",
1602 "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
1603 });
1604 },
1605
1606 handleInfoCounts: function handleInfoCounts(request, response, user) {
1607 if (this.handleInfoConditional(request, response, user)) {
1608 return;
1609 }
1610
1611 let counts = this.infoCounts(user);
1612 let body = JSON.stringify(counts);
1613
1614 this.respond(request, response, 200, "OK", body, {
1615 "Content-Type": "application/json",
1616 "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
1617 });
1618 },
1619
1620 handleInfoUsage: function handleInfoUsage(request, response, user) {
1621 if (this.handleInfoConditional(request, response, user)) {
1622 return;
1623 }
1624
1625 let body = JSON.stringify(this.infoUsage(user));
1626 this.respond(request, response, 200, "OK", body, {
1627 "Content-Type": "application/json",
1628 "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
1629 });
1630 },
1631
1632 handleInfoQuota: function handleInfoQuota(request, response, user) {
1633 if (this.handleInfoConditional(request, response, user)) {
1634 return;
1635 }
1636
1637 let body = JSON.stringify(this.infoQuota(user));
1638 this.respond(request, response, 200, "OK", body, {
1639 "Content-Type": "application/json",
1640 "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
1641 });
1642 },
1643 };
1644
1645 /**
1646 * Helper to create a storage server for a set of users.
1647 *
1648 * Each user is specified by a map of username to password.
1649 */
1650 this.storageServerForUsers =
1651 function storageServerForUsers(users, contents, callback) {
1652 let server = new StorageServer(callback);
1653 for (let [user, pass] in Iterator(users)) {
1654 server.registerUser(user, pass);
1655 server.createContents(user, contents);
1656 }
1657 server.start();
1658 return server;
1659 }

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial