michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public
michael@0:  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0:  * You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0: 
michael@0: /**
michael@0:  * This file contains an implementation of the Storage Server in JavaScript.
michael@0:  *
michael@0:  * The server should not be used for any production purposes.
michael@0:  */
michael@0: 
michael@0: const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
michael@0: 
michael@0: this.EXPORTED_SYMBOLS = [
michael@0:   "ServerBSO",
michael@0:   "StorageServerCallback",
michael@0:   "StorageServerCollection",
michael@0:   "StorageServer",
michael@0:   "storageServerForUsers",
michael@0: ];
michael@0: 
michael@0: Cu.import("resource://testing-common/httpd.js");
michael@0: Cu.import("resource://services-common/async.js");
michael@0: Cu.import("resource://gre/modules/Log.jsm");
michael@0: Cu.import("resource://services-common/utils.js");
michael@0: 
michael@0: const STORAGE_HTTP_LOGGER = "Services.Common.Test.Server";
michael@0: const STORAGE_API_VERSION = "2.0";
michael@0: 
michael@0: // Use the same method that record.js does, which mirrors the server.
michael@0: function new_timestamp() {
michael@0:   return Math.round(Date.now());
michael@0: }
michael@0: 
michael@0: function isInteger(s) {
michael@0:   let re = /^[0-9]+$/;
michael@0:   return re.test(s);
michael@0: }
michael@0: 
michael@0: function writeHttpBody(response, body) {
michael@0:   if (!body) {
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   response.bodyOutputStream.write(body, body.length);
michael@0: }
michael@0: 
michael@0: function sendMozSvcError(request, response, code) {
michael@0:   response.setStatusLine(request.httpVersion, 400, "Bad Request");
michael@0:   response.setHeader("Content-Type", "text/plain", false);
michael@0:   response.bodyOutputStream.write(code, code.length);
michael@0: }
michael@0: 
michael@0: /**
michael@0:  * Represent a BSO on the server.
michael@0:  *
michael@0:  * A BSO is constructed from an ID, content, and a modified time.
michael@0:  *
michael@0:  * @param id
michael@0:  *        (string) ID of the BSO being created.
michael@0:  * @param payload
michael@0:  *        (strong|object) Payload for the BSO. Should ideally be a string. If
michael@0:  *        an object is passed, it will be fed into JSON.stringify and that
michael@0:  *        output will be set as the payload.
michael@0:  * @param modified
michael@0:  *        (number) Milliseconds since UNIX epoch that the BSO was last
michael@0:  *        modified. If not defined or null, the current time will be used.
michael@0:  */
michael@0: this.ServerBSO = function ServerBSO(id, payload, modified) {
michael@0:   if (!id) {
michael@0:     throw new Error("No ID for ServerBSO!");
michael@0:   }
michael@0: 
michael@0:   if (!id.match(/^[a-zA-Z0-9_-]{1,64}$/)) {
michael@0:     throw new Error("BSO ID is invalid: " + id);
michael@0:   }
michael@0: 
michael@0:   this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
michael@0: 
michael@0:   this.id = id;
michael@0:   if (!payload) {
michael@0:     return;
michael@0:   }
michael@0: 
michael@0:   CommonUtils.ensureMillisecondsTimestamp(modified);
michael@0: 
michael@0:   if (typeof payload == "object") {
michael@0:     payload = JSON.stringify(payload);
michael@0:   }
michael@0: 
michael@0:   this.payload = payload;
michael@0:   this.modified = modified || new_timestamp();
michael@0: }
michael@0: ServerBSO.prototype = {
michael@0:   FIELDS: [
michael@0:     "id",
michael@0:     "modified",
michael@0:     "payload",
michael@0:     "ttl",
michael@0:     "sortindex",
michael@0:   ],
michael@0: 
michael@0:   toJSON: function toJSON() {
michael@0:     let obj = {};
michael@0: 
michael@0:     for each (let key in this.FIELDS) {
michael@0:       if (this[key] !== undefined) {
michael@0:         obj[key] = this[key];
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return obj;
michael@0:   },
michael@0: 
michael@0:   delete: function delete_() {
michael@0:     this.deleted = true;
michael@0: 
michael@0:     delete this.payload;
michael@0:     delete this.modified;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Handler for GET requests for this BSO.
michael@0:    */
michael@0:   getHandler: function getHandler(request, response) {
michael@0:     let code = 200;
michael@0:     let status = "OK";
michael@0:     let body;
michael@0: 
michael@0:     function sendResponse() {
michael@0:       response.setStatusLine(request.httpVersion, code, status);
michael@0:       writeHttpBody(response, body);
michael@0:     }
michael@0: 
michael@0:     if (request.hasHeader("x-if-modified-since")) {
michael@0:       let headerModified = parseInt(request.getHeader("x-if-modified-since"),
michael@0:                                     10);
michael@0:       CommonUtils.ensureMillisecondsTimestamp(headerModified);
michael@0: 
michael@0:       if (headerModified >= this.modified) {
michael@0:         code = 304;
michael@0:         status = "Not Modified";
michael@0: 
michael@0:         sendResponse();
michael@0:         return;
michael@0:       }
michael@0:     } else if (request.hasHeader("x-if-unmodified-since")) {
michael@0:       let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
michael@0:                                      10);
michael@0:       let serverModified = this.modified;
michael@0: 
michael@0:       if (serverModified > requestModified) {
michael@0:         code = 412;
michael@0:         status = "Precondition Failed";
michael@0:         sendResponse();
michael@0:         return;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (!this.deleted) {
michael@0:       body = JSON.stringify(this.toJSON());
michael@0:       response.setHeader("Content-Type", "application/json", false);
michael@0:       response.setHeader("X-Last-Modified", "" + this.modified, false);
michael@0:     } else {
michael@0:       code = 404;
michael@0:       status = "Not Found";
michael@0:     }
michael@0: 
michael@0:     sendResponse();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Handler for PUT requests for this BSO.
michael@0:    */
michael@0:   putHandler: function putHandler(request, response) {
michael@0:     if (request.hasHeader("Content-Type")) {
michael@0:       let ct = request.getHeader("Content-Type");
michael@0:       if (ct != "application/json") {
michael@0:         throw HTTP_415;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     let input = CommonUtils.readBytesFromInputStream(request.bodyInputStream);
michael@0:     let parsed;
michael@0:     try {
michael@0:       parsed = JSON.parse(input);
michael@0:     } catch (ex) {
michael@0:       return sendMozSvcError(request, response, "8");
michael@0:     }
michael@0: 
michael@0:     if (typeof(parsed) != "object") {
michael@0:       return sendMozSvcError(request, response, "8");
michael@0:     }
michael@0: 
michael@0:     // Don't update if a conditional request fails preconditions.
michael@0:     if (request.hasHeader("x-if-unmodified-since")) {
michael@0:       let reqModified = parseInt(request.getHeader("x-if-unmodified-since"));
michael@0: 
michael@0:       if (reqModified < this.modified) {
michael@0:         response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
michael@0:         return;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     let code, status;
michael@0:     if (this.payload) {
michael@0:       code = 204;
michael@0:       status = "No Content";
michael@0:     } else {
michael@0:       code = 201;
michael@0:       status = "Created";
michael@0:     }
michael@0: 
michael@0:     // Alert when we see unrecognized fields.
michael@0:     for (let [key, value] in Iterator(parsed)) {
michael@0:       switch (key) {
michael@0:         case "payload":
michael@0:           if (typeof(value) != "string") {
michael@0:             sendMozSvcError(request, response, "8");
michael@0:             return true;
michael@0:           }
michael@0: 
michael@0:           this.payload = value;
michael@0:           break;
michael@0: 
michael@0:         case "ttl":
michael@0:           if (!isInteger(value)) {
michael@0:             sendMozSvcError(request, response, "8");
michael@0:             return true;
michael@0:           }
michael@0:           this.ttl = parseInt(value, 10);
michael@0:           break;
michael@0: 
michael@0:         case "sortindex":
michael@0:           if (!isInteger(value) || value.length > 9) {
michael@0:             sendMozSvcError(request, response, "8");
michael@0:             return true;
michael@0:           }
michael@0:           this.sortindex = parseInt(value, 10);
michael@0:           break;
michael@0: 
michael@0:         case "id":
michael@0:           break;
michael@0: 
michael@0:         default:
michael@0:           this._log.warn("Unexpected field in BSO record: " + key);
michael@0:           sendMozSvcError(request, response, "8");
michael@0:           return true;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     this.modified = request.timestamp;
michael@0:     this.deleted = false;
michael@0:     response.setHeader("X-Last-Modified", "" + this.modified, false);
michael@0: 
michael@0:     response.setStatusLine(request.httpVersion, code, status);
michael@0:   },
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Represent a collection on the server.
michael@0:  *
michael@0:  * The '_bsos' attribute is a mapping of id -> ServerBSO objects.
michael@0:  *
michael@0:  * Note that if you want these records to be accessible individually,
michael@0:  * you need to register their handlers with the server separately, or use a
michael@0:  * containing HTTP server that will do so on your behalf.
michael@0:  *
michael@0:  * @param bsos
michael@0:  *        An object mapping BSO IDs to ServerBSOs.
michael@0:  * @param acceptNew
michael@0:  *        If true, POSTs to this collection URI will result in new BSOs being
michael@0:  *        created and wired in on the fly.
michael@0:  * @param timestamp
michael@0:  *        An optional timestamp value to initialize the modified time of the
michael@0:  *        collection. This should be in the format returned by new_timestamp().
michael@0:  */
michael@0: this.StorageServerCollection =
michael@0:  function StorageServerCollection(bsos, acceptNew, timestamp=new_timestamp()) {
michael@0:   this._bsos = bsos || {};
michael@0:   this.acceptNew = acceptNew || false;
michael@0: 
michael@0:   /*
michael@0:    * Track modified timestamp.
michael@0:    * We can't just use the timestamps of contained BSOs: an empty collection
michael@0:    * has a modified time.
michael@0:    */
michael@0:   CommonUtils.ensureMillisecondsTimestamp(timestamp);
michael@0:   this._timestamp = timestamp;
michael@0: 
michael@0:   this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
michael@0: }
michael@0: StorageServerCollection.prototype = {
michael@0:   BATCH_MAX_COUNT: 100,         // # of records.
michael@0:   BATCH_MAX_SIZE: 1024 * 1024,  // # bytes.
michael@0: 
michael@0:   _timestamp: null,
michael@0: 
michael@0:   get timestamp() {
michael@0:     return this._timestamp;
michael@0:   },
michael@0: 
michael@0:   set timestamp(timestamp) {
michael@0:     CommonUtils.ensureMillisecondsTimestamp(timestamp);
michael@0:     this._timestamp = timestamp;
michael@0:   },
michael@0: 
michael@0:   get totalPayloadSize() {
michael@0:     let size = 0;
michael@0:     for each (let bso in this.bsos()) {
michael@0:       size += bso.payload.length;
michael@0:     }
michael@0: 
michael@0:     return size;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Convenience accessor for our BSO keys.
michael@0:    * Excludes deleted items, of course.
michael@0:    *
michael@0:    * @param filter
michael@0:    *        A predicate function (applied to the ID and BSO) which dictates
michael@0:    *        whether to include the BSO's ID in the output.
michael@0:    *
michael@0:    * @return an array of IDs.
michael@0:    */
michael@0:   keys: function keys(filter) {
michael@0:     return [id for ([id, bso] in Iterator(this._bsos))
michael@0:                if (!bso.deleted && (!filter || filter(id, bso)))];
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Convenience method to get an array of BSOs.
michael@0:    * Optionally provide a filter function.
michael@0:    *
michael@0:    * @param filter
michael@0:    *        A predicate function, applied to the BSO, which dictates whether to
michael@0:    *        include the BSO in the output.
michael@0:    *
michael@0:    * @return an array of ServerBSOs.
michael@0:    */
michael@0:   bsos: function bsos(filter) {
michael@0:     let os = [bso for ([id, bso] in Iterator(this._bsos))
michael@0:               if (!bso.deleted)];
michael@0: 
michael@0:     if (!filter) {
michael@0:       return os;
michael@0:     }
michael@0: 
michael@0:     return os.filter(filter);
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain a BSO by ID.
michael@0:    */
michael@0:   bso: function bso(id) {
michael@0:     return this._bsos[id];
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain the payload of a specific BSO.
michael@0:    *
michael@0:    * Raises if the specified BSO does not exist.
michael@0:    */
michael@0:   payload: function payload(id) {
michael@0:     return this.bso(id).payload;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Insert the provided BSO under its ID.
michael@0:    *
michael@0:    * @return the provided BSO.
michael@0:    */
michael@0:   insertBSO: function insertBSO(bso) {
michael@0:     return this._bsos[bso.id] = bso;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Insert the provided payload as part of a new ServerBSO with the provided
michael@0:    * ID.
michael@0:    *
michael@0:    * @param id
michael@0:    *        The GUID for the BSO.
michael@0:    * @param payload
michael@0:    *        The payload, as provided to the ServerBSO constructor.
michael@0:    * @param modified
michael@0:    *        An optional modified time for the ServerBSO. If not specified, the
michael@0:    *        current time will be used.
michael@0:    *
michael@0:    * @return the inserted BSO.
michael@0:    */
michael@0:   insert: function insert(id, payload, modified) {
michael@0:     return this.insertBSO(new ServerBSO(id, payload, modified));
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Removes an object entirely from the collection.
michael@0:    *
michael@0:    * @param id
michael@0:    *        (string) ID to remove.
michael@0:    */
michael@0:   remove: function remove(id) {
michael@0:     delete this._bsos[id];
michael@0:   },
michael@0: 
michael@0:   _inResultSet: function _inResultSet(bso, options) {
michael@0:     if (!bso.payload) {
michael@0:       return false;
michael@0:     }
michael@0: 
michael@0:     if (options.ids) {
michael@0:       if (options.ids.indexOf(bso.id) == -1) {
michael@0:         return false;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (options.newer) {
michael@0:       if (bso.modified <= options.newer) {
michael@0:         return false;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (options.older) {
michael@0:       if (bso.modified >= options.older) {
michael@0:         return false;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return true;
michael@0:   },
michael@0: 
michael@0:   count: function count(options) {
michael@0:     options = options || {};
michael@0:     let c = 0;
michael@0:     for (let [id, bso] in Iterator(this._bsos)) {
michael@0:       if (bso.modified && this._inResultSet(bso, options)) {
michael@0:         c++;
michael@0:       }
michael@0:     }
michael@0:     return c;
michael@0:   },
michael@0: 
michael@0:   get: function get(options) {
michael@0:     let data = [];
michael@0:     for each (let bso in this._bsos) {
michael@0:       if (!bso.modified) {
michael@0:         continue;
michael@0:       }
michael@0: 
michael@0:       if (!this._inResultSet(bso, options)) {
michael@0:         continue;
michael@0:       }
michael@0: 
michael@0:       data.push(bso);
michael@0:     }
michael@0: 
michael@0:     if (options.sort) {
michael@0:       if (options.sort == "oldest") {
michael@0:         data.sort(function sortOldest(a, b) {
michael@0:           if (a.modified == b.modified) {
michael@0:             return 0;
michael@0:           }
michael@0: 
michael@0:           return a.modified < b.modified ? -1 : 1;
michael@0:         });
michael@0:       } else if (options.sort == "newest") {
michael@0:         data.sort(function sortNewest(a, b) {
michael@0:           if (a.modified == b.modified) {
michael@0:             return 0;
michael@0:           }
michael@0: 
michael@0:           return a.modified > b.modified ? -1 : 1;
michael@0:         });
michael@0:       } else if (options.sort == "index") {
michael@0:         data.sort(function sortIndex(a, b) {
michael@0:           if (a.sortindex == b.sortindex) {
michael@0:             return 0;
michael@0:           }
michael@0: 
michael@0:           if (a.sortindex !== undefined && b.sortindex == undefined) {
michael@0:             return 1;
michael@0:           }
michael@0: 
michael@0:           if (a.sortindex === undefined && b.sortindex !== undefined) {
michael@0:             return -1;
michael@0:           }
michael@0: 
michael@0:           return a.sortindex > b.sortindex ? -1 : 1;
michael@0:         });
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (options.limit) {
michael@0:       data = data.slice(0, options.limit);
michael@0:     }
michael@0: 
michael@0:     return data;
michael@0:   },
michael@0: 
michael@0:   post: function post(input, timestamp) {
michael@0:     let success = [];
michael@0:     let failed = {};
michael@0:     let count = 0;
michael@0:     let size = 0;
michael@0: 
michael@0:     // This will count records where we have an existing ServerBSO
michael@0:     // registered with us as successful and all other records as failed.
michael@0:     for each (let record in input) {
michael@0:       count += 1;
michael@0:       if (count > this.BATCH_MAX_COUNT) {
michael@0:         failed[record.id] = "Max record count exceeded.";
michael@0:         continue;
michael@0:       }
michael@0: 
michael@0:       if (typeof(record.payload) != "string") {
michael@0:         failed[record.id] = "Payload is not a string!";
michael@0:         continue;
michael@0:       }
michael@0: 
michael@0:       size += record.payload.length;
michael@0:       if (size > this.BATCH_MAX_SIZE) {
michael@0:         failed[record.id] = "Payload max size exceeded!";
michael@0:         continue;
michael@0:       }
michael@0: 
michael@0:       if (record.sortindex) {
michael@0:         if (!isInteger(record.sortindex)) {
michael@0:           failed[record.id] = "sortindex is not an integer.";
michael@0:           continue;
michael@0:         }
michael@0: 
michael@0:         if (record.sortindex.length > 9) {
michael@0:           failed[record.id] = "sortindex is too long.";
michael@0:           continue;
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       if ("ttl" in record) {
michael@0:         if (!isInteger(record.ttl)) {
michael@0:           failed[record.id] = "ttl is not an integer.";
michael@0:           continue;
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       try {
michael@0:         let bso = this.bso(record.id);
michael@0:         if (!bso && this.acceptNew) {
michael@0:           this._log.debug("Creating BSO " + JSON.stringify(record.id) +
michael@0:                           " on the fly.");
michael@0:           bso = new ServerBSO(record.id);
michael@0:           this.insertBSO(bso);
michael@0:         }
michael@0:         if (bso) {
michael@0:           bso.payload = record.payload;
michael@0:           bso.modified = timestamp;
michael@0:           bso.deleted = false;
michael@0:           success.push(record.id);
michael@0: 
michael@0:           if (record.sortindex) {
michael@0:             bso.sortindex = parseInt(record.sortindex, 10);
michael@0:           }
michael@0: 
michael@0:         } else {
michael@0:           failed[record.id] = "no bso configured";
michael@0:         }
michael@0:       } catch (ex) {
michael@0:         this._log.info("Exception when processing BSO: " +
michael@0:                        CommonUtils.exceptionStr(ex));
michael@0:         failed[record.id] = "Exception when processing.";
michael@0:       }
michael@0:     }
michael@0:     return {success: success, failed: failed};
michael@0:   },
michael@0: 
michael@0:   delete: function delete_(options) {
michael@0:     options = options || {};
michael@0: 
michael@0:     // Protocol 2.0 only allows the "ids" query string argument.
michael@0:     let keys = Object.keys(options).filter(function(k) {
michael@0:       return k != "ids";
michael@0:     });
michael@0:     if (keys.length) {
michael@0:       this._log.warn("Invalid query string parameter to collection delete: " +
michael@0:                      keys.join(", "));
michael@0:       throw new Error("Malformed client request.");
michael@0:     }
michael@0: 
michael@0:     if (options.ids && options.ids.length > this.BATCH_MAX_COUNT) {
michael@0:       throw HTTP_400;
michael@0:     }
michael@0: 
michael@0:     let deleted = [];
michael@0:     for (let [id, bso] in Iterator(this._bsos)) {
michael@0:       if (this._inResultSet(bso, options)) {
michael@0:         this._log.debug("Deleting " + JSON.stringify(bso));
michael@0:         deleted.push(bso.id);
michael@0:         bso.delete();
michael@0:       }
michael@0:     }
michael@0:     return deleted;
michael@0:   },
michael@0: 
michael@0:   parseOptions: function parseOptions(request) {
michael@0:     let options = {};
michael@0: 
michael@0:     for each (let chunk in request.queryString.split("&")) {
michael@0:       if (!chunk) {
michael@0:         continue;
michael@0:       }
michael@0:       chunk = chunk.split("=");
michael@0:       let key = decodeURIComponent(chunk[0]);
michael@0:       if (chunk.length == 1) {
michael@0:         options[key] = "";
michael@0:       } else {
michael@0:         options[key] = decodeURIComponent(chunk[1]);
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (options.ids) {
michael@0:       options.ids = options.ids.split(",");
michael@0:     }
michael@0: 
michael@0:     if (options.newer) {
michael@0:       if (!isInteger(options.newer)) {
michael@0:         throw HTTP_400;
michael@0:       }
michael@0: 
michael@0:       CommonUtils.ensureMillisecondsTimestamp(options.newer);
michael@0:       options.newer = parseInt(options.newer, 10);
michael@0:     }
michael@0: 
michael@0:     if (options.older) {
michael@0:       if (!isInteger(options.older)) {
michael@0:         throw HTTP_400;
michael@0:       }
michael@0: 
michael@0:       CommonUtils.ensureMillisecondsTimestamp(options.older);
michael@0:       options.older = parseInt(options.older, 10);
michael@0:     }
michael@0: 
michael@0:     if (options.limit) {
michael@0:       if (!isInteger(options.limit)) {
michael@0:         throw HTTP_400;
michael@0:       }
michael@0: 
michael@0:       options.limit = parseInt(options.limit, 10);
michael@0:     }
michael@0: 
michael@0:     return options;
michael@0:   },
michael@0: 
michael@0:   getHandler: function getHandler(request, response) {
michael@0:     let options = this.parseOptions(request);
michael@0:     let data = this.get(options);
michael@0: 
michael@0:     if (request.hasHeader("x-if-modified-since")) {
michael@0:       let requestModified = parseInt(request.getHeader("x-if-modified-since"),
michael@0:                                      10);
michael@0:       let newestBSO = 0;
michael@0:       for each (let bso in data) {
michael@0:         if (bso.modified > newestBSO) {
michael@0:           newestBSO = bso.modified;
michael@0:         }
michael@0:       }
michael@0: 
michael@0:       if (requestModified >= newestBSO) {
michael@0:         response.setHeader("X-Last-Modified", "" + newestBSO);
michael@0:         response.setStatusLine(request.httpVersion, 304, "Not Modified");
michael@0:         return;
michael@0:       }
michael@0:     } else if (request.hasHeader("x-if-unmodified-since")) {
michael@0:       let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
michael@0:                                      10);
michael@0:       let serverModified = this.timestamp;
michael@0: 
michael@0:       if (serverModified > requestModified) {
michael@0:         response.setHeader("X-Last-Modified", "" + serverModified);
michael@0:         response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
michael@0:         return;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     if (options.full) {
michael@0:       data = data.map(function map(bso) {
michael@0:         return bso.toJSON();
michael@0:       });
michael@0:     } else {
michael@0:       data = data.map(function map(bso) {
michael@0:         return bso.id;
michael@0:       });
michael@0:     }
michael@0: 
michael@0:     // application/json is default media type.
michael@0:     let newlines = false;
michael@0:     if (request.hasHeader("accept")) {
michael@0:       let accept = request.getHeader("accept");
michael@0:       if (accept == "application/newlines") {
michael@0:         newlines = true;
michael@0:       } else if (accept != "application/json") {
michael@0:         throw HTTP_406;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     let body;
michael@0:     if (newlines) {
michael@0:       response.setHeader("Content-Type", "application/newlines", false);
michael@0:       let normalized = data.map(function map(d) {
michael@0:         return JSON.stringify(d);
michael@0:       });
michael@0: 
michael@0:       body = normalized.join("\n") + "\n";
michael@0:     } else {
michael@0:       response.setHeader("Content-Type", "application/json", false);
michael@0:       body = JSON.stringify({items: data});
michael@0:     }
michael@0: 
michael@0:     this._log.info("Records: " + data.length);
michael@0:     response.setHeader("X-Num-Records", "" + data.length, false);
michael@0:     response.setHeader("X-Last-Modified", "" + this.timestamp, false);
michael@0:     response.setStatusLine(request.httpVersion, 200, "OK");
michael@0:     response.bodyOutputStream.write(body, body.length);
michael@0:   },
michael@0: 
michael@0:   postHandler: function postHandler(request, response) {
michael@0:     let options = this.parseOptions(request);
michael@0: 
michael@0:     if (!request.hasHeader("content-type")) {
michael@0:       this._log.info("No Content-Type request header!");
michael@0:       throw HTTP_400;
michael@0:     }
michael@0: 
michael@0:     let inputStream = request.bodyInputStream;
michael@0:     let inputBody = CommonUtils.readBytesFromInputStream(inputStream);
michael@0:     let input = [];
michael@0: 
michael@0:     let inputMediaType = request.getHeader("content-type");
michael@0:     if (inputMediaType == "application/json") {
michael@0:       try {
michael@0:         input = JSON.parse(inputBody);
michael@0:       } catch (ex) {
michael@0:         this._log.info("JSON parse error on input body!");
michael@0:         throw HTTP_400;
michael@0:       }
michael@0: 
michael@0:       if (!Array.isArray(input)) {
michael@0:         this._log.info("Input JSON type not an array!");
michael@0:         return sendMozSvcError(request, response, "8");
michael@0:       }
michael@0:     } else if (inputMediaType == "application/newlines") {
michael@0:       for each (let line in inputBody.split("\n")) {
michael@0:         let record;
michael@0:         try {
michael@0:           record = JSON.parse(line);
michael@0:         } catch (ex) {
michael@0:           this._log.info("JSON parse error on line!");
michael@0:           return sendMozSvcError(request, response, "8");
michael@0:         }
michael@0: 
michael@0:         input.push(record);
michael@0:       }
michael@0:     } else {
michael@0:       this._log.info("Unknown media type: " + inputMediaType);
michael@0:       throw HTTP_415;
michael@0:     }
michael@0: 
michael@0:     if (this._ensureUnmodifiedSince(request, response)) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let res = this.post(input, request.timestamp);
michael@0:     let body = JSON.stringify(res);
michael@0:     response.setHeader("Content-Type", "application/json", false);
michael@0:     this.timestamp = request.timestamp;
michael@0:     response.setHeader("X-Last-Modified", "" + this.timestamp, false);
michael@0: 
michael@0:     response.setStatusLine(request.httpVersion, "200", "OK");
michael@0:     response.bodyOutputStream.write(body, body.length);
michael@0:   },
michael@0: 
michael@0:   deleteHandler: function deleteHandler(request, response) {
michael@0:     this._log.debug("Invoking StorageServerCollection.DELETE.");
michael@0: 
michael@0:     let options = this.parseOptions(request);
michael@0: 
michael@0:     if (this._ensureUnmodifiedSince(request, response)) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let deleted = this.delete(options);
michael@0:     response.deleted = deleted;
michael@0:     this.timestamp = request.timestamp;
michael@0: 
michael@0:     response.setStatusLine(request.httpVersion, 204, "No Content");
michael@0:   },
michael@0: 
michael@0:   handler: function handler() {
michael@0:     let self = this;
michael@0: 
michael@0:     return function(request, response) {
michael@0:       switch(request.method) {
michael@0:         case "GET":
michael@0:           return self.getHandler(request, response);
michael@0: 
michael@0:         case "POST":
michael@0:           return self.postHandler(request, response);
michael@0: 
michael@0:         case "DELETE":
michael@0:           return self.deleteHandler(request, response);
michael@0: 
michael@0:       }
michael@0: 
michael@0:       request.setHeader("Allow", "GET,POST,DELETE");
michael@0:       response.setStatusLine(request.httpVersion, 405, "Method Not Allowed");
michael@0:     };
michael@0:   },
michael@0: 
michael@0:   _ensureUnmodifiedSince: function _ensureUnmodifiedSince(request, response) {
michael@0:     if (!request.hasHeader("x-if-unmodified-since")) {
michael@0:       return false;
michael@0:     }
michael@0: 
michael@0:     let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
michael@0:                                    10);
michael@0:     let serverModified = this.timestamp;
michael@0: 
michael@0:     this._log.debug("Request modified time: " + requestModified +
michael@0:                     "; Server modified time: " + serverModified);
michael@0:     if (serverModified <= requestModified) {
michael@0:       return false;
michael@0:     }
michael@0: 
michael@0:     this._log.info("Conditional request rejected because client time older " +
michael@0:                    "than collection timestamp.");
michael@0:     response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
michael@0:     return true;
michael@0:   },
michael@0: };
michael@0: 
michael@0: 
michael@0: //===========================================================================//
michael@0: // httpd.js-based Storage server.                                            //
michael@0: //===========================================================================//
michael@0: 
michael@0: /**
michael@0:  * In general, the preferred way of using StorageServer is to directly
michael@0:  * introspect it. Callbacks are available for operations which are hard to
michael@0:  * verify through introspection, such as deletions.
michael@0:  *
michael@0:  * One of the goals of this server is to provide enough hooks for test code to
michael@0:  * find out what it needs without monkeypatching. Use this object as your
michael@0:  * prototype, and override as appropriate.
michael@0:  */
michael@0: this.StorageServerCallback = {
michael@0:   onCollectionDeleted: function onCollectionDeleted(user, collection) {},
michael@0:   onItemDeleted: function onItemDeleted(user, collection, bsoID) {},
michael@0: 
michael@0:   /**
michael@0:    * Called at the top of every request.
michael@0:    *
michael@0:    * Allows the test to inspect the request. Hooks should be careful not to
michael@0:    * modify or change state of the request or they may impact future processing.
michael@0:    */
michael@0:   onRequest: function onRequest(request) {},
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Construct a new test Storage server. Takes a callback object (e.g.,
michael@0:  * StorageServerCallback) as input.
michael@0:  */
michael@0: this.StorageServer = function StorageServer(callback) {
michael@0:   this.callback     = callback || {__proto__: StorageServerCallback};
michael@0:   this.server       = new HttpServer();
michael@0:   this.started      = false;
michael@0:   this.users        = {};
michael@0:   this.requestCount = 0;
michael@0:   this._log         = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
michael@0: 
michael@0:   // Install our own default handler. This allows us to mess around with the
michael@0:   // whole URL space.
michael@0:   let handler = this.server._handler;
michael@0:   handler._handleDefault = this.handleDefault.bind(this, handler);
michael@0: }
michael@0: StorageServer.prototype = {
michael@0:   DEFAULT_QUOTA: 1024 * 1024, // # bytes.
michael@0: 
michael@0:   server: null,    // HttpServer.
michael@0:   users:  null,    // Map of username => {collections, password}.
michael@0: 
michael@0:   /**
michael@0:    * If true, the server will allow any arbitrary user to be used.
michael@0:    *
michael@0:    * No authentication will be performed. Whatever user is detected from the
michael@0:    * URL or auth headers will be created (if needed) and used.
michael@0:    */
michael@0:   allowAllUsers: false,
michael@0: 
michael@0:   /**
michael@0:    * Start the StorageServer's underlying HTTP server.
michael@0:    *
michael@0:    * @param port
michael@0:    *        The numeric port on which to start. A falsy value implies to
michael@0:    *        select any available port.
michael@0:    * @param cb
michael@0:    *        A callback function (of no arguments) which is invoked after
michael@0:    *        startup.
michael@0:    */
michael@0:   start: function start(port, cb) {
michael@0:     if (this.started) {
michael@0:       this._log.warn("Warning: server already started on " + this.port);
michael@0:       return;
michael@0:     }
michael@0:     if (!port) {
michael@0:       port = -1;
michael@0:     }
michael@0:     this.port = port;
michael@0: 
michael@0:     try {
michael@0:       this.server.start(this.port);
michael@0:       this.port = this.server.identity.primaryPort;
michael@0:       this.started = true;
michael@0:       if (cb) {
michael@0:         cb();
michael@0:       }
michael@0:     } catch (ex) {
michael@0:       _("==========================================");
michael@0:       _("Got exception starting Storage HTTP server on port " + this.port);
michael@0:       _("Error: " + CommonUtils.exceptionStr(ex));
michael@0:       _("Is there a process already listening on port " + this.port + "?");
michael@0:       _("==========================================");
michael@0:       do_throw(ex);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Start the server synchronously.
michael@0:    *
michael@0:    * @param port
michael@0:    *        The numeric port on which to start. The default is to choose
michael@0:    *        any available port.
michael@0:    */
michael@0:   startSynchronous: function startSynchronous(port=-1) {
michael@0:     let cb = Async.makeSpinningCallback();
michael@0:     this.start(port, cb);
michael@0:     cb.wait();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Stop the StorageServer's HTTP server.
michael@0:    *
michael@0:    * @param cb
michael@0:    *        A callback function. Invoked after the server has been stopped.
michael@0:    *
michael@0:    */
michael@0:   stop: function stop(cb) {
michael@0:     if (!this.started) {
michael@0:       this._log.warn("StorageServer: Warning: server not running. Can't stop " +
michael@0:                      "me now!");
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     this.server.stop(cb);
michael@0:     this.started = false;
michael@0:   },
michael@0: 
michael@0:   serverTime: function serverTime() {
michael@0:     return new_timestamp();
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Create a new user, complete with an empty set of collections.
michael@0:    *
michael@0:    * @param username
michael@0:    *        The username to use. An Error will be thrown if a user by that name
michael@0:    *        already exists.
michael@0:    * @param password
michael@0:    *        A password string.
michael@0:    *
michael@0:    * @return a user object, as would be returned by server.user(username).
michael@0:    */
michael@0:   registerUser: function registerUser(username, password) {
michael@0:     if (username in this.users) {
michael@0:       throw new Error("User already exists.");
michael@0:     }
michael@0: 
michael@0:     if (!isFinite(parseInt(username))) {
michael@0:       throw new Error("Usernames must be numeric: " + username);
michael@0:     }
michael@0: 
michael@0:     this._log.info("Registering new user with server: " + username);
michael@0:     this.users[username] = {
michael@0:       password: password,
michael@0:       collections: {},
michael@0:       quota: this.DEFAULT_QUOTA,
michael@0:     };
michael@0:     return this.user(username);
michael@0:   },
michael@0: 
michael@0:   userExists: function userExists(username) {
michael@0:     return username in this.users;
michael@0:   },
michael@0: 
michael@0:   getCollection: function getCollection(username, collection) {
michael@0:     return this.users[username].collections[collection];
michael@0:   },
michael@0: 
michael@0:   _insertCollection: function _insertCollection(collections, collection, bsos) {
michael@0:     let coll = new StorageServerCollection(bsos, true);
michael@0:     coll.collectionHandler = coll.handler();
michael@0:     collections[collection] = coll;
michael@0:     return coll;
michael@0:   },
michael@0: 
michael@0:   createCollection: function createCollection(username, collection, bsos) {
michael@0:     if (!(username in this.users)) {
michael@0:       throw new Error("Unknown user.");
michael@0:     }
michael@0:     let collections = this.users[username].collections;
michael@0:     if (collection in collections) {
michael@0:       throw new Error("Collection already exists.");
michael@0:     }
michael@0:     return this._insertCollection(collections, collection, bsos);
michael@0:   },
michael@0: 
michael@0:   deleteCollection: function deleteCollection(username, collection) {
michael@0:     if (!(username in this.users)) {
michael@0:       throw new Error("Unknown user.");
michael@0:     }
michael@0:     delete this.users[username].collections[collection];
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Accept a map like the following:
michael@0:    * {
michael@0:    *   meta: {global: {version: 1, ...}},
michael@0:    *   crypto: {"keys": {}, foo: {bar: 2}},
michael@0:    *   bookmarks: {}
michael@0:    * }
michael@0:    * to cause collections and BSOs to be created.
michael@0:    * If a collection already exists, no error is raised.
michael@0:    * If a BSO already exists, it will be updated to the new contents.
michael@0:    */
michael@0:   createContents: function createContents(username, collections) {
michael@0:     if (!(username in this.users)) {
michael@0:       throw new Error("Unknown user.");
michael@0:     }
michael@0:     let userCollections = this.users[username].collections;
michael@0:     for (let [id, contents] in Iterator(collections)) {
michael@0:       let coll = userCollections[id] ||
michael@0:                  this._insertCollection(userCollections, id);
michael@0:       for (let [bsoID, payload] in Iterator(contents)) {
michael@0:         coll.insert(bsoID, payload);
michael@0:       }
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Insert a BSO in an existing collection.
michael@0:    */
michael@0:   insertBSO: function insertBSO(username, collection, bso) {
michael@0:     if (!(username in this.users)) {
michael@0:       throw new Error("Unknown user.");
michael@0:     }
michael@0:     let userCollections = this.users[username].collections;
michael@0:     if (!(collection in userCollections)) {
michael@0:       throw new Error("Unknown collection.");
michael@0:     }
michael@0:     userCollections[collection].insertBSO(bso);
michael@0:     return bso;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Delete all of the collections for the named user.
michael@0:    *
michael@0:    * @param username
michael@0:    *        The name of the affected user.
michael@0:    */
michael@0:   deleteCollections: function deleteCollections(username) {
michael@0:     if (!(username in this.users)) {
michael@0:       throw new Error("Unknown user.");
michael@0:     }
michael@0:     let userCollections = this.users[username].collections;
michael@0:     for each (let [name, coll] in Iterator(userCollections)) {
michael@0:       this._log.trace("Bulk deleting " + name + " for " + username + "...");
michael@0:       coll.delete({});
michael@0:     }
michael@0:     this.users[username].collections = {};
michael@0:   },
michael@0: 
michael@0:   getQuota: function getQuota(username) {
michael@0:     if (!(username in this.users)) {
michael@0:       throw new Error("Unknown user.");
michael@0:     }
michael@0: 
michael@0:     return this.users[username].quota;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Obtain the newest timestamp of all collections for a user.
michael@0:    */
michael@0:   newestCollectionTimestamp: function newestCollectionTimestamp(username) {
michael@0:     let collections = this.users[username].collections;
michael@0:     let newest = 0;
michael@0:     for each (let collection in collections) {
michael@0:       if (collection.timestamp > newest) {
michael@0:         newest = collection.timestamp;
michael@0:       }
michael@0:     }
michael@0: 
michael@0:     return newest;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Compute the object that is returned for an info/collections request.
michael@0:    */
michael@0:   infoCollections: function infoCollections(username) {
michael@0:     let responseObject = {};
michael@0:     let colls = this.users[username].collections;
michael@0:     for (let coll in colls) {
michael@0:       responseObject[coll] = colls[coll].timestamp;
michael@0:     }
michael@0:     this._log.trace("StorageServer: info/collections returning " +
michael@0:                     JSON.stringify(responseObject));
michael@0:     return responseObject;
michael@0:   },
michael@0: 
michael@0:   infoCounts: function infoCounts(username) {
michael@0:     let data = {};
michael@0:     let collections = this.users[username].collections;
michael@0:     for (let [k, v] in Iterator(collections)) {
michael@0:       let count = v.count();
michael@0:       if (!count) {
michael@0:         continue;
michael@0:       }
michael@0: 
michael@0:       data[k] = count;
michael@0:     }
michael@0: 
michael@0:     return data;
michael@0:   },
michael@0: 
michael@0:   infoUsage: function infoUsage(username) {
michael@0:     let data = {};
michael@0:     let collections = this.users[username].collections;
michael@0:     for (let [k, v] in Iterator(collections)) {
michael@0:       data[k] = v.totalPayloadSize;
michael@0:     }
michael@0: 
michael@0:     return data;
michael@0:   },
michael@0: 
michael@0:   infoQuota: function infoQuota(username) {
michael@0:     let total = 0;
michael@0:     for each (let value in this.infoUsage(username)) {
michael@0:       total += value;
michael@0:     }
michael@0: 
michael@0:     return {
michael@0:       quota: this.getQuota(username),
michael@0:       usage: total
michael@0:     };
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Simple accessor to allow collective binding and abbreviation of a bunch of
michael@0:    * methods. Yay!
michael@0:    * Use like this:
michael@0:    *
michael@0:    *   let u = server.user("john");
michael@0:    *   u.collection("bookmarks").bso("abcdefg").payload;  // Etc.
michael@0:    *
michael@0:    * @return a proxy for the user data stored in this server.
michael@0:    */
michael@0:   user: function user(username) {
michael@0:     let collection       = this.getCollection.bind(this, username);
michael@0:     let createCollection = this.createCollection.bind(this, username);
michael@0:     let createContents   = this.createContents.bind(this, username);
michael@0:     let modified         = function (collectionName) {
michael@0:       return collection(collectionName).timestamp;
michael@0:     }
michael@0:     let deleteCollections = this.deleteCollections.bind(this, username);
michael@0:     let quota             = this.getQuota.bind(this, username);
michael@0:     return {
michael@0:       collection:        collection,
michael@0:       createCollection:  createCollection,
michael@0:       createContents:    createContents,
michael@0:       deleteCollections: deleteCollections,
michael@0:       modified:          modified,
michael@0:       quota:             quota,
michael@0:     };
michael@0:   },
michael@0: 
michael@0:   _pruneExpired: function _pruneExpired() {
michael@0:     let now = Date.now();
michael@0: 
michael@0:     for each (let user in this.users) {
michael@0:       for each (let collection in user.collections) {
michael@0:         for each (let bso in collection.bsos()) {
michael@0:           // ttl === 0 is a special case, so we can't simply !ttl.
michael@0:           if (typeof(bso.ttl) != "number") {
michael@0:             continue;
michael@0:           }
michael@0: 
michael@0:           let ttlDate = bso.modified + (bso.ttl * 1000);
michael@0:           if (ttlDate < now) {
michael@0:             this._log.info("Deleting BSO because TTL expired: " + bso.id);
michael@0:             bso.delete();
michael@0:           }
michael@0:         }
michael@0:       }
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /*
michael@0:    * Regular expressions for splitting up Storage request paths.
michael@0:    * Storage URLs are of the form:
michael@0:    *   /$apipath/$version/$userid/$further
michael@0:    * where $further is usually:
michael@0:    *   storage/$collection/$bso
michael@0:    * or
michael@0:    *   storage/$collection
michael@0:    * or
michael@0:    *   info/$op
michael@0:    *
michael@0:    * We assume for the sake of simplicity that $apipath is empty.
michael@0:    *
michael@0:    * N.B., we don't follow any kind of username spec here, because as far as I
michael@0:    * can tell there isn't one. See Bug 689671. Instead we follow the Python
michael@0:    * server code.
michael@0:    *
michael@0:    * Path: [all, version, first, rest]
michael@0:    * Storage: [all, collection?, id?]
michael@0:    */
michael@0:   pathRE: /^\/([0-9]+(?:\.[0-9]+)?)(?:\/([0-9]+)\/([^\/]+)(?:\/(.+))?)?$/,
michael@0:   storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/,
michael@0: 
michael@0:   defaultHeaders: {},
michael@0: 
michael@0:   /**
michael@0:    * HTTP response utility.
michael@0:    */
michael@0:   respond: function respond(req, resp, code, status, body, headers, timestamp) {
michael@0:     this._log.info("Response: " + code + " " + status);
michael@0:     resp.setStatusLine(req.httpVersion, code, status);
michael@0:     for each (let [header, value] in Iterator(headers || this.defaultHeaders)) {
michael@0:       resp.setHeader(header, value, false);
michael@0:     }
michael@0: 
michael@0:     if (timestamp) {
michael@0:       resp.setHeader("X-Timestamp", "" + timestamp, false);
michael@0:     }
michael@0: 
michael@0:     if (body) {
michael@0:       resp.bodyOutputStream.write(body, body.length);
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * This is invoked by the HttpServer. `this` is bound to the StorageServer;
michael@0:    * `handler` is the HttpServer's handler.
michael@0:    *
michael@0:    * TODO: need to use the correct Storage API response codes and errors here.
michael@0:    */
michael@0:   handleDefault: function handleDefault(handler, req, resp) {
michael@0:     this.requestCount++;
michael@0:     let timestamp = new_timestamp();
michael@0:     try {
michael@0:       this._handleDefault(handler, req, resp, timestamp);
michael@0:     } catch (e) {
michael@0:       if (e instanceof HttpError) {
michael@0:         this.respond(req, resp, e.code, e.description, "", {}, timestamp);
michael@0:       } else {
michael@0:         this._log.warn(CommonUtils.exceptionStr(e));
michael@0:         throw e;
michael@0:       }
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   _handleDefault: function _handleDefault(handler, req, resp, timestamp) {
michael@0:     let path = req.path;
michael@0:     if (req.queryString.length) {
michael@0:       path += "?" + req.queryString;
michael@0:     }
michael@0: 
michael@0:     this._log.debug("StorageServer: Handling request: " + req.method + " " +
michael@0:                     path);
michael@0: 
michael@0:     if (this.callback.onRequest) {
michael@0:       this.callback.onRequest(req);
michael@0:     }
michael@0: 
michael@0:     // Prune expired records for all users at top of request. This is the
michael@0:     // easiest way to process TTLs since all requests go through here.
michael@0:     this._pruneExpired();
michael@0: 
michael@0:     req.timestamp = timestamp;
michael@0:     resp.setHeader("X-Timestamp", "" + timestamp, false);
michael@0: 
michael@0:     let parts = this.pathRE.exec(req.path);
michael@0:     if (!parts) {
michael@0:       this._log.debug("StorageServer: Unexpected request: bad URL " + req.path);
michael@0:       throw HTTP_404;
michael@0:     }
michael@0: 
michael@0:     let [all, version, userPath, first, rest] = parts;
michael@0:     if (version != STORAGE_API_VERSION) {
michael@0:       this._log.debug("StorageServer: Unknown version.");
michael@0:       throw HTTP_404;
michael@0:     }
michael@0: 
michael@0:     let username;
michael@0: 
michael@0:     // By default, the server requires users to be authenticated. When a
michael@0:     // request arrives, the user must have been previously configured and
michael@0:     // the request must have authentication. In "allow all users" mode, we
michael@0:     // take the username from the URL, create the user on the fly, and don't
michael@0:     // perform any authentication.
michael@0:     if (!this.allowAllUsers) {
michael@0:       // Enforce authentication.
michael@0:       if (!req.hasHeader("authorization")) {
michael@0:         this.respond(req, resp, 401, "Authorization Required", "{}", {
michael@0:           "WWW-Authenticate": 'Basic realm="secret"'
michael@0:         });
michael@0:         return;
michael@0:       }
michael@0: 
michael@0:       let ensureUserExists = function ensureUserExists(username) {
michael@0:         if (this.userExists(username)) {
michael@0:           return;
michael@0:         }
michael@0: 
michael@0:         this._log.info("StorageServer: Unknown user: " + username);
michael@0:         throw HTTP_401;
michael@0:       }.bind(this);
michael@0: 
michael@0:       let auth = req.getHeader("authorization");
michael@0:       this._log.debug("Authorization: " + auth);
michael@0: 
michael@0:       if (auth.indexOf("Basic ") == 0) {
michael@0:         let decoded = CommonUtils.safeAtoB(auth.substr(6));
michael@0:         this._log.debug("Decoded Basic Auth: " + decoded);
michael@0:         let [user, password] = decoded.split(":", 2);
michael@0: 
michael@0:         if (!password) {
michael@0:           this._log.debug("Malformed HTTP Basic Authorization header: " + auth);
michael@0:           throw HTTP_400;
michael@0:         }
michael@0: 
michael@0:         this._log.debug("Got HTTP Basic auth for user: " + user);
michael@0:         ensureUserExists(user);
michael@0:         username = user;
michael@0: 
michael@0:         if (this.users[user].password != password) {
michael@0:           this._log.debug("StorageServer: Provided password is not correct.");
michael@0:           throw HTTP_401;
michael@0:         }
michael@0:       // TODO support token auth.
michael@0:       } else {
michael@0:         this._log.debug("Unsupported HTTP authorization type: " + auth);
michael@0:         throw HTTP_500;
michael@0:       }
michael@0:     // All users mode.
michael@0:     } else {
michael@0:       // Auto create user with dummy password.
michael@0:       if (!this.userExists(userPath)) {
michael@0:         this.registerUser(userPath, "DUMMY-PASSWORD-*&%#");
michael@0:       }
michael@0: 
michael@0:       username = userPath;
michael@0:     }
michael@0: 
michael@0:     // Hand off to the appropriate handler for this path component.
michael@0:     if (first in this.toplevelHandlers) {
michael@0:       let handler = this.toplevelHandlers[first];
michael@0:       try {
michael@0:         return handler.call(this, handler, req, resp, version, username, rest);
michael@0:       } catch (ex) {
michael@0:         this._log.warn("Got exception during request: " +
michael@0:                        CommonUtils.exceptionStr(ex));
michael@0:         throw ex;
michael@0:       }
michael@0:     }
michael@0:     this._log.debug("StorageServer: Unknown top-level " + first);
michael@0:     throw HTTP_404;
michael@0:   },
michael@0: 
michael@0:   /**
michael@0:    * Collection of the handler methods we use for top-level path components.
michael@0:    */
michael@0:   toplevelHandlers: {
michael@0:     "storage": function handleStorage(handler, req, resp, version, username,
michael@0:                                       rest) {
michael@0:       let respond = this.respond.bind(this, req, resp);
michael@0:       if (!rest || !rest.length) {
michael@0:         this._log.debug("StorageServer: top-level storage " +
michael@0:                         req.method + " request.");
michael@0: 
michael@0:         if (req.method != "DELETE") {
michael@0:           respond(405, "Method Not Allowed", null, {"Allow": "DELETE"});
michael@0:           return;
michael@0:         }
michael@0: 
michael@0:         this.user(username).deleteCollections();
michael@0: 
michael@0:         respond(204, "No Content");
michael@0:         return;
michael@0:       }
michael@0: 
michael@0:       let match = this.storageRE.exec(rest);
michael@0:       if (!match) {
michael@0:         this._log.warn("StorageServer: Unknown storage operation " + rest);
michael@0:         throw HTTP_404;
michael@0:       }
michael@0:       let [all, collection, bsoID] = match;
michael@0:       let coll = this.getCollection(username, collection);
michael@0:       let collectionExisted = !!coll;
michael@0: 
michael@0:       switch (req.method) {
michael@0:         case "GET":
michael@0:           // Tried to GET on a collection that doesn't exist.
michael@0:           if (!coll) {
michael@0:             respond(404, "Not Found");
michael@0:             return;
michael@0:           }
michael@0: 
michael@0:           // No BSO URL parameter goes to collection handler.
michael@0:           if (!bsoID) {
michael@0:             return coll.collectionHandler(req, resp);
michael@0:           }
michael@0: 
michael@0:           // Handle non-existent BSO.
michael@0:           let bso = coll.bso(bsoID);
michael@0:           if (!bso) {
michael@0:             respond(404, "Not Found");
michael@0:             return;
michael@0:           }
michael@0: 
michael@0:           // Proxy to BSO handler.
michael@0:           return bso.getHandler(req, resp);
michael@0: 
michael@0:         case "DELETE":
michael@0:           // Collection doesn't exist.
michael@0:           if (!coll) {
michael@0:             respond(404, "Not Found");
michael@0:             return;
michael@0:           }
michael@0: 
michael@0:           // Deleting a specific BSO.
michael@0:           if (bsoID) {
michael@0:             let bso = coll.bso(bsoID);
michael@0: 
michael@0:             // BSO does not exist on the server. Nothing to do.
michael@0:             if (!bso) {
michael@0:               respond(404, "Not Found");
michael@0:               return;
michael@0:             }
michael@0: 
michael@0:             if (req.hasHeader("x-if-unmodified-since")) {
michael@0:               let modified = parseInt(req.getHeader("x-if-unmodified-since"));
michael@0:               CommonUtils.ensureMillisecondsTimestamp(modified);
michael@0: 
michael@0:               if (bso.modified > modified) {
michael@0:                 respond(412, "Precondition Failed");
michael@0:                 return;
michael@0:               }
michael@0:             }
michael@0: 
michael@0:             bso.delete();
michael@0:             coll.timestamp = req.timestamp;
michael@0:             this.callback.onItemDeleted(username, collection, bsoID);
michael@0:             respond(204, "No Content");
michael@0:             return;
michael@0:           }
michael@0: 
michael@0:           // Proxy to collection handler.
michael@0:           coll.collectionHandler(req, resp);
michael@0: 
michael@0:           // Spot if this is a DELETE for some IDs, and don't blow away the
michael@0:           // whole collection!
michael@0:           //
michael@0:           // We already handled deleting the BSOs by invoking the deleted
michael@0:           // collection's handler. However, in the case of
michael@0:           //
michael@0:           //   DELETE storage/foobar
michael@0:           //
michael@0:           // we also need to remove foobar from the collections map. This
michael@0:           // clause tries to differentiate the above request from
michael@0:           //
michael@0:           //  DELETE storage/foobar?ids=foo,baz
michael@0:           //
michael@0:           // and do the right thing.
michael@0:           // TODO: less hacky method.
michael@0:           if (-1 == req.queryString.indexOf("ids=")) {
michael@0:             // When you delete the entire collection, we drop it.
michael@0:             this._log.debug("Deleting entire collection.");
michael@0:             delete this.users[username].collections[collection];
michael@0:             this.callback.onCollectionDeleted(username, collection);
michael@0:           }
michael@0: 
michael@0:           // Notify of item deletion.
michael@0:           let deleted = resp.deleted || [];
michael@0:           for (let i = 0; i < deleted.length; ++i) {
michael@0:             this.callback.onItemDeleted(username, collection, deleted[i]);
michael@0:           }
michael@0:           return;
michael@0: 
michael@0:         case "POST":
michael@0:         case "PUT":
michael@0:           // Auto-create collection if it doesn't exist.
michael@0:           if (!coll) {
michael@0:             coll = this.createCollection(username, collection);
michael@0:           }
michael@0: 
michael@0:           try {
michael@0:             if (bsoID) {
michael@0:               let bso = coll.bso(bsoID);
michael@0:               if (!bso) {
michael@0:                 this._log.trace("StorageServer: creating BSO " + collection +
michael@0:                                 "/" + bsoID);
michael@0:                 try {
michael@0:                   bso = coll.insert(bsoID);
michael@0:                 } catch (ex) {
michael@0:                   return sendMozSvcError(req, resp, "8");
michael@0:                 }
michael@0:               }
michael@0: 
michael@0:               bso.putHandler(req, resp);
michael@0: 
michael@0:               coll.timestamp = req.timestamp;
michael@0:               return resp;
michael@0:             }
michael@0: 
michael@0:             return coll.collectionHandler(req, resp);
michael@0:           } catch (ex) {
michael@0:             if (ex instanceof HttpError) {
michael@0:               if (!collectionExisted) {
michael@0:                 this.deleteCollection(username, collection);
michael@0:               }
michael@0:             }
michael@0: 
michael@0:             throw ex;
michael@0:           }
michael@0: 
michael@0:         default:
michael@0:           throw new Error("Request method " + req.method + " not implemented.");
michael@0:       }
michael@0:     },
michael@0: 
michael@0:     "info": function handleInfo(handler, req, resp, version, username, rest) {
michael@0:       switch (rest) {
michael@0:         case "collections":
michael@0:           return this.handleInfoCollections(req, resp, username);
michael@0: 
michael@0:         case "collection_counts":
michael@0:           return this.handleInfoCounts(req, resp, username);
michael@0: 
michael@0:         case "collection_usage":
michael@0:           return this.handleInfoUsage(req, resp, username);
michael@0: 
michael@0:         case "quota":
michael@0:           return this.handleInfoQuota(req, resp, username);
michael@0: 
michael@0:         default:
michael@0:           this._log.warn("StorageServer: Unknown info operation " + rest);
michael@0:           throw HTTP_404;
michael@0:       }
michael@0:     }
michael@0:   },
michael@0: 
michael@0:   handleInfoConditional: function handleInfoConditional(request, response,
michael@0:                                                         user) {
michael@0:     if (!request.hasHeader("x-if-modified-since")) {
michael@0:       return false;
michael@0:     }
michael@0: 
michael@0:     let requestModified = request.getHeader("x-if-modified-since");
michael@0:     requestModified = parseInt(requestModified, 10);
michael@0: 
michael@0:     let serverModified = this.newestCollectionTimestamp(user);
michael@0: 
michael@0:     this._log.info("Server mtime: " + serverModified + "; Client modified: " +
michael@0:                    requestModified);
michael@0:     if (serverModified > requestModified) {
michael@0:       return false;
michael@0:     }
michael@0: 
michael@0:     this.respond(request, response, 304, "Not Modified", null, {
michael@0:       "X-Last-Modified": "" + serverModified
michael@0:     });
michael@0: 
michael@0:     return true;
michael@0:   },
michael@0: 
michael@0:   handleInfoCollections: function handleInfoCollections(request, response,
michael@0:                                                         user) {
michael@0:     if (this.handleInfoConditional(request, response, user)) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let info = this.infoCollections(user);
michael@0:     let body = JSON.stringify(info);
michael@0:     this.respond(request, response, 200, "OK", body, {
michael@0:       "Content-Type":    "application/json",
michael@0:       "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   handleInfoCounts: function handleInfoCounts(request, response, user) {
michael@0:     if (this.handleInfoConditional(request, response, user)) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let counts = this.infoCounts(user);
michael@0:     let body = JSON.stringify(counts);
michael@0: 
michael@0:     this.respond(request, response, 200, "OK", body, {
michael@0:       "Content-Type":    "application/json",
michael@0:       "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   handleInfoUsage: function handleInfoUsage(request, response, user) {
michael@0:     if (this.handleInfoConditional(request, response, user)) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let body = JSON.stringify(this.infoUsage(user));
michael@0:     this.respond(request, response, 200, "OK", body, {
michael@0:       "Content-Type":    "application/json",
michael@0:       "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
michael@0:     });
michael@0:   },
michael@0: 
michael@0:   handleInfoQuota: function handleInfoQuota(request, response, user) {
michael@0:     if (this.handleInfoConditional(request, response, user)) {
michael@0:       return;
michael@0:     }
michael@0: 
michael@0:     let body = JSON.stringify(this.infoQuota(user));
michael@0:     this.respond(request, response, 200, "OK", body, {
michael@0:       "Content-Type":    "application/json",
michael@0:       "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
michael@0:     });
michael@0:   },
michael@0: };
michael@0: 
michael@0: /**
michael@0:  * Helper to create a storage server for a set of users.
michael@0:  *
michael@0:  * Each user is specified by a map of username to password.
michael@0:  */
michael@0: this.storageServerForUsers =
michael@0:  function storageServerForUsers(users, contents, callback) {
michael@0:   let server = new StorageServer(callback);
michael@0:   for (let [user, pass] in Iterator(users)) {
michael@0:     server.registerUser(user, pass);
michael@0:     server.createContents(user, contents);
michael@0:   }
michael@0:   server.start();
michael@0:   return server;
michael@0: }