michael@0: /* Any copyright is dedicated to the Public Domain.
michael@0:    http://creativecommons.org/publicdomain/zero/1.0/ */
michael@0: 
michael@0: Cu.import("resource://services-sync/engines/clients.js");
michael@0: Cu.import("resource://services-sync/constants.js");
michael@0: Cu.import("resource://services-sync/engines.js");
michael@0: Cu.import("resource://services-sync/keys.js");
michael@0: Cu.import("resource://services-sync/policies.js");
michael@0: Cu.import("resource://services-sync/service.js");
michael@0: Cu.import("resource://services-sync/status.js");
michael@0: Cu.import("resource://services-sync/util.js");
michael@0: Cu.import("resource://testing-common/services/sync/utils.js");
michael@0: Cu.import("resource://gre/modules/FileUtils.jsm");
michael@0: 
michael@0: const FAKE_SERVER_URL = "http://dummy:9000/";
michael@0: const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true);
michael@0: const LOG_PREFIX_SUCCESS = "success-";
michael@0: const LOG_PREFIX_ERROR   = "error-";
michael@0: 
michael@0: const PROLONGED_ERROR_DURATION =
michael@0:   (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') * 2) * 1000;
michael@0: 
michael@0: const NON_PROLONGED_ERROR_DURATION =
michael@0:   (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') / 2) * 1000;
michael@0: 
michael@0: Service.engineManager.clear();
michael@0: 
michael@0: function setLastSync(lastSyncValue) {
michael@0:   Svc.Prefs.set("lastSync", (new Date(Date.now() - lastSyncValue)).toString());
michael@0: }
michael@0: 
michael@0: function CatapultEngine() {
michael@0:   SyncEngine.call(this, "Catapult", Service);
michael@0: }
michael@0: CatapultEngine.prototype = {
michael@0:   __proto__: SyncEngine.prototype,
michael@0:   exception: null, // tests fill this in
michael@0:   _sync: function _sync() {
michael@0:     if (this.exception) {
michael@0:       throw this.exception;
michael@0:     }
michael@0:   }
michael@0: };
michael@0: 
michael@0: let engineManager = Service.engineManager;
michael@0: engineManager.register(CatapultEngine);
michael@0: 
michael@0: // This relies on Service/ErrorHandler being a singleton. Fixing this will take
michael@0: // a lot of work.
michael@0: let errorHandler = Service.errorHandler;
michael@0: 
michael@0: function run_test() {
michael@0:   initTestLogging("Trace");
michael@0: 
michael@0:   Log.repository.getLogger("Sync.Service").level = Log.Level.Trace;
michael@0:   Log.repository.getLogger("Sync.SyncScheduler").level = Log.Level.Trace;
michael@0:   Log.repository.getLogger("Sync.ErrorHandler").level = Log.Level.Trace;
michael@0: 
michael@0:   ensureLegacyIdentityManager();
michael@0: 
michael@0:   run_next_test();
michael@0: }
michael@0: 
michael@0: function generateCredentialsChangedFailure() {
michael@0:   // Make sync fail due to changed credentials. We simply re-encrypt
michael@0:   // the keys with a different Sync Key, without changing the local one.
michael@0:   let newSyncKeyBundle = new SyncKeyBundle("johndoe", "23456234562345623456234562");
michael@0:   let keys = Service.collectionKeys.asWBO();
michael@0:   keys.encrypt(newSyncKeyBundle);
michael@0:   keys.upload(Service.resource(Service.cryptoKeysURL));
michael@0: }
michael@0: 
michael@0: function service_unavailable(request, response) {
michael@0:   let body = "Service Unavailable";
michael@0:   response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
michael@0:   response.setHeader("Retry-After", "42");
michael@0:   response.bodyOutputStream.write(body, body.length);
michael@0: }
michael@0: 
michael@0: function sync_httpd_setup() {
michael@0:   let global = new ServerWBO("global", {
michael@0:     syncID: Service.syncID,
michael@0:     storageVersion: STORAGE_VERSION,
michael@0:     engines: {clients: {version: Service.clientsEngine.version,
michael@0:                         syncID: Service.clientsEngine.syncID},
michael@0:               catapult: {version: engineManager.get("catapult").version,
michael@0:                          syncID: engineManager.get("catapult").syncID}}
michael@0:   });
michael@0:   let clientsColl = new ServerCollection({}, true);
michael@0: 
michael@0:   // Tracking info/collections.
michael@0:   let collectionsHelper = track_collections_helper();
michael@0:   let upd = collectionsHelper.with_updated_collection;
michael@0: 
michael@0:   let handler_401 = httpd_handler(401, "Unauthorized");
michael@0:   return httpd_setup({
michael@0:     // Normal server behaviour.
michael@0:     "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
michael@0:     "/1.1/johndoe/info/collections": collectionsHelper.handler,
michael@0:     "/1.1/johndoe/storage/crypto/keys":
michael@0:       upd("crypto", (new ServerWBO("keys")).handler()),
michael@0:     "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
michael@0: 
michael@0:     // Credentials are wrong or node reallocated.
michael@0:     "/1.1/janedoe/storage/meta/global": handler_401,
michael@0:     "/1.1/janedoe/info/collections": handler_401,
michael@0: 
michael@0:     // Maintenance or overloaded (503 + Retry-After) at info/collections.
michael@0:     "/maintenance/1.1/broken.info/info/collections": service_unavailable,
michael@0: 
michael@0:     // Maintenance or overloaded (503 + Retry-After) at meta/global.
michael@0:     "/maintenance/1.1/broken.meta/storage/meta/global": service_unavailable,
michael@0:     "/maintenance/1.1/broken.meta/info/collections": collectionsHelper.handler,
michael@0: 
michael@0:     // Maintenance or overloaded (503 + Retry-After) at crypto/keys.
michael@0:     "/maintenance/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()),
michael@0:     "/maintenance/1.1/broken.keys/info/collections": collectionsHelper.handler,
michael@0:     "/maintenance/1.1/broken.keys/storage/crypto/keys": service_unavailable,
michael@0: 
michael@0:     // Maintenance or overloaded (503 + Retry-After) at wiping collection.
michael@0:     "/maintenance/1.1/broken.wipe/info/collections": collectionsHelper.handler,
michael@0:     "/maintenance/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()),
michael@0:     "/maintenance/1.1/broken.wipe/storage/crypto/keys":
michael@0:       upd("crypto", (new ServerWBO("keys")).handler()),
michael@0:     "/maintenance/1.1/broken.wipe/storage": service_unavailable,
michael@0:     "/maintenance/1.1/broken.wipe/storage/clients": upd("clients", clientsColl.handler()),
michael@0:     "/maintenance/1.1/broken.wipe/storage/catapult": service_unavailable
michael@0:   });
michael@0: }
michael@0: 
michael@0: function setUp(server) {
michael@0:   return configureIdentity({username: "johndoe"}).then(
michael@0:     () => {
michael@0:       Service.serverURL  = server.baseURI + "/";
michael@0:       Service.clusterURL = server.baseURI + "/";
michael@0:     }
michael@0:   ).then(
michael@0:     () => generateAndUploadKeys()
michael@0:   );
michael@0: }
michael@0: 
michael@0: function generateAndUploadKeys() {
michael@0:   generateNewKeys(Service.collectionKeys);
michael@0:   let serverKeys = Service.collectionKeys.asWBO("crypto", "keys");
michael@0:   serverKeys.encrypt(Service.identity.syncKeyBundle);
michael@0:   return serverKeys.upload(Service.resource(Service.cryptoKeysURL)).success;
michael@0: }
michael@0: 
michael@0: function clean() {
michael@0:   Service.startOver();
michael@0:   Status.resetSync();
michael@0:   Status.resetBackoff();
michael@0:   errorHandler.didReportProlongedError = false;
michael@0: }
michael@0: 
michael@0: add_identity_test(this, function test_401_logout() {
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   // By calling sync, we ensure we're logged in.
michael@0:   Service.sync();
michael@0:   do_check_eq(Status.sync, SYNC_SUCCEEDED);
michael@0:   do_check_true(Service.isLoggedIn);
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:service:sync:error", onSyncError);
michael@0:   function onSyncError() {
michael@0:     _("Got weave:service:sync:error in first sync.");
michael@0:     Svc.Obs.remove("weave:service:sync:error", onSyncError);
michael@0: 
michael@0:     // Wait for the automatic next sync.
michael@0:     function onLoginError() {
michael@0:       _("Got weave:service:login:error in second sync.");
michael@0:       Svc.Obs.remove("weave:service:login:error", onLoginError);
michael@0: 
michael@0:       do_check_eq(Status.login, LOGIN_FAILED_LOGIN_REJECTED);
michael@0:       do_check_false(Service.isLoggedIn);
michael@0: 
michael@0:       // Clean up.
michael@0:       Utils.nextTick(function () {
michael@0:         Service.startOver();
michael@0:         server.stop(deferred.resolve);
michael@0:       });
michael@0:     }
michael@0:     Svc.Obs.add("weave:service:login:error", onLoginError);
michael@0:   }
michael@0: 
michael@0:   // Make sync fail due to login rejected.
michael@0:   yield configureIdentity({username: "janedoe"});
michael@0:   Service._updateCachedURLs();
michael@0: 
michael@0:   _("Starting first sync.");
michael@0:   Service.sync();
michael@0:   _("First sync done.");
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_credentials_changed_logout() {
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   // By calling sync, we ensure we're logged in.
michael@0:   Service.sync();
michael@0:   do_check_eq(Status.sync, SYNC_SUCCEEDED);
michael@0:   do_check_true(Service.isLoggedIn);
michael@0: 
michael@0:   generateCredentialsChangedFailure();
michael@0:   Service.sync();
michael@0: 
michael@0:   do_check_eq(Status.sync, CREDENTIALS_CHANGED);
michael@0:   do_check_false(Service.isLoggedIn);
michael@0: 
michael@0:   // Clean up.
michael@0:   Service.startOver();
michael@0:   let deferred = Promise.defer();
michael@0:   server.stop(deferred.resolve);
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_no_lastSync_pref() {
michael@0:   // Test reported error.
michael@0:   Status.resetSync();
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.sync = CREDENTIALS_CHANGED;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test unreported error.
michael@0:   Status.resetSync();
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.login = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_shouldReportError() {
michael@0:   Status.login = MASTER_PASSWORD_LOCKED;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Give ourselves a clusterURL so that the temporary 401 no-error situation
michael@0:   // doesn't come into play.
michael@0:   Service.serverURL  = FAKE_SERVER_URL;
michael@0:   Service.clusterURL = FAKE_SERVER_URL;
michael@0: 
michael@0:   // Test dontIgnoreErrors, non-network, non-prolonged, login error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.login = LOGIN_FAILED_NO_PASSWORD;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test dontIgnoreErrors, non-network, non-prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.sync = CREDENTIALS_CHANGED;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test dontIgnoreErrors, non-network, prolonged, login error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.login = LOGIN_FAILED_NO_PASSWORD;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test dontIgnoreErrors, non-network, prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.sync = CREDENTIALS_CHANGED;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test dontIgnoreErrors, network, non-prolonged, login error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.login = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test dontIgnoreErrors, network, non-prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.sync = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test dontIgnoreErrors, network, prolonged, login error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.login = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test dontIgnoreErrors, network, prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.sync = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Test non-network, prolonged, login error reported
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.login = LOGIN_FAILED_NO_PASSWORD;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Second time with prolonged error and without resetting
michael@0:   // didReportProlongedError, sync error should not be reported.
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.login = LOGIN_FAILED_NO_PASSWORD;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0:   do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test non-network, prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   errorHandler.didReportProlongedError = false;
michael@0:   Status.sync = CREDENTIALS_CHANGED;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_true(errorHandler.didReportProlongedError);
michael@0:   errorHandler.didReportProlongedError = false;
michael@0: 
michael@0:   // Test network, prolonged, login error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.login = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_true(errorHandler.didReportProlongedError);
michael@0:   errorHandler.didReportProlongedError = false;
michael@0: 
michael@0:   // Test network, prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.sync = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_true(errorHandler.didReportProlongedError);
michael@0:   errorHandler.didReportProlongedError = false;
michael@0: 
michael@0:   // Test non-network, non-prolonged, login error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.login = LOGIN_FAILED_NO_PASSWORD;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test non-network, non-prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.sync = CREDENTIALS_CHANGED;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test network, non-prolonged, login error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.login = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test network, non-prolonged, sync error reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.sync = LOGIN_FAILED_NETWORK_ERROR;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test server maintenance, sync errors are not reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.sync = SERVER_MAINTENANCE;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test server maintenance, login errors are not reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.login = SERVER_MAINTENANCE;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test prolonged, server maintenance, sync errors are reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.sync = SERVER_MAINTENANCE;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_true(errorHandler.didReportProlongedError);
michael@0:   errorHandler.didReportProlongedError = false;
michael@0: 
michael@0:   // Test prolonged, server maintenance, login errors are reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = false;
michael@0:   Status.login = SERVER_MAINTENANCE;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_true(errorHandler.didReportProlongedError);
michael@0:   errorHandler.didReportProlongedError = false;
michael@0: 
michael@0:   // Test dontIgnoreErrors, server maintenance, sync errors are reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.sync = SERVER_MAINTENANCE;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   // dontIgnoreErrors means we don't set didReportProlongedError
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test dontIgnoreErrors, server maintenance, login errors are reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.login = SERVER_MAINTENANCE;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test dontIgnoreErrors, prolonged, server maintenance,
michael@0:   // sync errors are reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.sync = SERVER_MAINTENANCE;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:   // Test dontIgnoreErrors, prolonged, server maintenance,
michael@0:   // login errors are reported
michael@0:   Status.resetSync();
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.dontIgnoreErrors = true;
michael@0:   Status.login = SERVER_MAINTENANCE;
michael@0:   do_check_true(errorHandler.shouldReportError());
michael@0:   do_check_false(errorHandler.didReportProlongedError);
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_shouldReportError_master_password() {
michael@0:   _("Test error ignored due to locked master password");
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   // Monkey patch Service.verifyLogin to imitate
michael@0:   // master password being locked.
michael@0:   Service._verifyLogin = Service.verifyLogin;
michael@0:   Service.verifyLogin = function () {
michael@0:     Status.login = MASTER_PASSWORD_LOCKED;
michael@0:     return false;
michael@0:   };
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0: 
michael@0:   // Clean up.
michael@0:   Service.verifyLogin = Service._verifyLogin;
michael@0:   clean();
michael@0:   let deferred = Promise.defer();
michael@0:   server.stop(deferred.resolve);
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: // XXX - how to arrange for 'Service.identity.basicPassword = null;' in
michael@0: // an fxaccounts environment?
michael@0: add_task(function test_login_syncAndReportErrors_non_network_error() {
michael@0:   // Test non-network errors are reported
michael@0:   // when calling syncAndReportErrors
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0:   Service.identity.basicPassword = null;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onSyncError);
michael@0:     do_check_eq(Status.login, LOGIN_FAILED_NO_PASSWORD);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_sync_syncAndReportErrors_non_network_error() {
michael@0:   // Test non-network errors are reported
michael@0:   // when calling syncAndReportErrors
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   // By calling sync, we ensure we're logged in.
michael@0:   Service.sync();
michael@0:   do_check_eq(Status.sync, SYNC_SUCCEEDED);
michael@0:   do_check_true(Service.isLoggedIn);
michael@0: 
michael@0:   generateCredentialsChangedFailure();
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     do_check_eq(Status.sync, CREDENTIALS_CHANGED);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: // XXX - how to arrange for 'Service.identity.basicPassword = null;' in
michael@0: // an fxaccounts environment?
michael@0: add_task(function test_login_syncAndReportErrors_prolonged_non_network_error() {
michael@0:   // Test prolonged, non-network errors are
michael@0:   // reported when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0:   Service.identity.basicPassword = null;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onSyncError);
michael@0:     do_check_eq(Status.login, LOGIN_FAILED_NO_PASSWORD);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_sync_syncAndReportErrors_prolonged_non_network_error() {
michael@0:   // Test prolonged, non-network errors are
michael@0:   // reported when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   // By calling sync, we ensure we're logged in.
michael@0:   Service.sync();
michael@0:   do_check_eq(Status.sync, SYNC_SUCCEEDED);
michael@0:   do_check_true(Service.isLoggedIn);
michael@0: 
michael@0:   generateCredentialsChangedFailure();
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     do_check_eq(Status.sync, CREDENTIALS_CHANGED);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_login_syncAndReportErrors_network_error() {
michael@0:   // Test network errors are reported when calling syncAndReportErrors.
michael@0:   yield configureIdentity({username: "broken.wipe"});
michael@0:   Service.serverURL  = FAKE_SERVER_URL;
michael@0:   Service.clusterURL = FAKE_SERVER_URL;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onSyncError);
michael@0:     do_check_eq(Status.login, LOGIN_FAILED_NETWORK_ERROR);
michael@0: 
michael@0:     clean();
michael@0:     deferred.resolve();
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: 
michael@0: add_test(function test_sync_syncAndReportErrors_network_error() {
michael@0:   // Test network errors are reported when calling syncAndReportErrors.
michael@0:   Services.io.offline = true;
michael@0: 
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
michael@0: 
michael@0:     Services.io.offline = false;
michael@0:     clean();
michael@0:     run_next_test();
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_login_syncAndReportErrors_prolonged_network_error() {
michael@0:   // Test prolonged, network errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   yield configureIdentity({username: "johndoe"});
michael@0: 
michael@0:   Service.serverURL  = FAKE_SERVER_URL;
michael@0:   Service.clusterURL = FAKE_SERVER_URL;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onSyncError);
michael@0:     do_check_eq(Status.login, LOGIN_FAILED_NETWORK_ERROR);
michael@0: 
michael@0:     clean();
michael@0:     deferred.resolve();
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_test(function test_sync_syncAndReportErrors_prolonged_network_error() {
michael@0:   // Test prolonged, network errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   Services.io.offline = true;
michael@0: 
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
michael@0: 
michael@0:     Services.io.offline = false;
michael@0:     clean();
michael@0:     run_next_test();
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0: });
michael@0: 
michael@0: add_task(function test_login_prolonged_non_network_error() {
michael@0:   // Test prolonged, non-network errors are reported
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0:   Service.identity.basicPassword = null;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onSyncError);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_task(function test_sync_prolonged_non_network_error() {
michael@0:   // Test prolonged, non-network errors are reported
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   // By calling sync, we ensure we're logged in.
michael@0:   Service.sync();
michael@0:   do_check_eq(Status.sync, SYNC_SUCCEEDED);
michael@0:   do_check_true(Service.isLoggedIn);
michael@0: 
michael@0:   generateCredentialsChangedFailure();
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_login_prolonged_network_error() {
michael@0:   // Test prolonged, network errors are reported
michael@0:   yield configureIdentity({username: "johndoe"});
michael@0:   Service.serverURL  = FAKE_SERVER_URL;
michael@0:   Service.clusterURL = FAKE_SERVER_URL;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onSyncError);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     deferred.resolve();
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_test(function test_sync_prolonged_network_error() {
michael@0:   // Test prolonged, network errors are reported
michael@0:   Services.io.offline = true;
michael@0: 
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     Services.io.offline = false;
michael@0:     clean();
michael@0:     run_next_test();
michael@0:   });
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0: });
michael@0: 
michael@0: add_task(function test_login_non_network_error() {
michael@0:   // Test non-network errors are reported
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0:   Service.identity.basicPassword = null;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onSyncError);
michael@0:     do_check_eq(Status.login, LOGIN_FAILED_NO_PASSWORD);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_task(function test_sync_non_network_error() {
michael@0:   // Test non-network errors are reported
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   // By calling sync, we ensure we're logged in.
michael@0:   Service.sync();
michael@0:   do_check_eq(Status.sync, SYNC_SUCCEEDED);
michael@0:   do_check_true(Service.isLoggedIn);
michael@0: 
michael@0:   generateCredentialsChangedFailure();
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onSyncError() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     do_check_eq(Status.sync, CREDENTIALS_CHANGED);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_login_network_error() {
michael@0:   yield configureIdentity({username: "johndoe"});
michael@0:   Service.serverURL  = FAKE_SERVER_URL;
michael@0:   Service.clusterURL = FAKE_SERVER_URL;
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   // Test network errors are not reported.
michael@0:   Svc.Obs.add("weave:ui:clear-error", function onClearError() {
michael@0:     Svc.Obs.remove("weave:ui:clear-error", onClearError);
michael@0: 
michael@0:     do_check_eq(Status.login, LOGIN_FAILED_NETWORK_ERROR);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     Services.io.offline = false;
michael@0:     clean();
michael@0:     deferred.resolve()
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_test(function test_sync_network_error() {
michael@0:   // Test network errors are not reported.
michael@0:   Services.io.offline = true;
michael@0: 
michael@0:   Svc.Obs.add("weave:ui:sync:finish", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:sync:finish", onUIUpdate);
michael@0:     do_check_eq(Status.sync, LOGIN_FAILED_NETWORK_ERROR);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     Services.io.offline = false;
michael@0:     clean();
michael@0:     run_next_test();
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_sync_server_maintenance_error() {
michael@0:   // Test server maintenance errors are not reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   const BACKOFF = 42;
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.enabled = true;
michael@0:   engine.exception = {status: 503,
michael@0:                       headers: {"retry-after": BACKOFF}};
michael@0: 
michael@0:   function onSyncError() {
michael@0:     do_throw("Shouldn't get here!");
michael@0:   }
michael@0:   Svc.Obs.add("weave:ui:sync:error", onSyncError);
michael@0: 
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:finish", function onSyncFinish() {
michael@0:     Svc.Obs.remove("weave:ui:sync:finish", onSyncFinish);
michael@0: 
michael@0:     do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
michael@0:     do_check_eq(Status.sync, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onSyncError);
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_info_collections_login_server_maintenance_error() {
michael@0:   // Test info/collections server maintenance errors are not reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   Service.username = "broken.info";
michael@0:   yield configureIdentity({username: "broken.info"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   function onUIUpdate() {
michael@0:     do_throw("Shouldn't experience UI update!");
michael@0:   }
michael@0:   Svc.Obs.add("weave:ui:login:error", onUIUpdate);
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
michael@0:     Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);
michael@0: 
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_meta_global_login_server_maintenance_error() {
michael@0:   // Test meta/global server maintenance errors are not reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.meta"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   function onUIUpdate() {
michael@0:     do_throw("Shouldn't get here!");
michael@0:   }
michael@0:   Svc.Obs.add("weave:ui:login:error", onUIUpdate);
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
michael@0:     Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);
michael@0: 
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_crypto_keys_login_server_maintenance_error() {
michael@0:   // Test crypto/keys server maintenance errors are not reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.keys"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   // Force re-download of keys
michael@0:   Service.collectionKeys.clear();
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   function onUIUpdate() {
michael@0:     do_throw("Shouldn't get here!");
michael@0:   }
michael@0:   Svc.Obs.add("weave:ui:login:error", onUIUpdate);
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
michael@0:     Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);
michael@0: 
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_task(function test_sync_prolonged_server_maintenance_error() {
michael@0:   // Test prolonged server maintenance errors are reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   const BACKOFF = 42;
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.enabled = true;
michael@0:   engine.exception = {status: 503,
michael@0:                       headers: {"retry-after": BACKOFF}};
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_info_collections_login_prolonged_server_maintenance_error(){
michael@0:   // Test info/collections prolonged server maintenance errors are reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.info"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_meta_global_login_prolonged_server_maintenance_error(){
michael@0:   // Test meta/global prolonged server maintenance errors are reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.meta"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_download_crypto_keys_login_prolonged_server_maintenance_error(){
michael@0:   // Test crypto/keys prolonged server maintenance errors are reported.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.keys"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0:   // Force re-download of keys
michael@0:   Service.collectionKeys.clear();
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_upload_crypto_keys_login_prolonged_server_maintenance_error(){
michael@0:   // Test crypto/keys prolonged server maintenance errors are reported.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   // Start off with an empty account, do not upload a key.
michael@0:   yield configureIdentity({username: "broken.keys"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_wipeServer_login_prolonged_server_maintenance_error(){
michael@0:   // Test that we report prolonged server maintenance errors that occur whilst
michael@0:   // wiping the server.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   // Start off with an empty account, do not upload a key.
michael@0:   yield configureIdentity({username: "broken.wipe"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_wipeRemote_prolonged_server_maintenance_error(){
michael@0:   // Test that we report prolonged server maintenance errors that occur whilst
michael@0:   // wiping all remote devices.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   server.registerPathHandler("/1.1/broken.wipe/storage/catapult", service_unavailable);
michael@0:   yield configureIdentity({username: "broken.wipe"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0:   generateAndUploadKeys();
michael@0: 
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.exception = null;
michael@0:   engine.enabled = true;
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
michael@0:     do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
michael@0:     do_check_true(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   Svc.Prefs.set("firstSync", "wipeRemote");
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_task(function test_sync_syncAndReportErrors_server_maintenance_error() {
michael@0:   // Test server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   const BACKOFF = 42;
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.enabled = true;
michael@0:   engine.exception = {status: 503,
michael@0:                       headers: {"retry-after": BACKOFF}};
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
michael@0:     do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
michael@0:     do_check_eq(Status.sync, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_info_collections_login_syncAndReportErrors_server_maintenance_error() {
michael@0:   // Test info/collections server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.info"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_meta_global_login_syncAndReportErrors_server_maintenance_error() {
michael@0:   // Test meta/global server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.meta"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
michael@0:   // Test crypto/keys server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.keys"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0:   // Force re-download of keys
michael@0:   Service.collectionKeys.clear();
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
michael@0:   // Test crypto/keys server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   // Start off with an empty account, do not upload a key.
michael@0:   yield configureIdentity({username: "broken.keys"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_wipeServer_login_syncAndReportErrors_server_maintenance_error() {
michael@0:   // Test crypto/keys server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   // Start off with an empty account, do not upload a key.
michael@0:   yield configureIdentity({username: "broken.wipe"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_wipeRemote_syncAndReportErrors_server_maintenance_error(){
michael@0:   // Test that we report prolonged server maintenance errors that occur whilst
michael@0:   // wiping all remote devices.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   yield configureIdentity({username: "broken.wipe"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0:   generateAndUploadKeys();
michael@0: 
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.exception = null;
michael@0:   engine.enabled = true;
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, SYNC_FAILED);
michael@0:     do_check_eq(Status.sync, SERVER_MAINTENANCE);
michael@0:     do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   Svc.Prefs.set("firstSync", "wipeRemote");
michael@0:   setLastSync(NON_PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_task(function test_sync_syncAndReportErrors_prolonged_server_maintenance_error() {
michael@0:   // Test prolonged server maintenance errors are
michael@0:   // reported when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   const BACKOFF = 42;
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.enabled = true;
michael@0:   engine.exception = {status: 503,
michael@0:                       headers: {"retry-after": BACKOFF}};
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
michael@0:     do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
michael@0:     do_check_eq(Status.sync, SERVER_MAINTENANCE);
michael@0:     // syncAndReportErrors means dontIgnoreErrors, which means
michael@0:     // didReportProlongedError not touched.
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_info_collections_login_syncAndReportErrors_prolonged_server_maintenance_error() {
michael@0:   // Test info/collections server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.info"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     // syncAndReportErrors means dontIgnoreErrors, which means
michael@0:     // didReportProlongedError not touched.
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_meta_global_login_syncAndReportErrors_prolonged_server_maintenance_error() {
michael@0:   // Test meta/global server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.meta"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     // syncAndReportErrors means dontIgnoreErrors, which means
michael@0:     // didReportProlongedError not touched.
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_download_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
michael@0:   // Test crypto/keys server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0:   yield setUp(server);
michael@0: 
michael@0:   yield configureIdentity({username: "broken.keys"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0:   // Force re-download of keys
michael@0:   Service.collectionKeys.clear();
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     // syncAndReportErrors means dontIgnoreErrors, which means
michael@0:     // didReportProlongedError not touched.
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_upload_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
michael@0:   // Test crypto/keys server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   // Start off with an empty account, do not upload a key.
michael@0:   yield configureIdentity({username: "broken.keys"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     // syncAndReportErrors means dontIgnoreErrors, which means
michael@0:     // didReportProlongedError not touched.
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_identity_test(this, function test_wipeServer_login_syncAndReportErrors_prolonged_server_maintenance_error() {
michael@0:   // Test crypto/keys server maintenance errors are reported
michael@0:   // when calling syncAndReportErrors.
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   // Start off with an empty account, do not upload a key.
michael@0:   yield configureIdentity({username: "broken.wipe"});
michael@0:   Service.serverURL = server.baseURI + "/maintenance/";
michael@0:   Service.clusterURL = server.baseURI + "/maintenance/";
michael@0: 
michael@0:   let backoffInterval;
michael@0:   Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
michael@0:     Svc.Obs.remove("weave:service:backoff:interval", observe);
michael@0:     backoffInterval = subject;
michael@0:   });
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
michael@0:     Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
michael@0:     do_check_true(Status.enforceBackoff);
michael@0:     do_check_eq(backoffInterval, 42);
michael@0:     do_check_eq(Status.service, LOGIN_FAILED);
michael@0:     do_check_eq(Status.login, SERVER_MAINTENANCE);
michael@0:     // syncAndReportErrors means dontIgnoreErrors, which means
michael@0:     // didReportProlongedError not touched.
michael@0:     do_check_false(errorHandler.didReportProlongedError);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_false(Status.enforceBackoff);
michael@0:   do_check_eq(Status.service, STATUS_OK);
michael@0: 
michael@0:   setLastSync(PROLONGED_ERROR_DURATION);
michael@0:   errorHandler.syncAndReportErrors();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_task(function test_sync_engine_generic_fail() {
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.enabled = true;
michael@0:   engine.sync = function sync() {
michael@0:     Svc.Obs.notify("weave:engine:sync:error", "", "catapult");
michael@0:   };
michael@0: 
michael@0:   let log = Log.repository.getLogger("Sync.ErrorHandler");
michael@0:   Svc.Prefs.set("log.appender.file.logOnError", true);
michael@0: 
michael@0:   do_check_eq(Status.engines["catapult"], undefined);
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   // Don't wait for reset-file-log until the sync is underway.
michael@0:   // This avoids us catching a delayed notification from an earlier test.
michael@0:   Svc.Obs.add("weave:engine:sync:finish", function onEngineFinish() {
michael@0:     Svc.Obs.remove("weave:engine:sync:finish", onEngineFinish);
michael@0: 
michael@0:     log.info("Adding reset-file-log observer.");
michael@0:     Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
michael@0:       Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
michael@0: 
michael@0:       // Put these checks here, not after sync(), so that we aren't racing the
michael@0:       // log handler... which resets everything just a few lines below!
michael@0:       _("Status.engines: " + JSON.stringify(Status.engines));
michael@0:       do_check_eq(Status.engines["catapult"], ENGINE_UNKNOWN_FAIL);
michael@0:       do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
michael@0: 
michael@0:       // Test Error log was written on SYNC_FAILED_PARTIAL.
michael@0:       let entries = logsdir.directoryEntries;
michael@0:       do_check_true(entries.hasMoreElements());
michael@0:       let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
michael@0:       do_check_eq(logfile.leafName.slice(0, LOG_PREFIX_ERROR.length),
michael@0:                   LOG_PREFIX_ERROR);
michael@0: 
michael@0:       clean();
michael@0:       server.stop(deferred.resolve);
michael@0:     });
michael@0:   });
michael@0: 
michael@0:   do_check_true(yield setUp(server));
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });
michael@0: 
michael@0: add_test(function test_logs_on_sync_error_despite_shouldReportError() {
michael@0:   _("Ensure that an error is still logged when weave:service:sync:error " +
michael@0:     "is notified, despite shouldReportError returning false.");
michael@0: 
michael@0:   let log = Log.repository.getLogger("Sync.ErrorHandler");
michael@0:   Svc.Prefs.set("log.appender.file.logOnError", true);
michael@0:   log.info("TESTING");
michael@0: 
michael@0:   // Ensure that we report no error.
michael@0:   Status.login = MASTER_PASSWORD_LOCKED;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0: 
michael@0:   Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
michael@0:     Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
michael@0: 
michael@0:     // Test that error log was written.
michael@0:     let entries = logsdir.directoryEntries;
michael@0:     do_check_true(entries.hasMoreElements());
michael@0:     let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
michael@0:     do_check_eq(logfile.leafName.slice(0, LOG_PREFIX_ERROR.length),
michael@0:                 LOG_PREFIX_ERROR);
michael@0: 
michael@0:     clean();
michael@0:     run_next_test();
michael@0:   });
michael@0:   Svc.Obs.notify("weave:service:sync:error", {});
michael@0: });
michael@0: 
michael@0: add_test(function test_logs_on_login_error_despite_shouldReportError() {
michael@0:   _("Ensure that an error is still logged when weave:service:login:error " +
michael@0:     "is notified, despite shouldReportError returning false.");
michael@0: 
michael@0:   let log = Log.repository.getLogger("Sync.ErrorHandler");
michael@0:   Svc.Prefs.set("log.appender.file.logOnError", true);
michael@0:   log.info("TESTING");
michael@0: 
michael@0:   // Ensure that we report no error.
michael@0:   Status.login = MASTER_PASSWORD_LOCKED;
michael@0:   do_check_false(errorHandler.shouldReportError());
michael@0: 
michael@0:   Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
michael@0:     Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
michael@0: 
michael@0:     // Test that error log was written.
michael@0:     let entries = logsdir.directoryEntries;
michael@0:     do_check_true(entries.hasMoreElements());
michael@0:     let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
michael@0:     do_check_eq(logfile.leafName.slice(0, LOG_PREFIX_ERROR.length),
michael@0:                 LOG_PREFIX_ERROR);
michael@0: 
michael@0:     clean();
michael@0:     run_next_test();
michael@0:   });
michael@0:   Svc.Obs.notify("weave:service:login:error", {});
michael@0: });
michael@0: 
michael@0: // This test should be the last one since it monkeypatches the engine object
michael@0: // and we should only have one engine object throughout the file (bug 629664).
michael@0: add_task(function test_engine_applyFailed() {
michael@0:   let server = sync_httpd_setup();
michael@0: 
michael@0:   let engine = engineManager.get("catapult");
michael@0:   engine.enabled = true;
michael@0:   delete engine.exception;
michael@0:   engine.sync = function sync() {
michael@0:     Svc.Obs.notify("weave:engine:sync:applied", {newFailed:1}, "catapult");
michael@0:   };
michael@0: 
michael@0:   let log = Log.repository.getLogger("Sync.ErrorHandler");
michael@0:   Svc.Prefs.set("log.appender.file.logOnError", true);
michael@0: 
michael@0:   let deferred = Promise.defer();
michael@0:   Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
michael@0:     Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
michael@0: 
michael@0:     do_check_eq(Status.engines["catapult"], ENGINE_APPLY_FAIL);
michael@0:     do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
michael@0: 
michael@0:     // Test Error log was written on SYNC_FAILED_PARTIAL.
michael@0:     let entries = logsdir.directoryEntries;
michael@0:     do_check_true(entries.hasMoreElements());
michael@0:     let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
michael@0:     do_check_eq(logfile.leafName.slice(0, LOG_PREFIX_ERROR.length),
michael@0:                 LOG_PREFIX_ERROR);
michael@0: 
michael@0:     clean();
michael@0:     server.stop(deferred.resolve);
michael@0:   });
michael@0: 
michael@0:   do_check_eq(Status.engines["catapult"], undefined);
michael@0:   do_check_true(yield setUp(server));
michael@0:   Service.sync();
michael@0:   yield deferred.promise;
michael@0: });