1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/mobile/android/base/background/healthreport/upload/AndroidSubmissionClient.java Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,463 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +package org.mozilla.gecko.background.healthreport.upload;
1.9 +
1.10 +import java.io.IOException;
1.11 +import java.io.UnsupportedEncodingException;
1.12 +import java.net.URISyntaxException;
1.13 +import java.util.ArrayList;
1.14 +import java.util.Collection;
1.15 +
1.16 +import org.json.JSONException;
1.17 +import org.json.JSONObject;
1.18 +import org.mozilla.gecko.BrowserLocaleManager;
1.19 +import org.mozilla.gecko.background.bagheera.BagheeraClient;
1.20 +import org.mozilla.gecko.background.bagheera.BagheeraRequestDelegate;
1.21 +import org.mozilla.gecko.background.common.GlobalConstants;
1.22 +import org.mozilla.gecko.background.common.log.Logger;
1.23 +import org.mozilla.gecko.background.healthreport.Environment;
1.24 +import org.mozilla.gecko.background.healthreport.EnvironmentBuilder;
1.25 +import org.mozilla.gecko.background.healthreport.HealthReportConstants;
1.26 +import org.mozilla.gecko.background.healthreport.HealthReportDatabaseStorage;
1.27 +import org.mozilla.gecko.background.healthreport.HealthReportGenerator;
1.28 +import org.mozilla.gecko.background.healthreport.HealthReportStorage;
1.29 +import org.mozilla.gecko.background.healthreport.HealthReportStorage.Field;
1.30 +import org.mozilla.gecko.background.healthreport.HealthReportStorage.MeasurementFields;
1.31 +import org.mozilla.gecko.background.healthreport.ProfileInformationCache;
1.32 +import org.mozilla.gecko.sync.net.BaseResource;
1.33 +
1.34 +import android.content.ContentProviderClient;
1.35 +import android.content.Context;
1.36 +import android.content.SharedPreferences;
1.37 +import ch.boye.httpclientandroidlib.HttpResponse;
1.38 +
1.39 +public class AndroidSubmissionClient implements SubmissionClient {
1.40 + protected static final String LOG_TAG = AndroidSubmissionClient.class.getSimpleName();
1.41 +
1.42 + private static final String MEASUREMENT_NAME_SUBMISSIONS = "org.mozilla.healthreport.submissions";
1.43 + private static final int MEASUREMENT_VERSION_SUBMISSIONS = 1;
1.44 +
1.45 + protected final Context context;
1.46 + protected final SharedPreferences sharedPreferences;
1.47 + protected final String profilePath;
1.48 +
1.49 + public AndroidSubmissionClient(Context context, SharedPreferences sharedPreferences, String profilePath) {
1.50 + this.context = context;
1.51 + this.sharedPreferences = sharedPreferences;
1.52 + this.profilePath = profilePath;
1.53 + }
1.54 +
1.55 + public SharedPreferences getSharedPreferences() {
1.56 + return sharedPreferences;
1.57 + }
1.58 +
1.59 + public String getDocumentServerURI() {
1.60 + return getSharedPreferences().getString(HealthReportConstants.PREF_DOCUMENT_SERVER_URI, HealthReportConstants.DEFAULT_DOCUMENT_SERVER_URI);
1.61 + }
1.62 +
1.63 + public String getDocumentServerNamespace() {
1.64 + return getSharedPreferences().getString(HealthReportConstants.PREF_DOCUMENT_SERVER_NAMESPACE, HealthReportConstants.DEFAULT_DOCUMENT_SERVER_NAMESPACE);
1.65 + }
1.66 +
1.67 + public long getLastUploadLocalTime() {
1.68 + return getSharedPreferences().getLong(HealthReportConstants.PREF_LAST_UPLOAD_LOCAL_TIME, 0L);
1.69 + }
1.70 +
1.71 + public String getLastUploadDocumentId() {
1.72 + return getSharedPreferences().getString(HealthReportConstants.PREF_LAST_UPLOAD_DOCUMENT_ID, null);
1.73 + }
1.74 +
1.75 + public boolean hasUploadBeenRequested() {
1.76 + return getSharedPreferences().contains(HealthReportConstants.PREF_LAST_UPLOAD_REQUESTED);
1.77 + }
1.78 +
1.79 + public void setLastUploadLocalTimeAndDocumentId(long localTime, String id) {
1.80 + getSharedPreferences().edit()
1.81 + .putLong(HealthReportConstants.PREF_LAST_UPLOAD_LOCAL_TIME, localTime)
1.82 + .putString(HealthReportConstants.PREF_LAST_UPLOAD_DOCUMENT_ID, id)
1.83 + .commit();
1.84 + }
1.85 +
1.86 + protected HealthReportDatabaseStorage getStorage(final ContentProviderClient client) {
1.87 + return EnvironmentBuilder.getStorage(client, profilePath);
1.88 + }
1.89 +
1.90 + protected JSONObject generateDocument(final long localTime, final long last,
1.91 + final SubmissionsTracker tracker) throws JSONException {
1.92 + final long since = localTime - GlobalConstants.MILLISECONDS_PER_SIX_MONTHS;
1.93 + final HealthReportGenerator generator = tracker.getGenerator();
1.94 + return generator.generateDocument(since, last, profilePath);
1.95 + }
1.96 +
1.97 + protected void uploadPayload(String id, String payload, Collection<String> oldIds, BagheeraRequestDelegate uploadDelegate) {
1.98 + final BagheeraClient client = new BagheeraClient(getDocumentServerURI());
1.99 +
1.100 + Logger.pii(LOG_TAG, "New health report has id " + id +
1.101 + "and obsoletes " + (oldIds != null ? Integer.toString(oldIds.size()) : "no") + " old ids.");
1.102 +
1.103 + try {
1.104 + client.uploadJSONDocument(getDocumentServerNamespace(),
1.105 + id,
1.106 + payload,
1.107 + oldIds,
1.108 + uploadDelegate);
1.109 + } catch (Exception e) {
1.110 + uploadDelegate.handleError(e);
1.111 + }
1.112 + }
1.113 +
1.114 + @Override
1.115 + public void upload(long localTime, String id, Collection<String> oldIds, Delegate delegate) {
1.116 + // We abuse the life-cycle of an Android ContentProvider slightly by holding
1.117 + // onto a ContentProviderClient while we generate a payload. This keeps our
1.118 + // database storage alive, and may also allow us to share a database
1.119 + // connection with a BrowserHealthRecorder from Fennec. The ContentProvider
1.120 + // owns all underlying Storage instances, so we don't need to explicitly
1.121 + // close them.
1.122 + ContentProviderClient client = EnvironmentBuilder.getContentProviderClient(context);
1.123 + if (client == null) {
1.124 + // TODO: Bug 910898 - Store client failure in SharedPrefs so we can increment next time with storage.
1.125 + delegate.onHardFailure(localTime, null, "Could not fetch content provider client.", null);
1.126 + return;
1.127 + }
1.128 +
1.129 + try {
1.130 + // Storage instance is owned by HealthReportProvider, so we don't need to
1.131 + // close it. It's worth noting that this call will fail if called
1.132 + // out-of-process.
1.133 + final HealthReportDatabaseStorage storage = getStorage(client);
1.134 + if (storage == null) {
1.135 + // TODO: Bug 910898 - Store error in SharedPrefs so we can increment next time with storage.
1.136 + delegate.onHardFailure(localTime, null, "No storage when generating report.", null);
1.137 + return;
1.138 + }
1.139 +
1.140 + long last = Math.max(getLastUploadLocalTime(), HealthReportConstants.EARLIEST_LAST_PING);
1.141 + if (!storage.hasEventSince(last)) {
1.142 + delegate.onHardFailure(localTime, null, "No new events in storage.", null);
1.143 + return;
1.144 + }
1.145 +
1.146 + initializeStorageForUploadProviders(storage);
1.147 +
1.148 + final SubmissionsTracker tracker =
1.149 + getSubmissionsTracker(storage, localTime, hasUploadBeenRequested());
1.150 + try {
1.151 + // TODO: Bug 910898 - Add errors from sharedPrefs to tracker.
1.152 + final JSONObject document = generateDocument(localTime, last, tracker);
1.153 + if (document == null) {
1.154 + delegate.onHardFailure(localTime, null, "Generator returned null document.", null);
1.155 + return;
1.156 + }
1.157 +
1.158 + final BagheeraRequestDelegate uploadDelegate = tracker.getDelegate(delegate, localTime,
1.159 + true, id);
1.160 + this.uploadPayload(id, document.toString(), oldIds, uploadDelegate);
1.161 + } catch (Exception e) {
1.162 + // Incrementing the failure count here could potentially cause the failure count to be
1.163 + // incremented twice, but this helper class checks and prevents this.
1.164 + tracker.incrementUploadClientFailureCount();
1.165 + throw e;
1.166 + }
1.167 + } catch (Exception e) {
1.168 + // TODO: Bug 910898 - Store client failure in SharedPrefs so we can increment next time with storage.
1.169 + Logger.warn(LOG_TAG, "Got exception generating document.", e);
1.170 + delegate.onHardFailure(localTime, null, "Got exception uploading.", e);
1.171 + return;
1.172 + } finally {
1.173 + client.release();
1.174 + }
1.175 + }
1.176 +
1.177 + protected SubmissionsTracker getSubmissionsTracker(final HealthReportStorage storage,
1.178 + final long localTime, final boolean hasUploadBeenRequested) {
1.179 + return new SubmissionsTracker(storage, localTime, hasUploadBeenRequested);
1.180 + }
1.181 +
1.182 + @Override
1.183 + public void delete(final long localTime, final String id, Delegate delegate) {
1.184 + final BagheeraClient client = new BagheeraClient(getDocumentServerURI());
1.185 +
1.186 + Logger.pii(LOG_TAG, "Deleting health report with id " + id + ".");
1.187 +
1.188 + BagheeraRequestDelegate deleteDelegate = new RequestDelegate(delegate, localTime, false, id);
1.189 + try {
1.190 + client.deleteDocument(getDocumentServerNamespace(), id, deleteDelegate);
1.191 + } catch (Exception e) {
1.192 + deleteDelegate.handleError(e);
1.193 + }
1.194 + }
1.195 +
1.196 + protected class RequestDelegate implements BagheeraRequestDelegate {
1.197 + protected final Delegate delegate;
1.198 + protected final boolean isUpload;
1.199 + protected final String methodString;
1.200 + protected final long localTime;
1.201 + protected final String id;
1.202 +
1.203 + public RequestDelegate(Delegate delegate, long localTime, boolean isUpload, String id) {
1.204 + this.delegate = delegate;
1.205 + this.localTime = localTime;
1.206 + this.isUpload = isUpload;
1.207 + this.methodString = this.isUpload ? "upload" : "delete";
1.208 + this.id = id;
1.209 + }
1.210 +
1.211 + @Override
1.212 + public String getUserAgent() {
1.213 + return HealthReportConstants.USER_AGENT;
1.214 + }
1.215 +
1.216 + @Override
1.217 + public void handleSuccess(int status, String namespace, String id, HttpResponse response) {
1.218 + BaseResource.consumeEntity(response);
1.219 + if (isUpload) {
1.220 + setLastUploadLocalTimeAndDocumentId(localTime, id);
1.221 + }
1.222 + Logger.debug(LOG_TAG, "Successful " + methodString + " at " + localTime + ".");
1.223 + delegate.onSuccess(localTime, id);
1.224 + }
1.225 +
1.226 + /**
1.227 + * Bagheera status codes:
1.228 + *
1.229 + * 403 Forbidden - Violated access restrictions. Most likely because of the method used.
1.230 + * 413 Request Too Large - Request payload was larger than the configured maximum.
1.231 + * 400 Bad Request - Returned if the POST/PUT failed validation in some manner.
1.232 + * 404 Not Found - Returned if the URI path doesn't exist or if the URI was not in the proper format.
1.233 + * 500 Server Error - General server error. Someone with access should look at the logs for more details.
1.234 + */
1.235 + @Override
1.236 + public void handleFailure(int status, String namespace, HttpResponse response) {
1.237 + BaseResource.consumeEntity(response);
1.238 + Logger.debug(LOG_TAG, "Failed " + methodString + " at " + localTime + ".");
1.239 + if (status >= 500) {
1.240 + delegate.onSoftFailure(localTime, id, "Got status " + status + " from server.", null);
1.241 + return;
1.242 + }
1.243 + // Things are either bad locally (bad payload format, too much data) or
1.244 + // bad remotely (badly configured server, temporarily unavailable). Try
1.245 + // again tomorrow.
1.246 + delegate.onHardFailure(localTime, id, "Got status " + status + " from server.", null);
1.247 + }
1.248 +
1.249 + @Override
1.250 + public void handleError(Exception e) {
1.251 + Logger.debug(LOG_TAG, "Exception during " + methodString + " at " + localTime + ".", e);
1.252 + if (e instanceof IOException) {
1.253 + // Let's assume IO exceptions are Android dropping the network.
1.254 + delegate.onSoftFailure(localTime, id, "Got exception during " + methodString + ".", e);
1.255 + return;
1.256 + }
1.257 + delegate.onHardFailure(localTime, id, "Got exception during " + methodString + ".", e);
1.258 + }
1.259 + };
1.260 +
1.261 + private void initializeStorageForUploadProviders(HealthReportDatabaseStorage storage) {
1.262 + storage.beginInitialization();
1.263 + try {
1.264 + initializeSubmissionsProvider(storage);
1.265 + storage.finishInitialization();
1.266 + } catch (Exception e) {
1.267 + // TODO: Bug 910898 - Store error in SharedPrefs so we can increment next time with storage.
1.268 + storage.abortInitialization();
1.269 + throw new IllegalStateException("Could not initialize storage for upload provider.", e);
1.270 + }
1.271 + }
1.272 +
1.273 + private void initializeSubmissionsProvider(HealthReportDatabaseStorage storage) {
1.274 + storage.ensureMeasurementInitialized(
1.275 + MEASUREMENT_NAME_SUBMISSIONS,
1.276 + MEASUREMENT_VERSION_SUBMISSIONS,
1.277 + new MeasurementFields() {
1.278 + @Override
1.279 + public Iterable<FieldSpec> getFields() {
1.280 + final ArrayList<FieldSpec> out = new ArrayList<FieldSpec>();
1.281 + for (SubmissionsFieldName fieldName : SubmissionsFieldName.values()) {
1.282 + FieldSpec spec = new FieldSpec(fieldName.getName(), Field.TYPE_INTEGER_COUNTER);
1.283 + out.add(spec);
1.284 + }
1.285 + return out;
1.286 + }
1.287 + });
1.288 + }
1.289 +
1.290 + public static enum SubmissionsFieldName {
1.291 + FIRST_ATTEMPT("firstDocumentUploadAttempt"),
1.292 + CONTINUATION_ATTEMPT("continuationDocumentUploadAttempt"),
1.293 + SUCCESS("uploadSuccess"),
1.294 + TRANSPORT_FAILURE("uploadTransportFailure"),
1.295 + SERVER_FAILURE("uploadServerFailure"),
1.296 + CLIENT_FAILURE("uploadClientFailure");
1.297 +
1.298 + private final String name;
1.299 +
1.300 + SubmissionsFieldName(String name) {
1.301 + this.name = name;
1.302 + }
1.303 +
1.304 + public String getName() {
1.305 + return name;
1.306 + }
1.307 +
1.308 + public int getID(HealthReportStorage storage) {
1.309 + final Field field = storage.getField(MEASUREMENT_NAME_SUBMISSIONS,
1.310 + MEASUREMENT_VERSION_SUBMISSIONS,
1.311 + name);
1.312 + return field.getID();
1.313 + }
1.314 + }
1.315 +
1.316 + /**
1.317 + * Encapsulates the counting mechanisms for submissions status counts. Ensures multiple failures
1.318 + * and successes are not recorded for a single instance.
1.319 + */
1.320 + public class SubmissionsTracker {
1.321 + private final HealthReportStorage storage;
1.322 + private final ProfileInformationCache profileCache;
1.323 + private final int day;
1.324 + private final int envID;
1.325 +
1.326 + private boolean isUploadStatusCountIncremented;
1.327 +
1.328 + public SubmissionsTracker(final HealthReportStorage storage, final long localTime,
1.329 + final boolean hasUploadBeenRequested) throws IllegalStateException {
1.330 + this.storage = storage;
1.331 + this.profileCache = getProfileInformationCache();
1.332 + this.day = storage.getDay(localTime);
1.333 + this.envID = registerCurrentEnvironment();
1.334 +
1.335 + this.isUploadStatusCountIncremented = false;
1.336 +
1.337 + if (!hasUploadBeenRequested) {
1.338 + incrementFirstUploadAttemptCount();
1.339 + } else {
1.340 + incrementContinuationAttemptCount();
1.341 + }
1.342 + }
1.343 +
1.344 + protected ProfileInformationCache getProfileInformationCache() {
1.345 + final ProfileInformationCache profileCache = new ProfileInformationCache(profilePath);
1.346 + if (!profileCache.restoreUnlessInitialized()) {
1.347 + Logger.warn(LOG_TAG, "Not enough profile information to compute current environment.");
1.348 + throw new IllegalStateException("Could not retrieve current environment.");
1.349 + }
1.350 + return profileCache;
1.351 + }
1.352 +
1.353 + protected int registerCurrentEnvironment() {
1.354 + return EnvironmentBuilder.registerCurrentEnvironment(storage, profileCache);
1.355 + }
1.356 +
1.357 + protected void incrementFirstUploadAttemptCount() {
1.358 + Logger.debug(LOG_TAG, "Incrementing first upload attempt field.");
1.359 + storage.incrementDailyCount(envID, day, SubmissionsFieldName.FIRST_ATTEMPT.getID(storage));
1.360 + }
1.361 +
1.362 + protected void incrementContinuationAttemptCount() {
1.363 + Logger.debug(LOG_TAG, "Incrementing continuation upload attempt field.");
1.364 + storage.incrementDailyCount(envID, day, SubmissionsFieldName.CONTINUATION_ATTEMPT.getID(storage));
1.365 + }
1.366 +
1.367 + public void incrementUploadSuccessCount() {
1.368 + incrementStatusCount(SubmissionsFieldName.SUCCESS.getID(storage), "success");
1.369 + }
1.370 +
1.371 + public void incrementUploadClientFailureCount() {
1.372 + incrementStatusCount(SubmissionsFieldName.CLIENT_FAILURE.getID(storage), "client failure");
1.373 + }
1.374 +
1.375 + public void incrementUploadTransportFailureCount() {
1.376 + incrementStatusCount(SubmissionsFieldName.TRANSPORT_FAILURE.getID(storage), "transport failure");
1.377 + }
1.378 +
1.379 + public void incrementUploadServerFailureCount() {
1.380 + incrementStatusCount(SubmissionsFieldName.SERVER_FAILURE.getID(storage), "server failure");
1.381 + }
1.382 +
1.383 + private void incrementStatusCount(final int fieldID, final String countType) {
1.384 + if (!isUploadStatusCountIncremented) {
1.385 + Logger.debug(LOG_TAG, "Incrementing upload attempt " + countType + " count.");
1.386 + storage.incrementDailyCount(envID, day, fieldID);
1.387 + isUploadStatusCountIncremented = true;
1.388 + } else {
1.389 + Logger.warn(LOG_TAG, "Upload status count already incremented - not incrementing " +
1.390 + countType + " count.");
1.391 + }
1.392 + }
1.393 +
1.394 + public TrackingGenerator getGenerator() {
1.395 + return new TrackingGenerator();
1.396 + }
1.397 +
1.398 + public class TrackingGenerator extends HealthReportGenerator {
1.399 + public TrackingGenerator() {
1.400 + super(storage);
1.401 + }
1.402 +
1.403 + @Override
1.404 + public JSONObject generateDocument(long since, long lastPingTime,
1.405 + String generationProfilePath) throws JSONException {
1.406 +
1.407 + // Let's make sure we have an accurate locale.
1.408 + BrowserLocaleManager.getInstance().getAndApplyPersistedLocale(context);
1.409 +
1.410 + final JSONObject document;
1.411 + // If the given profilePath matches the one we cached for the tracker, use the cached env.
1.412 + if (profilePath != null && profilePath.equals(generationProfilePath)) {
1.413 + final Environment environment = getCurrentEnvironment();
1.414 + document = super.generateDocument(since, lastPingTime, environment);
1.415 + } else {
1.416 + document = super.generateDocument(since, lastPingTime, generationProfilePath);
1.417 + }
1.418 +
1.419 + if (document == null) {
1.420 + incrementUploadClientFailureCount();
1.421 + }
1.422 + return document;
1.423 + }
1.424 +
1.425 + protected Environment getCurrentEnvironment() {
1.426 + return EnvironmentBuilder.getCurrentEnvironment(profileCache);
1.427 + }
1.428 + }
1.429 +
1.430 + public TrackingRequestDelegate getDelegate(final Delegate delegate, final long localTime,
1.431 + final boolean isUpload, final String id) {
1.432 + return new TrackingRequestDelegate(delegate, localTime, isUpload, id);
1.433 + }
1.434 +
1.435 + public class TrackingRequestDelegate extends RequestDelegate {
1.436 + public TrackingRequestDelegate(final Delegate delegate, final long localTime,
1.437 + final boolean isUpload, final String id) {
1.438 + super(delegate, localTime, isUpload, id);
1.439 + }
1.440 +
1.441 + @Override
1.442 + public void handleSuccess(int status, String namespace, String id, HttpResponse response) {
1.443 + super.handleSuccess(status, namespace, id, response);
1.444 + incrementUploadSuccessCount();
1.445 + }
1.446 +
1.447 + @Override
1.448 + public void handleFailure(int status, String namespace, HttpResponse response) {
1.449 + super.handleFailure(status, namespace, response);
1.450 + incrementUploadServerFailureCount();
1.451 + }
1.452 +
1.453 + @Override
1.454 + public void handleError(Exception e) {
1.455 + super.handleError(e);
1.456 + if (e instanceof IllegalArgumentException ||
1.457 + e instanceof UnsupportedEncodingException ||
1.458 + e instanceof URISyntaxException) {
1.459 + incrementUploadClientFailureCount();
1.460 + } else {
1.461 + incrementUploadTransportFailureCount();
1.462 + }
1.463 + }
1.464 + }
1.465 + }
1.466 +}
