michael@0: /* This Source Code Form is subject to the terms of the Mozilla Public michael@0: * License, v. 2.0. If a copy of the MPL was not distributed with this michael@0: * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ michael@0: michael@0: package org.mozilla.gecko.background.healthreport; michael@0: michael@0: import java.util.HashSet; michael@0: import java.util.Iterator; michael@0: import java.util.Set; michael@0: michael@0: import org.json.JSONException; michael@0: import org.json.JSONObject; michael@0: import org.mozilla.gecko.background.common.DateUtils.DateFormatter; michael@0: import org.mozilla.gecko.background.common.log.Logger; michael@0: import org.mozilla.gecko.background.healthreport.HealthReportStorage.Field; michael@0: michael@0: import android.database.Cursor; michael@0: import android.util.SparseArray; michael@0: michael@0: public class HealthReportGenerator { michael@0: private static final int PAYLOAD_VERSION = 3; michael@0: michael@0: private static final String LOG_TAG = "GeckoHealthGen"; michael@0: michael@0: private final HealthReportStorage storage; michael@0: private final DateFormatter dateFormatter; michael@0: michael@0: public HealthReportGenerator(HealthReportStorage storage) { michael@0: this.storage = storage; michael@0: this.dateFormatter = new DateFormatter(); michael@0: } michael@0: michael@0: @SuppressWarnings("static-method") michael@0: protected long now() { michael@0: return System.currentTimeMillis(); michael@0: } michael@0: michael@0: /** michael@0: * Ensure that you have initialized the Locale to your satisfaction michael@0: * prior to calling this method. michael@0: * michael@0: * @return null if no environment could be computed, or else the resulting document. michael@0: * @throws JSONException if there was an error adding environment data to the resulting document. michael@0: */ michael@0: public JSONObject generateDocument(long since, long lastPingTime, String profilePath) throws JSONException { michael@0: Logger.info(LOG_TAG, "Generating FHR document from " + since + "; last ping " + lastPingTime); michael@0: Logger.pii(LOG_TAG, "Generating for profile " + profilePath); michael@0: michael@0: ProfileInformationCache cache = new ProfileInformationCache(profilePath); michael@0: if (!cache.restoreUnlessInitialized()) { michael@0: Logger.warn(LOG_TAG, "Not enough profile information to compute current environment."); michael@0: return null; michael@0: } michael@0: Environment current = EnvironmentBuilder.getCurrentEnvironment(cache); michael@0: return generateDocument(since, lastPingTime, current); michael@0: } michael@0: michael@0: /** michael@0: * The document consists of: michael@0: * michael@0: * michael@0: * michael@0: * days is a map from date strings to {hash: {measurement: {_v: version, fields...}}}. michael@0: * @throws JSONException if there was an error adding environment data to the resulting document. michael@0: */ michael@0: public JSONObject generateDocument(long since, long lastPingTime, Environment currentEnvironment) throws JSONException { michael@0: final String currentHash = currentEnvironment.getHash(); michael@0: michael@0: Logger.debug(LOG_TAG, "Current environment hash: " + currentHash); michael@0: if (currentHash == null) { michael@0: Logger.warn(LOG_TAG, "Current hash is null; aborting."); michael@0: return null; michael@0: } michael@0: michael@0: // We want to map field IDs to some strings as we go. michael@0: SparseArray envs = storage.getEnvironmentRecordsByID(); michael@0: michael@0: JSONObject document = new JSONObject(); michael@0: michael@0: if (lastPingTime >= HealthReportConstants.EARLIEST_LAST_PING) { michael@0: document.put("lastPingDate", dateFormatter.getDateString(lastPingTime)); michael@0: } michael@0: michael@0: document.put("thisPingDate", dateFormatter.getDateString(now())); michael@0: document.put("version", PAYLOAD_VERSION); michael@0: michael@0: document.put("environments", getEnvironmentsJSON(currentEnvironment, envs)); michael@0: document.put("data", getDataJSON(currentEnvironment, envs, since)); michael@0: michael@0: return document; michael@0: } michael@0: michael@0: protected JSONObject getDataJSON(Environment currentEnvironment, michael@0: SparseArray envs, long since) throws JSONException { michael@0: SparseArray fields = storage.getFieldsByID(); michael@0: michael@0: JSONObject days = getDaysJSON(currentEnvironment, envs, fields, since); michael@0: michael@0: JSONObject last = new JSONObject(); michael@0: michael@0: JSONObject data = new JSONObject(); michael@0: data.put("days", days); michael@0: data.put("last", last); michael@0: return data; michael@0: } michael@0: michael@0: protected JSONObject getDaysJSON(Environment currentEnvironment, SparseArray envs, SparseArray fields, long since) throws JSONException { michael@0: if (Logger.shouldLogVerbose(LOG_TAG)) { michael@0: for (int i = 0; i < envs.size(); ++i) { michael@0: Logger.trace(LOG_TAG, "Days environment " + envs.keyAt(i) + ": " + envs.get(envs.keyAt(i)).getHash()); michael@0: } michael@0: } michael@0: michael@0: JSONObject days = new JSONObject(); michael@0: Cursor cursor = storage.getRawEventsSince(since); michael@0: try { michael@0: if (!cursor.moveToFirst()) { michael@0: return days; michael@0: } michael@0: michael@0: // A classic walking partition. michael@0: // Columns are "date", "env", "field", "value". michael@0: // Note that we care about the type (integer, string) and kind michael@0: // (last/counter, discrete) of each field. michael@0: // Each field will be accessed once for each date/env pair, so michael@0: // Field memoizes these facts. michael@0: // We also care about which measurement contains each field. michael@0: int lastDate = -1; michael@0: int lastEnv = -1; michael@0: JSONObject dateObject = null; michael@0: JSONObject envObject = null; michael@0: michael@0: while (!cursor.isAfterLast()) { michael@0: int cEnv = cursor.getInt(1); michael@0: if (cEnv == -1 || michael@0: (cEnv != lastEnv && michael@0: envs.indexOfKey(cEnv) < 0)) { michael@0: Logger.warn(LOG_TAG, "Invalid environment " + cEnv + " in cursor. Skipping."); michael@0: cursor.moveToNext(); michael@0: continue; michael@0: } michael@0: michael@0: int cDate = cursor.getInt(0); michael@0: int cField = cursor.getInt(2); michael@0: michael@0: Logger.trace(LOG_TAG, "Event row: " + cDate + ", " + cEnv + ", " + cField); michael@0: boolean dateChanged = cDate != lastDate; michael@0: boolean envChanged = cEnv != lastEnv; michael@0: michael@0: if (dateChanged) { michael@0: if (dateObject != null) { michael@0: days.put(dateFormatter.getDateStringForDay(lastDate), dateObject); michael@0: } michael@0: dateObject = new JSONObject(); michael@0: lastDate = cDate; michael@0: } michael@0: michael@0: if (dateChanged || envChanged) { michael@0: envObject = new JSONObject(); michael@0: // This is safe because we checked above that cEnv is valid. michael@0: dateObject.put(envs.get(cEnv).getHash(), envObject); michael@0: lastEnv = cEnv; michael@0: } michael@0: michael@0: final Field field = fields.get(cField); michael@0: JSONObject measurement = envObject.optJSONObject(field.measurementName); michael@0: if (measurement == null) { michael@0: // We will never have more than one measurement version within a michael@0: // single environment -- to do so involves changing the build ID. And michael@0: // even if we did, we have no way to represent it. So just build the michael@0: // output object once. michael@0: measurement = new JSONObject(); michael@0: measurement.put("_v", field.measurementVersion); michael@0: envObject.put(field.measurementName, measurement); michael@0: } michael@0: michael@0: // How we record depends on the type of the field, so we michael@0: // break this out into a separate method for clarity. michael@0: recordMeasurementFromCursor(field, measurement, cursor); michael@0: michael@0: cursor.moveToNext(); michael@0: continue; michael@0: } michael@0: days.put(dateFormatter.getDateStringForDay(lastDate), dateObject); michael@0: } finally { michael@0: cursor.close(); michael@0: } michael@0: return days; michael@0: } michael@0: michael@0: /** michael@0: * Return the {@link JSONObject} parsed from the provided index of the given michael@0: * cursor, or {@link JSONObject#NULL} if either SQL NULL or michael@0: * string "null" is present at that index. michael@0: */ michael@0: private static Object getJSONAtIndex(Cursor cursor, int index) throws JSONException { michael@0: if (cursor.isNull(index)) { michael@0: return JSONObject.NULL; michael@0: } michael@0: final String value = cursor.getString(index); michael@0: if ("null".equals(value)) { michael@0: return JSONObject.NULL; michael@0: } michael@0: return new JSONObject(value); michael@0: } michael@0: michael@0: protected static void recordMeasurementFromCursor(final Field field, michael@0: JSONObject measurement, michael@0: Cursor cursor) michael@0: throws JSONException { michael@0: if (field.isDiscreteField()) { michael@0: // Discrete counted. Increment the named counter. michael@0: if (field.isCountedField()) { michael@0: if (!field.isStringField()) { michael@0: throw new IllegalStateException("Unable to handle non-string counted types."); michael@0: } michael@0: HealthReportUtils.count(measurement, field.fieldName, cursor.getString(3)); michael@0: return; michael@0: } michael@0: michael@0: // Discrete string or integer. Append it. michael@0: if (field.isStringField()) { michael@0: HealthReportUtils.append(measurement, field.fieldName, cursor.getString(3)); michael@0: return; michael@0: } michael@0: if (field.isJSONField()) { michael@0: HealthReportUtils.append(measurement, field.fieldName, getJSONAtIndex(cursor, 3)); michael@0: return; michael@0: } michael@0: if (field.isIntegerField()) { michael@0: HealthReportUtils.append(measurement, field.fieldName, cursor.getLong(3)); michael@0: return; michael@0: } michael@0: throw new IllegalStateException("Unknown field type: " + field.flags); michael@0: } michael@0: michael@0: // Non-discrete -- must be LAST or COUNTER, so just accumulate the value. michael@0: if (field.isStringField()) { michael@0: measurement.put(field.fieldName, cursor.getString(3)); michael@0: return; michael@0: } michael@0: if (field.isJSONField()) { michael@0: measurement.put(field.fieldName, getJSONAtIndex(cursor, 3)); michael@0: return; michael@0: } michael@0: measurement.put(field.fieldName, cursor.getLong(3)); michael@0: } michael@0: michael@0: public static JSONObject getEnvironmentsJSON(Environment currentEnvironment, michael@0: SparseArray envs) throws JSONException { michael@0: JSONObject environments = new JSONObject(); michael@0: michael@0: // Always do this, even if it hasn't recorded anything in the DB. michael@0: environments.put("current", jsonify(currentEnvironment, null)); michael@0: michael@0: String currentHash = currentEnvironment.getHash(); michael@0: for (int i = 0; i < envs.size(); i++) { michael@0: Environment e = envs.valueAt(i); michael@0: if (currentHash.equals(e.getHash())) { michael@0: continue; michael@0: } michael@0: environments.put(e.getHash(), jsonify(e, currentEnvironment)); michael@0: } michael@0: return environments; michael@0: } michael@0: michael@0: public static JSONObject jsonify(Environment e, Environment current) throws JSONException { michael@0: JSONObject age = getProfileAge(e, current); michael@0: JSONObject sysinfo = getSysInfo(e, current); michael@0: JSONObject gecko = getGeckoInfo(e, current); michael@0: JSONObject appinfo = getAppInfo(e, current); michael@0: JSONObject counts = getAddonCounts(e, current); michael@0: michael@0: JSONObject out = new JSONObject(); michael@0: if (age != null) michael@0: out.put("org.mozilla.profile.age", age); michael@0: if (sysinfo != null) michael@0: out.put("org.mozilla.sysinfo.sysinfo", sysinfo); michael@0: if (gecko != null) michael@0: out.put("geckoAppInfo", gecko); michael@0: if (appinfo != null) michael@0: out.put("org.mozilla.appInfo.appinfo", appinfo); michael@0: if (counts != null) michael@0: out.put("org.mozilla.addons.counts", counts); michael@0: michael@0: JSONObject active = getActiveAddons(e, current); michael@0: if (active != null) michael@0: out.put("org.mozilla.addons.active", active); michael@0: michael@0: if (current == null) { michael@0: out.put("hash", e.getHash()); michael@0: } michael@0: return out; michael@0: } michael@0: michael@0: private static JSONObject getProfileAge(Environment e, Environment current) throws JSONException { michael@0: JSONObject age = new JSONObject(); michael@0: int changes = 0; michael@0: if (current == null || current.profileCreation != e.profileCreation) { michael@0: age.put("profileCreation", e.profileCreation); michael@0: changes++; michael@0: } michael@0: if (current != null && changes == 0) { michael@0: return null; michael@0: } michael@0: age.put("_v", 1); michael@0: return age; michael@0: } michael@0: michael@0: private static JSONObject getSysInfo(Environment e, Environment current) throws JSONException { michael@0: JSONObject sysinfo = new JSONObject(); michael@0: int changes = 0; michael@0: if (current == null || current.cpuCount != e.cpuCount) { michael@0: sysinfo.put("cpuCount", e.cpuCount); michael@0: changes++; michael@0: } michael@0: if (current == null || current.memoryMB != e.memoryMB) { michael@0: sysinfo.put("memoryMB", e.memoryMB); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.architecture.equals(e.architecture)) { michael@0: sysinfo.put("architecture", e.architecture); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.sysName.equals(e.sysName)) { michael@0: sysinfo.put("name", e.sysName); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.sysVersion.equals(e.sysVersion)) { michael@0: sysinfo.put("version", e.sysVersion); michael@0: changes++; michael@0: } michael@0: if (current != null && changes == 0) { michael@0: return null; michael@0: } michael@0: sysinfo.put("_v", 1); michael@0: return sysinfo; michael@0: } michael@0: michael@0: private static JSONObject getGeckoInfo(Environment e, Environment current) throws JSONException { michael@0: JSONObject gecko = new JSONObject(); michael@0: int changes = 0; michael@0: if (current == null || !current.vendor.equals(e.vendor)) { michael@0: gecko.put("vendor", e.vendor); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.appName.equals(e.appName)) { michael@0: gecko.put("name", e.appName); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.appID.equals(e.appID)) { michael@0: gecko.put("id", e.appID); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.appVersion.equals(e.appVersion)) { michael@0: gecko.put("version", e.appVersion); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.appBuildID.equals(e.appBuildID)) { michael@0: gecko.put("appBuildID", e.appBuildID); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.platformVersion.equals(e.platformVersion)) { michael@0: gecko.put("platformVersion", e.platformVersion); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.platformBuildID.equals(e.platformBuildID)) { michael@0: gecko.put("platformBuildID", e.platformBuildID); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.os.equals(e.os)) { michael@0: gecko.put("os", e.os); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.xpcomabi.equals(e.xpcomabi)) { michael@0: gecko.put("xpcomabi", e.xpcomabi); michael@0: changes++; michael@0: } michael@0: if (current == null || !current.updateChannel.equals(e.updateChannel)) { michael@0: gecko.put("updateChannel", e.updateChannel); michael@0: changes++; michael@0: } michael@0: if (current != null && changes == 0) { michael@0: return null; michael@0: } michael@0: gecko.put("_v", 1); michael@0: return gecko; michael@0: } michael@0: michael@0: // Null-safe string comparison. michael@0: private static boolean stringsDiffer(final String a, final String b) { michael@0: if (a == null) { michael@0: return b != null; michael@0: } michael@0: return !a.equals(b); michael@0: } michael@0: michael@0: private static JSONObject getAppInfo(Environment e, Environment current) throws JSONException { michael@0: JSONObject appinfo = new JSONObject(); michael@0: michael@0: Logger.debug(LOG_TAG, "Generating appinfo for v" + e.version + " env " + e.hash); michael@0: michael@0: // Is the environment in question newer than the diff target, or is michael@0: // there no diff target? michael@0: final boolean outdated = current == null || michael@0: e.version > current.version; michael@0: michael@0: // Is the environment in question a different version (lower or higher), michael@0: // or is there no diff target? michael@0: final boolean differ = outdated || current.version > e.version; michael@0: michael@0: // Always produce an output object if there's a version mismatch or this michael@0: // isn't a diff. Otherwise, track as we go if there's any difference. michael@0: boolean changed = differ; michael@0: michael@0: switch (e.version) { michael@0: // There's a straightforward correspondence between environment versions michael@0: // and appinfo versions. michael@0: case 2: michael@0: appinfo.put("_v", 3); michael@0: break; michael@0: case 1: michael@0: appinfo.put("_v", 2); michael@0: break; michael@0: default: michael@0: Logger.warn(LOG_TAG, "Unknown environment version: " + e.version); michael@0: return appinfo; michael@0: } michael@0: michael@0: switch (e.version) { michael@0: case 2: michael@0: if (populateAppInfoV2(appinfo, e, current, outdated)) { michael@0: changed = true; michael@0: } michael@0: // Fall through. michael@0: michael@0: case 1: michael@0: // There is no older version than v1, so don't check outdated. michael@0: if (populateAppInfoV1(e, current, appinfo)) { michael@0: changed = true; michael@0: } michael@0: } michael@0: michael@0: if (!changed) { michael@0: return null; michael@0: } michael@0: michael@0: return appinfo; michael@0: } michael@0: michael@0: private static boolean populateAppInfoV1(Environment e, michael@0: Environment current, michael@0: JSONObject appinfo) michael@0: throws JSONException { michael@0: boolean changes = false; michael@0: if (current == null || current.isBlocklistEnabled != e.isBlocklistEnabled) { michael@0: appinfo.put("isBlocklistEnabled", e.isBlocklistEnabled); michael@0: changes = true; michael@0: } michael@0: michael@0: if (current == null || current.isTelemetryEnabled != e.isTelemetryEnabled) { michael@0: appinfo.put("isTelemetryEnabled", e.isTelemetryEnabled); michael@0: changes = true; michael@0: } michael@0: michael@0: return changes; michael@0: } michael@0: michael@0: private static boolean populateAppInfoV2(JSONObject appinfo, michael@0: Environment e, michael@0: Environment current, michael@0: final boolean outdated) michael@0: throws JSONException { michael@0: boolean changes = false; michael@0: if (outdated || michael@0: stringsDiffer(current.osLocale, e.osLocale)) { michael@0: appinfo.put("osLocale", e.osLocale); michael@0: changes = true; michael@0: } michael@0: michael@0: if (outdated || michael@0: stringsDiffer(current.appLocale, e.appLocale)) { michael@0: appinfo.put("appLocale", e.appLocale); michael@0: changes = true; michael@0: } michael@0: michael@0: if (outdated || michael@0: stringsDiffer(current.distribution, e.distribution)) { michael@0: appinfo.put("distribution", e.distribution); michael@0: changes = true; michael@0: } michael@0: michael@0: if (outdated || michael@0: current.acceptLangSet != e.acceptLangSet) { michael@0: appinfo.put("acceptLangIsUserSet", e.acceptLangSet); michael@0: changes = true; michael@0: } michael@0: return changes; michael@0: } michael@0: michael@0: private static JSONObject getAddonCounts(Environment e, Environment current) throws JSONException { michael@0: JSONObject counts = new JSONObject(); michael@0: int changes = 0; michael@0: if (current == null || current.extensionCount != e.extensionCount) { michael@0: counts.put("extension", e.extensionCount); michael@0: changes++; michael@0: } michael@0: if (current == null || current.pluginCount != e.pluginCount) { michael@0: counts.put("plugin", e.pluginCount); michael@0: changes++; michael@0: } michael@0: if (current == null || current.themeCount != e.themeCount) { michael@0: counts.put("theme", e.themeCount); michael@0: changes++; michael@0: } michael@0: if (current != null && changes == 0) { michael@0: return null; michael@0: } michael@0: counts.put("_v", 1); michael@0: return counts; michael@0: } michael@0: michael@0: /** michael@0: * Compute the *tree* difference set between the two objects. If the two michael@0: * objects are identical, returns null. If from is michael@0: * null, returns to. If to is michael@0: * null, behaves as if to were an empty object. michael@0: * michael@0: * (Note that this method does not check for {@link JSONObject#NULL}, because michael@0: * by definition it can't be provided as input to this method.) michael@0: * michael@0: * This behavior is intended to simplify life for callers: a missing object michael@0: * can be viewed as (and behaves as) an empty map, to a useful extent, rather michael@0: * than throwing an exception. michael@0: * michael@0: * @param from michael@0: * a JSONObject. michael@0: * @param to michael@0: * a JSONObject. michael@0: * @param includeNull michael@0: * if true, keys present in from but not in michael@0: * to are included as {@link JSONObject#NULL} in the michael@0: * output. michael@0: * michael@0: * @return a JSONObject, or null if the two objects are identical. michael@0: * @throws JSONException michael@0: * should not occur, but... michael@0: */ michael@0: public static JSONObject diff(JSONObject from, michael@0: JSONObject to, michael@0: boolean includeNull) throws JSONException { michael@0: if (from == null) { michael@0: return to; michael@0: } michael@0: michael@0: if (to == null) { michael@0: return diff(from, new JSONObject(), includeNull); michael@0: } michael@0: michael@0: JSONObject out = new JSONObject(); michael@0: michael@0: HashSet toKeys = includeNull ? new HashSet(to.length()) michael@0: : null; michael@0: michael@0: @SuppressWarnings("unchecked") michael@0: Iterator it = to.keys(); michael@0: while (it.hasNext()) { michael@0: String key = it.next(); michael@0: michael@0: // Track these as we go if we'll need them later. michael@0: if (includeNull) { michael@0: toKeys.add(key); michael@0: } michael@0: michael@0: Object value = to.get(key); michael@0: if (!from.has(key)) { michael@0: // It must be new. michael@0: out.put(key, value); michael@0: continue; michael@0: } michael@0: michael@0: // Not new? Then see if it changed. michael@0: Object old = from.get(key); michael@0: michael@0: // Two JSONObjects should be diffed. michael@0: if (old instanceof JSONObject && value instanceof JSONObject) { michael@0: JSONObject innerDiff = diff(((JSONObject) old), ((JSONObject) value), michael@0: includeNull); michael@0: // No change? No output. michael@0: if (innerDiff == null) { michael@0: continue; michael@0: } michael@0: michael@0: // Otherwise include the diff. michael@0: out.put(key, innerDiff); michael@0: continue; michael@0: } michael@0: michael@0: // A regular value, or a type change. Only skip if they're the same. michael@0: if (value.equals(old)) { michael@0: continue; michael@0: } michael@0: out.put(key, value); michael@0: } michael@0: michael@0: // Now -- if requested -- include any removed keys. michael@0: if (includeNull) { michael@0: Set fromKeys = HealthReportUtils.keySet(from); michael@0: fromKeys.removeAll(toKeys); michael@0: for (String notPresent : fromKeys) { michael@0: out.put(notPresent, JSONObject.NULL); michael@0: } michael@0: } michael@0: michael@0: if (out.length() == 0) { michael@0: return null; michael@0: } michael@0: return out; michael@0: } michael@0: michael@0: private static JSONObject getActiveAddons(Environment e, Environment current) throws JSONException { michael@0: // Just return the current add-on set, with a version annotation. michael@0: // To do so requires copying. michael@0: if (current == null) { michael@0: JSONObject out = e.getNonIgnoredAddons(); michael@0: if (out == null) { michael@0: Logger.warn(LOG_TAG, "Null add-ons to return in FHR document. Returning {}."); michael@0: out = new JSONObject(); // So that we always return something. michael@0: } michael@0: out.put("_v", 1); michael@0: return out; michael@0: } michael@0: michael@0: // Otherwise, return the diff. michael@0: JSONObject diff = diff(current.getNonIgnoredAddons(), e.getNonIgnoredAddons(), true); michael@0: if (diff == null) { michael@0: return null; michael@0: } michael@0: if (diff == e.addons) { michael@0: // Again, needs to copy. michael@0: return getActiveAddons(e, null); michael@0: } michael@0: michael@0: diff.put("_v", 1); michael@0: return diff; michael@0: } michael@0: }