1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/mobile/android/base/GeckoAccessibility.java Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,395 @@
1.4 +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
1.5 + * This Source Code Form is subject to the terms of the Mozilla Public
1.6 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.8 +
1.9 +package org.mozilla.gecko;
1.10 +
1.11 +import org.mozilla.gecko.gfx.LayerView;
1.12 +import org.mozilla.gecko.util.ThreadUtils;
1.13 +import org.mozilla.gecko.util.UiAsyncTask;
1.14 +
1.15 +import org.json.JSONArray;
1.16 +import org.json.JSONException;
1.17 +import org.json.JSONObject;
1.18 +
1.19 +import android.app.ActivityManager;
1.20 +import android.app.ActivityManager.RunningServiceInfo;
1.21 +import android.content.Context;
1.22 +import android.graphics.Rect;
1.23 +import android.os.Build;
1.24 +import android.os.Bundle;
1.25 +import android.util.Log;
1.26 +import android.view.View;
1.27 +import android.view.accessibility.AccessibilityEvent;
1.28 +import android.view.accessibility.AccessibilityManager;
1.29 +import android.view.accessibility.AccessibilityNodeInfo;
1.30 +import android.view.accessibility.AccessibilityNodeProvider;
1.31 +
1.32 +import com.googlecode.eyesfree.braille.selfbraille.SelfBrailleClient;
1.33 +import com.googlecode.eyesfree.braille.selfbraille.WriteData;
1.34 +
1.35 +import java.util.Arrays;
1.36 +import java.util.HashSet;
1.37 +import java.util.List;
1.38 +
1.39 +public class GeckoAccessibility {
1.40 + private static final String LOGTAG = "GeckoAccessibility";
1.41 + private static final int VIRTUAL_CURSOR_PREVIOUS = 1;
1.42 + private static final int VIRTUAL_CURSOR_POSITION = 2;
1.43 + private static final int VIRTUAL_CURSOR_NEXT = 3;
1.44 +
1.45 + private static boolean sEnabled = false;
1.46 + // Used to store the JSON message and populate the event later in the code path.
1.47 + private static JSONObject sEventMessage = null;
1.48 + private static AccessibilityNodeInfo sVirtualCursorNode = null;
1.49 +
1.50 + // This is the number Brailleback uses to start indexing routing keys.
1.51 + private static final int BRAILLE_CLICK_BASE_INDEX = -275000000;
1.52 + private static SelfBrailleClient sSelfBrailleClient = null;
1.53 +
1.54 + private static final HashSet<String> sServiceWhitelist =
1.55 + new HashSet<String>(Arrays.asList(new String[] {
1.56 + "com.google.android.marvin.talkback.TalkBackService", // Google Talkback screen reader
1.57 + "com.mot.readout.ScreenReader", // Motorola screen reader
1.58 + "info.spielproject.spiel.SpielService", // Spiel screen reader
1.59 + "es.codefactory.android.app.ma.MAAccessibilityService" // Codefactory Mobile Accessibility screen reader
1.60 + }));
1.61 +
1.62 + public static void updateAccessibilitySettings (final GeckoApp app) {
1.63 + new UiAsyncTask<Void, Void, Void>(ThreadUtils.getBackgroundHandler()) {
1.64 + @Override
1.65 + public Void doInBackground(Void... args) {
1.66 + JSONObject ret = new JSONObject();
1.67 + sEnabled = false;
1.68 + AccessibilityManager accessibilityManager =
1.69 + (AccessibilityManager) app.getSystemService(Context.ACCESSIBILITY_SERVICE);
1.70 + if (accessibilityManager.isEnabled()) {
1.71 + ActivityManager activityManager =
1.72 + (ActivityManager) app.getSystemService(Context.ACTIVITY_SERVICE);
1.73 + List<RunningServiceInfo> runningServices = activityManager.getRunningServices(Integer.MAX_VALUE);
1.74 +
1.75 + for (RunningServiceInfo runningServiceInfo : runningServices) {
1.76 + sEnabled = sServiceWhitelist.contains(runningServiceInfo.service.getClassName());
1.77 + if (sEnabled)
1.78 + break;
1.79 + }
1.80 + if (sEnabled && sSelfBrailleClient == null &&
1.81 + Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) {
1.82 + sSelfBrailleClient = new SelfBrailleClient(GeckoAppShell.getContext(), false);
1.83 + }
1.84 + }
1.85 +
1.86 + try {
1.87 + ret.put("enabled", sEnabled);
1.88 + } catch (Exception ex) {
1.89 + Log.e(LOGTAG, "Error building JSON arguments for Accessibility:Settings:", ex);
1.90 + }
1.91 +
1.92 + GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:Settings",
1.93 + ret.toString()));
1.94 + return null;
1.95 + }
1.96 +
1.97 + @Override
1.98 + public void onPostExecute(Void args) {
1.99 + // Disable the dynamic toolbar when enabling accessibility.
1.100 + // These features tend not to interact well.
1.101 + app.setAccessibilityEnabled(sEnabled);
1.102 + }
1.103 + }.execute();
1.104 + }
1.105 +
1.106 + private static void populateEventFromJSON (AccessibilityEvent event, JSONObject message) {
1.107 + final JSONArray textArray = message.optJSONArray("text");
1.108 + if (textArray != null) {
1.109 + for (int i = 0; i < textArray.length(); i++)
1.110 + event.getText().add(textArray.optString(i));
1.111 + }
1.112 +
1.113 + event.setContentDescription(message.optString("description"));
1.114 + event.setEnabled(message.optBoolean("enabled", true));
1.115 + event.setChecked(message.optBoolean("checked"));
1.116 + event.setPassword(message.optBoolean("password"));
1.117 + event.setAddedCount(message.optInt("addedCount", -1));
1.118 + event.setRemovedCount(message.optInt("removedCount", -1));
1.119 + event.setFromIndex(message.optInt("fromIndex", -1));
1.120 + event.setItemCount(message.optInt("itemCount", -1));
1.121 + event.setCurrentItemIndex(message.optInt("currentItemIndex", -1));
1.122 + event.setBeforeText(message.optString("beforeText"));
1.123 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
1.124 + event.setToIndex(message.optInt("toIndex", -1));
1.125 + event.setScrollable(message.optBoolean("scrollable"));
1.126 + event.setScrollX(message.optInt("scrollX", -1));
1.127 + event.setScrollY(message.optInt("scrollY", -1));
1.128 + }
1.129 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1) {
1.130 + event.setMaxScrollX(message.optInt("maxScrollX", -1));
1.131 + event.setMaxScrollY(message.optInt("maxScrollY", -1));
1.132 + }
1.133 + }
1.134 +
1.135 + private static void sendDirectAccessibilityEvent(int eventType, JSONObject message) {
1.136 + final AccessibilityEvent accEvent = AccessibilityEvent.obtain(eventType);
1.137 + accEvent.setClassName(GeckoAccessibility.class.getName());
1.138 + accEvent.setPackageName(GeckoAppShell.getContext().getPackageName());
1.139 + populateEventFromJSON(accEvent, message);
1.140 + AccessibilityManager accessibilityManager =
1.141 + (AccessibilityManager) GeckoAppShell.getContext().getSystemService(Context.ACCESSIBILITY_SERVICE);
1.142 + try {
1.143 + accessibilityManager.sendAccessibilityEvent(accEvent);
1.144 + } catch (IllegalStateException e) {
1.145 + // Accessibility is off.
1.146 + }
1.147 + }
1.148 +
1.149 + public static void sendAccessibilityEvent (final JSONObject message) {
1.150 + if (!sEnabled)
1.151 + return;
1.152 +
1.153 + final int eventType = message.optInt("eventType", -1);
1.154 + if (eventType < 0) {
1.155 + Log.e(LOGTAG, "No accessibility event type provided");
1.156 + return;
1.157 + }
1.158 +
1.159 + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) {
1.160 + // Before Jelly Bean we send events directly from here while spoofing the source by setting
1.161 + // the package and class name manually.
1.162 + ThreadUtils.postToBackgroundThread(new Runnable() {
1.163 + @Override
1.164 + public void run() {
1.165 + sendDirectAccessibilityEvent(eventType, message);
1.166 + }
1.167 + });
1.168 + } else {
1.169 + // In Jelly Bean we populate an AccessibilityNodeInfo with the minimal amount of data to have
1.170 + // it work with TalkBack.
1.171 + final LayerView view = GeckoAppShell.getLayerView();
1.172 + if (view == null)
1.173 + return;
1.174 +
1.175 + if (sVirtualCursorNode == null)
1.176 + sVirtualCursorNode = AccessibilityNodeInfo.obtain(view, VIRTUAL_CURSOR_POSITION);
1.177 + sVirtualCursorNode.setEnabled(message.optBoolean("enabled", true));
1.178 + sVirtualCursorNode.setClickable(message.optBoolean("clickable"));
1.179 + sVirtualCursorNode.setCheckable(message.optBoolean("checkable"));
1.180 + sVirtualCursorNode.setChecked(message.optBoolean("checked"));
1.181 + sVirtualCursorNode.setPassword(message.optBoolean("password"));
1.182 +
1.183 + final JSONArray textArray = message.optJSONArray("text");
1.184 + StringBuilder sb = new StringBuilder();
1.185 + if (textArray != null && textArray.length() > 0) {
1.186 + sb.append(textArray.optString(0));
1.187 + for (int i = 1; i < textArray.length(); i++) {
1.188 + sb.append(" ").append(textArray.optString(i));
1.189 + }
1.190 + }
1.191 + sVirtualCursorNode.setText(sb.toString());
1.192 + sVirtualCursorNode.setContentDescription(message.optString("description"));
1.193 +
1.194 + JSONObject bounds = message.optJSONObject("bounds");
1.195 + if (bounds != null) {
1.196 + Rect relativeBounds = new Rect(bounds.optInt("left"), bounds.optInt("top"),
1.197 + bounds.optInt("right"), bounds.optInt("bottom"));
1.198 + sVirtualCursorNode.setBoundsInParent(relativeBounds);
1.199 + int[] locationOnScreen = new int[2];
1.200 + view.getLocationOnScreen(locationOnScreen);
1.201 + Rect screenBounds = new Rect(relativeBounds);
1.202 + screenBounds.offset(locationOnScreen[0], locationOnScreen[1]);
1.203 + sVirtualCursorNode.setBoundsInScreen(screenBounds);
1.204 + }
1.205 +
1.206 + final JSONObject braille = message.optJSONObject("brailleOutput");
1.207 + if (braille != null) {
1.208 + sendBrailleText(view, braille.optString("text"),
1.209 + braille.optInt("selectionStart"), braille.optInt("selectionEnd"));
1.210 + }
1.211 +
1.212 + ThreadUtils.postToUiThread(new Runnable() {
1.213 + @Override
1.214 + public void run() {
1.215 + // If this is an accessibility focus, a lot of internal voodoo happens so we perform an
1.216 + // accessibility focus action on the view, and it in turn sends the right events.
1.217 + switch (eventType) {
1.218 + case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED:
1.219 + sEventMessage = message;
1.220 + view.performAccessibilityAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS, null);
1.221 + break;
1.222 + case AccessibilityEvent.TYPE_ANNOUNCEMENT:
1.223 + case AccessibilityEvent.TYPE_VIEW_SCROLLED:
1.224 + sEventMessage = null;
1.225 + final AccessibilityEvent accEvent = AccessibilityEvent.obtain(eventType);
1.226 + view.onInitializeAccessibilityEvent(accEvent);
1.227 + populateEventFromJSON(accEvent, message);
1.228 + view.getParent().requestSendAccessibilityEvent(view, accEvent);
1.229 + break;
1.230 + default:
1.231 + sEventMessage = message;
1.232 + view.sendAccessibilityEvent(eventType);
1.233 + break;
1.234 + }
1.235 + }
1.236 + });
1.237 +
1.238 + }
1.239 + }
1.240 +
1.241 + private static void sendBrailleText(final View view, final String text, final int selectionStart, final int selectionEnd) {
1.242 + AccessibilityNodeInfo info = AccessibilityNodeInfo.obtain(view, VIRTUAL_CURSOR_POSITION);
1.243 + WriteData data = WriteData.forInfo(info);
1.244 + data.setText(text);
1.245 + // Set either the focus blink or the current caret position/selection
1.246 + data.setSelectionStart(selectionStart);
1.247 + data.setSelectionEnd(selectionEnd);
1.248 + sSelfBrailleClient.write(data);
1.249 + }
1.250 +
1.251 + public static void setDelegate(LayerView layerview) {
1.252 + // Only use this delegate in Jelly Bean.
1.253 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) {
1.254 + layerview.setAccessibilityDelegate(new GeckoAccessibilityDelegate());
1.255 + layerview.setImportantForAccessibility(View.IMPORTANT_FOR_ACCESSIBILITY_YES);
1.256 + }
1.257 + }
1.258 +
1.259 + public static void onLayerViewFocusChanged(LayerView layerview, boolean gainFocus) {
1.260 + if (sEnabled)
1.261 + GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:Focus",
1.262 + gainFocus ? "true" : "false"));
1.263 + }
1.264 +
1.265 + public static class GeckoAccessibilityDelegate extends View.AccessibilityDelegate {
1.266 + AccessibilityNodeProvider mAccessibilityNodeProvider;
1.267 +
1.268 + @Override
1.269 + public void onPopulateAccessibilityEvent (View host, AccessibilityEvent event) {
1.270 + super.onPopulateAccessibilityEvent(host, event);
1.271 + if (sEventMessage != null) {
1.272 + populateEventFromJSON(event, sEventMessage);
1.273 + // No matter where the a11y focus is requested, we always force it back to the current vc position.
1.274 + event.setSource(host, VIRTUAL_CURSOR_POSITION);
1.275 + }
1.276 + // We save the hover enter event so that we could reuse it for a subsequent accessibility focus event.
1.277 + if (event.getEventType() != AccessibilityEvent.TYPE_VIEW_HOVER_ENTER)
1.278 + sEventMessage = null;
1.279 + }
1.280 +
1.281 + @Override
1.282 + public AccessibilityNodeProvider getAccessibilityNodeProvider(final View host) {
1.283 + if (mAccessibilityNodeProvider == null)
1.284 + // The accessibility node structure for web content consists of 3 LayerView child nodes:
1.285 + // 1. VIRTUAL_CURSOR_PREVIOUS: Represents the virtual cursor position that is previous to the
1.286 + // current one.
1.287 + // 2. VIRTUAL_CURSOR_POSITION: Represents the current position of the virtual cursor.
1.288 + // 3. VIRTUAL_CURSOR_NEXT: Represents the next virtual cursor position.
1.289 + mAccessibilityNodeProvider = new AccessibilityNodeProvider() {
1.290 + @Override
1.291 + public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualDescendantId) {
1.292 + AccessibilityNodeInfo info = (virtualDescendantId == VIRTUAL_CURSOR_POSITION && sVirtualCursorNode != null) ?
1.293 + AccessibilityNodeInfo.obtain(sVirtualCursorNode) :
1.294 + AccessibilityNodeInfo.obtain(host, virtualDescendantId);
1.295 +
1.296 + switch (virtualDescendantId) {
1.297 + case View.NO_ID:
1.298 + // This is the parent LayerView node, populate it with children.
1.299 + onInitializeAccessibilityNodeInfo(host, info);
1.300 + info.addChild(host, VIRTUAL_CURSOR_PREVIOUS);
1.301 + info.addChild(host, VIRTUAL_CURSOR_POSITION);
1.302 + info.addChild(host, VIRTUAL_CURSOR_NEXT);
1.303 + break;
1.304 + default:
1.305 + info.setParent(host);
1.306 + info.setSource(host, virtualDescendantId);
1.307 + info.setVisibleToUser(host.isShown());
1.308 + info.setPackageName(GeckoAppShell.getContext().getPackageName());
1.309 + info.setClassName(host.getClass().getName());
1.310 + info.addAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
1.311 + info.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS);
1.312 + info.addAction(AccessibilityNodeInfo.ACTION_CLICK);
1.313 + info.addAction(AccessibilityNodeInfo.ACTION_LONG_CLICK);
1.314 + info.addAction(AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY);
1.315 + info.addAction(AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY);
1.316 + info.setMovementGranularities(AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER |
1.317 + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD |
1.318 + AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH);
1.319 + break;
1.320 + }
1.321 + return info;
1.322 + }
1.323 +
1.324 + @Override
1.325 + public boolean performAction (int virtualViewId, int action, Bundle arguments) {
1.326 + if (action == AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS) {
1.327 + // The accessibility focus is permanently on the middle node, VIRTUAL_CURSOR_POSITION.
1.328 + // When accessibility focus is requested on one of its siblings we move the virtual cursor
1.329 + // either forward or backward depending on which sibling was selected.
1.330 +
1.331 + switch (virtualViewId) {
1.332 + case VIRTUAL_CURSOR_PREVIOUS:
1.333 + GeckoAppShell.
1.334 + sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:PreviousObject", null));
1.335 + return true;
1.336 + case VIRTUAL_CURSOR_NEXT:
1.337 + GeckoAppShell.
1.338 + sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:NextObject", null));
1.339 + return true;
1.340 + default:
1.341 + break;
1.342 + }
1.343 + } else if (action == AccessibilityNodeInfo.ACTION_CLICK && virtualViewId == VIRTUAL_CURSOR_POSITION) {
1.344 + GeckoAppShell.
1.345 + sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:ActivateObject", null));
1.346 + return true;
1.347 + } else if (action == AccessibilityNodeInfo.ACTION_LONG_CLICK && virtualViewId == VIRTUAL_CURSOR_POSITION) {
1.348 + GeckoAppShell.
1.349 + sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:LongPress", null));
1.350 + return true;
1.351 + } else if (action == AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY &&
1.352 + virtualViewId == VIRTUAL_CURSOR_POSITION) {
1.353 + // XXX: Self brailling gives this action with a bogus argument instead of an actual click action;
1.354 + // the argument value is the BRAILLE_CLICK_BASE_INDEX - the index of the routing key that was hit
1.355 + int granularity = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT);
1.356 + if (granularity < 0) {
1.357 + int keyIndex = BRAILLE_CLICK_BASE_INDEX - granularity;
1.358 + JSONObject activationData = new JSONObject();
1.359 + try {
1.360 + activationData.put("keyIndex", keyIndex);
1.361 + } catch (JSONException e) {
1.362 + return true;
1.363 + }
1.364 + GeckoAppShell.
1.365 + sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:ActivateObject", activationData.toString()));
1.366 + } else {
1.367 + JSONObject movementData = new JSONObject();
1.368 + try {
1.369 + movementData.put("direction", "Next");
1.370 + movementData.put("granularity", granularity);
1.371 + } catch (JSONException e) {
1.372 + return true;
1.373 + }
1.374 + GeckoAppShell.
1.375 + sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:MoveByGranularity", movementData.toString()));
1.376 + }
1.377 + return true;
1.378 + } else if (action == AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY &&
1.379 + virtualViewId == VIRTUAL_CURSOR_POSITION) {
1.380 + JSONObject movementData = new JSONObject();
1.381 + try {
1.382 + movementData.put("direction", "Previous");
1.383 + movementData.put("granularity", arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT));
1.384 + } catch (JSONException e) {
1.385 + return true;
1.386 + }
1.387 + GeckoAppShell.
1.388 + sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:MoveByGranularity", movementData.toString()));
1.389 + return true;
1.390 + }
1.391 + return host.performAccessibilityAction(action, arguments);
1.392 + }
1.393 + };
1.394 +
1.395 + return mAccessibilityNodeProvider;
1.396 + }
1.397 + }
1.398 +}
