• file
• revisions
• annotate
• diff
• comparison
• raw

mobile/android/base/GeckoAccessibility.java@b8a032363ba2 (annotated)

mobile/android/base/GeckoAccessibility.java

Thu, 22 Jan 2015 13:21:57 +0100

author

Michael Schloh von Bennewitz <michael@schloh.com>

date

Thu, 22 Jan 2015 13:21:57 +0100

branch

TOR_BUG_9701

changeset 15

b8a032363ba2

permissions

-rw-r--r--

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

michael@0	1	/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
michael@0	2	* This Source Code Form is subject to the terms of the Mozilla Public
michael@0	3	* License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0	4	* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
michael@0	5
michael@0	6	package org.mozilla.gecko;
michael@0	7
michael@0	8	import org.mozilla.gecko.gfx.LayerView;
michael@0	9	import org.mozilla.gecko.util.ThreadUtils;
michael@0	10	import org.mozilla.gecko.util.UiAsyncTask;
michael@0	11
michael@0	12	import org.json.JSONArray;
michael@0	13	import org.json.JSONException;
michael@0	14	import org.json.JSONObject;
michael@0	15
michael@0	16	import android.app.ActivityManager;
michael@0	17	import android.app.ActivityManager.RunningServiceInfo;
michael@0	18	import android.content.Context;
michael@0	19	import android.graphics.Rect;
michael@0	20	import android.os.Build;
michael@0	21	import android.os.Bundle;
michael@0	22	import android.util.Log;
michael@0	23	import android.view.View;
michael@0	24	import android.view.accessibility.AccessibilityEvent;
michael@0	25	import android.view.accessibility.AccessibilityManager;
michael@0	26	import android.view.accessibility.AccessibilityNodeInfo;
michael@0	27	import android.view.accessibility.AccessibilityNodeProvider;
michael@0	28
michael@0	29	import com.googlecode.eyesfree.braille.selfbraille.SelfBrailleClient;
michael@0	30	import com.googlecode.eyesfree.braille.selfbraille.WriteData;
michael@0	31
michael@0	32	import java.util.Arrays;
michael@0	33	import java.util.HashSet;
michael@0	34	import java.util.List;
michael@0	35
michael@0	36	public class GeckoAccessibility {
michael@0	37	private static final String LOGTAG = "GeckoAccessibility";
michael@0	38	private static final int VIRTUAL_CURSOR_PREVIOUS = 1;
michael@0	39	private static final int VIRTUAL_CURSOR_POSITION = 2;
michael@0	40	private static final int VIRTUAL_CURSOR_NEXT = 3;
michael@0	41
michael@0	42	private static boolean sEnabled = false;
michael@0	43	// Used to store the JSON message and populate the event later in the code path.
michael@0	44	private static JSONObject sEventMessage = null;
michael@0	45	private static AccessibilityNodeInfo sVirtualCursorNode = null;
michael@0	46
michael@0	47	// This is the number Brailleback uses to start indexing routing keys.
michael@0	48	private static final int BRAILLE_CLICK_BASE_INDEX = -275000000;
michael@0	49	private static SelfBrailleClient sSelfBrailleClient = null;
michael@0	50
michael@0	51	private static final HashSet<String> sServiceWhitelist =
michael@0	52	new HashSet<String>(Arrays.asList(new String[] {
michael@0	53	"com.google.android.marvin.talkback.TalkBackService", // Google Talkback screen reader
michael@0	54	"com.mot.readout.ScreenReader", // Motorola screen reader
michael@0	55	"info.spielproject.spiel.SpielService", // Spiel screen reader
michael@0	56	"es.codefactory.android.app.ma.MAAccessibilityService" // Codefactory Mobile Accessibility screen reader
michael@0	57	}));
michael@0	58
michael@0	59	public static void updateAccessibilitySettings (final GeckoApp app) {
michael@0	60	new UiAsyncTask<Void, Void, Void>(ThreadUtils.getBackgroundHandler()) {
michael@0	61	@Override
michael@0	62	public Void doInBackground(Void... args) {
michael@0	63	JSONObject ret = new JSONObject();
michael@0	64	sEnabled = false;
michael@0	65	AccessibilityManager accessibilityManager =
michael@0	66	(AccessibilityManager) app.getSystemService(Context.ACCESSIBILITY_SERVICE);
michael@0	67	if (accessibilityManager.isEnabled()) {
michael@0	68	ActivityManager activityManager =
michael@0	69	(ActivityManager) app.getSystemService(Context.ACTIVITY_SERVICE);
michael@0	70	List<RunningServiceInfo> runningServices = activityManager.getRunningServices(Integer.MAX_VALUE);
michael@0	71
michael@0	72	for (RunningServiceInfo runningServiceInfo : runningServices) {
michael@0	73	sEnabled = sServiceWhitelist.contains(runningServiceInfo.service.getClassName());
michael@0	74	if (sEnabled)
michael@0	75	break;
michael@0	76	}
michael@0	77	if (sEnabled && sSelfBrailleClient == null &&
michael@0	78	Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) {
michael@0	79	sSelfBrailleClient = new SelfBrailleClient(GeckoAppShell.getContext(), false);
michael@0	80	}
michael@0	81	}
michael@0	82
michael@0	83	try {
michael@0	84	ret.put("enabled", sEnabled);
michael@0	85	} catch (Exception ex) {
michael@0	86	Log.e(LOGTAG, "Error building JSON arguments for Accessibility:Settings:", ex);
michael@0	87	}
michael@0	88
michael@0	89	GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:Settings",
michael@0	90	ret.toString()));
michael@0	91	return null;
michael@0	92	}
michael@0	93
michael@0	94	@Override
michael@0	95	public void onPostExecute(Void args) {
michael@0	96	// Disable the dynamic toolbar when enabling accessibility.
michael@0	97	// These features tend not to interact well.
michael@0	98	app.setAccessibilityEnabled(sEnabled);
michael@0	99	}
michael@0	100	}.execute();
michael@0	101	}
michael@0	102
michael@0	103	private static void populateEventFromJSON (AccessibilityEvent event, JSONObject message) {
michael@0	104	final JSONArray textArray = message.optJSONArray("text");
michael@0	105	if (textArray != null) {
michael@0	106	for (int i = 0; i < textArray.length(); i++)
michael@0	107	event.getText().add(textArray.optString(i));
michael@0	108	}
michael@0	109
michael@0	110	event.setContentDescription(message.optString("description"));
michael@0	111	event.setEnabled(message.optBoolean("enabled", true));
michael@0	112	event.setChecked(message.optBoolean("checked"));
michael@0	113	event.setPassword(message.optBoolean("password"));
michael@0	114	event.setAddedCount(message.optInt("addedCount", -1));
michael@0	115	event.setRemovedCount(message.optInt("removedCount", -1));
michael@0	116	event.setFromIndex(message.optInt("fromIndex", -1));
michael@0	117	event.setItemCount(message.optInt("itemCount", -1));
michael@0	118	event.setCurrentItemIndex(message.optInt("currentItemIndex", -1));
michael@0	119	event.setBeforeText(message.optString("beforeText"));
michael@0	120	if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
michael@0	121	event.setToIndex(message.optInt("toIndex", -1));
michael@0	122	event.setScrollable(message.optBoolean("scrollable"));
michael@0	123	event.setScrollX(message.optInt("scrollX", -1));
michael@0	124	event.setScrollY(message.optInt("scrollY", -1));
michael@0	125	}
michael@0	126	if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1) {
michael@0	127	event.setMaxScrollX(message.optInt("maxScrollX", -1));
michael@0	128	event.setMaxScrollY(message.optInt("maxScrollY", -1));
michael@0	129	}
michael@0	130	}
michael@0	131
michael@0	132	private static void sendDirectAccessibilityEvent(int eventType, JSONObject message) {
michael@0	133	final AccessibilityEvent accEvent = AccessibilityEvent.obtain(eventType);
michael@0	134	accEvent.setClassName(GeckoAccessibility.class.getName());
michael@0	135	accEvent.setPackageName(GeckoAppShell.getContext().getPackageName());
michael@0	136	populateEventFromJSON(accEvent, message);
michael@0	137	AccessibilityManager accessibilityManager =
michael@0	138	(AccessibilityManager) GeckoAppShell.getContext().getSystemService(Context.ACCESSIBILITY_SERVICE);
michael@0	139	try {
michael@0	140	accessibilityManager.sendAccessibilityEvent(accEvent);
michael@0	141	} catch (IllegalStateException e) {
michael@0	142	// Accessibility is off.
michael@0	143	}
michael@0	144	}
michael@0	145
michael@0	146	public static void sendAccessibilityEvent (final JSONObject message) {
michael@0	147	if (!sEnabled)
michael@0	148	return;
michael@0	149
michael@0	150	final int eventType = message.optInt("eventType", -1);
michael@0	151	if (eventType < 0) {
michael@0	152	Log.e(LOGTAG, "No accessibility event type provided");
michael@0	153	return;
michael@0	154	}
michael@0	155
michael@0	156	if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) {
michael@0	157	// Before Jelly Bean we send events directly from here while spoofing the source by setting
michael@0	158	// the package and class name manually.
michael@0	159	ThreadUtils.postToBackgroundThread(new Runnable() {
michael@0	160	@Override
michael@0	161	public void run() {
michael@0	162	sendDirectAccessibilityEvent(eventType, message);
michael@0	163	}
michael@0	164	});
michael@0	165	} else {
michael@0	166	// In Jelly Bean we populate an AccessibilityNodeInfo with the minimal amount of data to have
michael@0	167	// it work with TalkBack.
michael@0	168	final LayerView view = GeckoAppShell.getLayerView();
michael@0	169	if (view == null)
michael@0	170	return;
michael@0	171
michael@0	172	if (sVirtualCursorNode == null)
michael@0	173	sVirtualCursorNode = AccessibilityNodeInfo.obtain(view, VIRTUAL_CURSOR_POSITION);
michael@0	174	sVirtualCursorNode.setEnabled(message.optBoolean("enabled", true));
michael@0	175	sVirtualCursorNode.setClickable(message.optBoolean("clickable"));
michael@0	176	sVirtualCursorNode.setCheckable(message.optBoolean("checkable"));
michael@0	177	sVirtualCursorNode.setChecked(message.optBoolean("checked"));
michael@0	178	sVirtualCursorNode.setPassword(message.optBoolean("password"));
michael@0	179
michael@0	180	final JSONArray textArray = message.optJSONArray("text");
michael@0	181	StringBuilder sb = new StringBuilder();
michael@0	182	if (textArray != null && textArray.length() > 0) {
michael@0	183	sb.append(textArray.optString(0));
michael@0	184	for (int i = 1; i < textArray.length(); i++) {
michael@0	185	sb.append(" ").append(textArray.optString(i));
michael@0	186	}
michael@0	187	}
michael@0	188	sVirtualCursorNode.setText(sb.toString());
michael@0	189	sVirtualCursorNode.setContentDescription(message.optString("description"));
michael@0	190
michael@0	191	JSONObject bounds = message.optJSONObject("bounds");
michael@0	192	if (bounds != null) {
michael@0	193	Rect relativeBounds = new Rect(bounds.optInt("left"), bounds.optInt("top"),
michael@0	194	bounds.optInt("right"), bounds.optInt("bottom"));
michael@0	195	sVirtualCursorNode.setBoundsInParent(relativeBounds);
michael@0	196	int[] locationOnScreen = new int[2];
michael@0	197	view.getLocationOnScreen(locationOnScreen);
michael@0	198	Rect screenBounds = new Rect(relativeBounds);
michael@0	199	screenBounds.offset(locationOnScreen[0], locationOnScreen[1]);
michael@0	200	sVirtualCursorNode.setBoundsInScreen(screenBounds);
michael@0	201	}
michael@0	202
michael@0	203	final JSONObject braille = message.optJSONObject("brailleOutput");
michael@0	204	if (braille != null) {
michael@0	205	sendBrailleText(view, braille.optString("text"),
michael@0	206	braille.optInt("selectionStart"), braille.optInt("selectionEnd"));
michael@0	207	}
michael@0	208
michael@0	209	ThreadUtils.postToUiThread(new Runnable() {
michael@0	210	@Override
michael@0	211	public void run() {
michael@0	212	// If this is an accessibility focus, a lot of internal voodoo happens so we perform an
michael@0	213	// accessibility focus action on the view, and it in turn sends the right events.
michael@0	214	switch (eventType) {
michael@0	215	case AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED:
michael@0	216	sEventMessage = message;
michael@0	217	view.performAccessibilityAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS, null);
michael@0	218	break;
michael@0	219	case AccessibilityEvent.TYPE_ANNOUNCEMENT:
michael@0	220	case AccessibilityEvent.TYPE_VIEW_SCROLLED:
michael@0	221	sEventMessage = null;
michael@0	222	final AccessibilityEvent accEvent = AccessibilityEvent.obtain(eventType);
michael@0	223	view.onInitializeAccessibilityEvent(accEvent);
michael@0	224	populateEventFromJSON(accEvent, message);
michael@0	225	view.getParent().requestSendAccessibilityEvent(view, accEvent);
michael@0	226	break;
michael@0	227	default:
michael@0	228	sEventMessage = message;
michael@0	229	view.sendAccessibilityEvent(eventType);
michael@0	230	break;
michael@0	231	}
michael@0	232	}
michael@0	233	});
michael@0	234
michael@0	235	}
michael@0	236	}
michael@0	237
michael@0	238	private static void sendBrailleText(final View view, final String text, final int selectionStart, final int selectionEnd) {
michael@0	239	AccessibilityNodeInfo info = AccessibilityNodeInfo.obtain(view, VIRTUAL_CURSOR_POSITION);
michael@0	240	WriteData data = WriteData.forInfo(info);
michael@0	241	data.setText(text);
michael@0	242	// Set either the focus blink or the current caret position/selection
michael@0	243	data.setSelectionStart(selectionStart);
michael@0	244	data.setSelectionEnd(selectionEnd);
michael@0	245	sSelfBrailleClient.write(data);
michael@0	246	}
michael@0	247
michael@0	248	public static void setDelegate(LayerView layerview) {
michael@0	249	// Only use this delegate in Jelly Bean.
michael@0	250	if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) {
michael@0	251	layerview.setAccessibilityDelegate(new GeckoAccessibilityDelegate());
michael@0	252	layerview.setImportantForAccessibility(View.IMPORTANT_FOR_ACCESSIBILITY_YES);
michael@0	253	}
michael@0	254	}
michael@0	255
michael@0	256	public static void onLayerViewFocusChanged(LayerView layerview, boolean gainFocus) {
michael@0	257	if (sEnabled)
michael@0	258	GeckoAppShell.sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:Focus",
michael@0	259	gainFocus ? "true" : "false"));
michael@0	260	}
michael@0	261
michael@0	262	public static class GeckoAccessibilityDelegate extends View.AccessibilityDelegate {
michael@0	263	AccessibilityNodeProvider mAccessibilityNodeProvider;
michael@0	264
michael@0	265	@Override
michael@0	266	public void onPopulateAccessibilityEvent (View host, AccessibilityEvent event) {
michael@0	267	super.onPopulateAccessibilityEvent(host, event);
michael@0	268	if (sEventMessage != null) {
michael@0	269	populateEventFromJSON(event, sEventMessage);
michael@0	270	// No matter where the a11y focus is requested, we always force it back to the current vc position.
michael@0	271	event.setSource(host, VIRTUAL_CURSOR_POSITION);
michael@0	272	}
michael@0	273	// We save the hover enter event so that we could reuse it for a subsequent accessibility focus event.
michael@0	274	if (event.getEventType() != AccessibilityEvent.TYPE_VIEW_HOVER_ENTER)
michael@0	275	sEventMessage = null;
michael@0	276	}
michael@0	277
michael@0	278	@Override
michael@0	279	public AccessibilityNodeProvider getAccessibilityNodeProvider(final View host) {
michael@0	280	if (mAccessibilityNodeProvider == null)
michael@0	281	// The accessibility node structure for web content consists of 3 LayerView child nodes:
michael@0	282	// 1. VIRTUAL_CURSOR_PREVIOUS: Represents the virtual cursor position that is previous to the
michael@0	283	// current one.
michael@0	284	// 2. VIRTUAL_CURSOR_POSITION: Represents the current position of the virtual cursor.
michael@0	285	// 3. VIRTUAL_CURSOR_NEXT: Represents the next virtual cursor position.
michael@0	286	mAccessibilityNodeProvider = new AccessibilityNodeProvider() {
michael@0	287	@Override
michael@0	288	public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualDescendantId) {
michael@0	289	AccessibilityNodeInfo info = (virtualDescendantId == VIRTUAL_CURSOR_POSITION && sVirtualCursorNode != null) ?
michael@0	290	AccessibilityNodeInfo.obtain(sVirtualCursorNode) :
michael@0	291	AccessibilityNodeInfo.obtain(host, virtualDescendantId);
michael@0	292
michael@0	293	switch (virtualDescendantId) {
michael@0	294	case View.NO_ID:
michael@0	295	// This is the parent LayerView node, populate it with children.
michael@0	296	onInitializeAccessibilityNodeInfo(host, info);
michael@0	297	info.addChild(host, VIRTUAL_CURSOR_PREVIOUS);
michael@0	298	info.addChild(host, VIRTUAL_CURSOR_POSITION);
michael@0	299	info.addChild(host, VIRTUAL_CURSOR_NEXT);
michael@0	300	break;
michael@0	301	default:
michael@0	302	info.setParent(host);
michael@0	303	info.setSource(host, virtualDescendantId);
michael@0	304	info.setVisibleToUser(host.isShown());
michael@0	305	info.setPackageName(GeckoAppShell.getContext().getPackageName());
michael@0	306	info.setClassName(host.getClass().getName());
michael@0	307	info.addAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS);
michael@0	308	info.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS);
michael@0	309	info.addAction(AccessibilityNodeInfo.ACTION_CLICK);
michael@0	310	info.addAction(AccessibilityNodeInfo.ACTION_LONG_CLICK);
michael@0	311	info.addAction(AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY);
michael@0	312	info.addAction(AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY);
michael@0	313	info.setMovementGranularities(AccessibilityNodeInfo.MOVEMENT_GRANULARITY_CHARACTER |
michael@0	314	AccessibilityNodeInfo.MOVEMENT_GRANULARITY_WORD |
michael@0	315	AccessibilityNodeInfo.MOVEMENT_GRANULARITY_PARAGRAPH);
michael@0	316	break;
michael@0	317	}
michael@0	318	return info;
michael@0	319	}
michael@0	320
michael@0	321	@Override
michael@0	322	public boolean performAction (int virtualViewId, int action, Bundle arguments) {
michael@0	323	if (action == AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS) {
michael@0	324	// The accessibility focus is permanently on the middle node, VIRTUAL_CURSOR_POSITION.
michael@0	325	// When accessibility focus is requested on one of its siblings we move the virtual cursor
michael@0	326	// either forward or backward depending on which sibling was selected.
michael@0	327
michael@0	328	switch (virtualViewId) {
michael@0	329	case VIRTUAL_CURSOR_PREVIOUS:
michael@0	330	GeckoAppShell.
michael@0	331	sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:PreviousObject", null));
michael@0	332	return true;
michael@0	333	case VIRTUAL_CURSOR_NEXT:
michael@0	334	GeckoAppShell.
michael@0	335	sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:NextObject", null));
michael@0	336	return true;
michael@0	337	default:
michael@0	338	break;
michael@0	339	}
michael@0	340	} else if (action == AccessibilityNodeInfo.ACTION_CLICK && virtualViewId == VIRTUAL_CURSOR_POSITION) {
michael@0	341	GeckoAppShell.
michael@0	342	sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:ActivateObject", null));
michael@0	343	return true;
michael@0	344	} else if (action == AccessibilityNodeInfo.ACTION_LONG_CLICK && virtualViewId == VIRTUAL_CURSOR_POSITION) {
michael@0	345	GeckoAppShell.
michael@0	346	sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:LongPress", null));
michael@0	347	return true;
michael@0	348	} else if (action == AccessibilityNodeInfo.ACTION_NEXT_AT_MOVEMENT_GRANULARITY &&
michael@0	349	virtualViewId == VIRTUAL_CURSOR_POSITION) {
michael@0	350	// XXX: Self brailling gives this action with a bogus argument instead of an actual click action;
michael@0	351	// the argument value is the BRAILLE_CLICK_BASE_INDEX - the index of the routing key that was hit
michael@0	352	int granularity = arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT);
michael@0	353	if (granularity < 0) {
michael@0	354	int keyIndex = BRAILLE_CLICK_BASE_INDEX - granularity;
michael@0	355	JSONObject activationData = new JSONObject();
michael@0	356	try {
michael@0	357	activationData.put("keyIndex", keyIndex);
michael@0	358	} catch (JSONException e) {
michael@0	359	return true;
michael@0	360	}
michael@0	361	GeckoAppShell.
michael@0	362	sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:ActivateObject", activationData.toString()));
michael@0	363	} else {
michael@0	364	JSONObject movementData = new JSONObject();
michael@0	365	try {
michael@0	366	movementData.put("direction", "Next");
michael@0	367	movementData.put("granularity", granularity);
michael@0	368	} catch (JSONException e) {
michael@0	369	return true;
michael@0	370	}
michael@0	371	GeckoAppShell.
michael@0	372	sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:MoveByGranularity", movementData.toString()));
michael@0	373	}
michael@0	374	return true;
michael@0	375	} else if (action == AccessibilityNodeInfo.ACTION_PREVIOUS_AT_MOVEMENT_GRANULARITY &&
michael@0	376	virtualViewId == VIRTUAL_CURSOR_POSITION) {
michael@0	377	JSONObject movementData = new JSONObject();
michael@0	378	try {
michael@0	379	movementData.put("direction", "Previous");
michael@0	380	movementData.put("granularity", arguments.getInt(AccessibilityNodeInfo.ACTION_ARGUMENT_MOVEMENT_GRANULARITY_INT));
michael@0	381	} catch (JSONException e) {
michael@0	382	return true;
michael@0	383	}
michael@0	384	GeckoAppShell.
michael@0	385	sendEventToGecko(GeckoEvent.createBroadcastEvent("Accessibility:MoveByGranularity", movementData.toString()));
michael@0	386	return true;
michael@0	387	}
michael@0	388	return host.performAccessibilityAction(action, arguments);
michael@0	389	}
michael@0	390	};
michael@0	391
michael@0	392	return mAccessibilityNodeProvider;
michael@0	393	}
michael@0	394	}
michael@0	395	}