1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/mobile/android/base/favicons/LoadFaviconTask.java Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,557 @@
1.4 +/* This Source Code Form is subject to the terms of the Mozilla Public
1.5 + * License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
1.7 +
1.8 +package org.mozilla.gecko.favicons;
1.9 +
1.10 +
1.11 +import android.content.ContentResolver;
1.12 +import android.graphics.Bitmap;
1.13 +import android.net.http.AndroidHttpClient;
1.14 +import android.os.Handler;
1.15 +import android.text.TextUtils;
1.16 +import android.util.Log;
1.17 +import org.apache.http.Header;
1.18 +import org.apache.http.HttpEntity;
1.19 +import org.apache.http.HttpResponse;
1.20 +import org.apache.http.client.methods.HttpGet;
1.21 +import org.mozilla.gecko.GeckoAppShell;
1.22 +import org.mozilla.gecko.db.BrowserDB;
1.23 +import org.mozilla.gecko.favicons.decoders.FaviconDecoder;
1.24 +import org.mozilla.gecko.favicons.decoders.LoadFaviconResult;
1.25 +import org.mozilla.gecko.util.GeckoJarReader;
1.26 +import org.mozilla.gecko.util.ThreadUtils;
1.27 +import org.mozilla.gecko.util.UiAsyncTask;
1.28 +import static org.mozilla.gecko.favicons.Favicons.context;
1.29 +
1.30 +import java.io.IOException;
1.31 +import java.io.InputStream;
1.32 +import java.net.URI;
1.33 +import java.net.URISyntaxException;
1.34 +import java.util.HashMap;
1.35 +import java.util.HashSet;
1.36 +import java.util.LinkedList;
1.37 +import java.util.concurrent.atomic.AtomicInteger;
1.38 +
1.39 +/**
1.40 + * Class representing the asynchronous task to load a Favicon which is not currently in the in-memory
1.41 + * cache.
1.42 + * The implementation initially tries to get the Favicon from the database. Upon failure, the icon
1.43 + * is loaded from the internet.
1.44 + */
1.45 +public class LoadFaviconTask extends UiAsyncTask<Void, Void, Bitmap> {
1.46 + private static final String LOGTAG = "LoadFaviconTask";
1.47 +
1.48 + // Access to this map needs to be synchronized prevent multiple jobs loading the same favicon
1.49 + // from executing concurrently.
1.50 + private static final HashMap<String, LoadFaviconTask> loadsInFlight = new HashMap<String, LoadFaviconTask>();
1.51 +
1.52 + public static final int FLAG_PERSIST = 1;
1.53 + public static final int FLAG_SCALE = 2;
1.54 + private static final int MAX_REDIRECTS_TO_FOLLOW = 5;
1.55 + // The default size of the buffer to use for downloading Favicons in the event no size is given
1.56 + // by the server.
1.57 + private static final int DEFAULT_FAVICON_BUFFER_SIZE = 25000;
1.58 +
1.59 + private static AtomicInteger nextFaviconLoadId = new AtomicInteger(0);
1.60 + private int id;
1.61 + private String pageUrl;
1.62 + private String faviconURL;
1.63 + private OnFaviconLoadedListener listener;
1.64 + private int flags;
1.65 +
1.66 + private final boolean onlyFromLocal;
1.67 +
1.68 + // Assuming square favicons, judging by width only is acceptable.
1.69 + protected int targetWidth;
1.70 + private LinkedList<LoadFaviconTask> chainees;
1.71 + private boolean isChaining;
1.72 +
1.73 + static AndroidHttpClient httpClient = AndroidHttpClient.newInstance(GeckoAppShell.getGeckoInterface().getDefaultUAString());
1.74 +
1.75 + public LoadFaviconTask(Handler backgroundThreadHandler,
1.76 + String pageUrl, String faviconUrl, int flags,
1.77 + OnFaviconLoadedListener listener) {
1.78 + this(backgroundThreadHandler, pageUrl, faviconUrl, flags, listener, -1, false);
1.79 + }
1.80 + public LoadFaviconTask(Handler backgroundThreadHandler,
1.81 + String pageUrl, String faviconUrl, int flags,
1.82 + OnFaviconLoadedListener listener, int targetWidth, boolean onlyFromLocal) {
1.83 + super(backgroundThreadHandler);
1.84 +
1.85 + id = nextFaviconLoadId.incrementAndGet();
1.86 +
1.87 + this.pageUrl = pageUrl;
1.88 + this.faviconURL = faviconUrl;
1.89 + this.listener = listener;
1.90 + this.flags = flags;
1.91 + this.targetWidth = targetWidth;
1.92 + this.onlyFromLocal = onlyFromLocal;
1.93 + }
1.94 +
1.95 + // Runs in background thread
1.96 + private LoadFaviconResult loadFaviconFromDb() {
1.97 + ContentResolver resolver = context.getContentResolver();
1.98 + return BrowserDB.getFaviconForFaviconUrl(resolver, faviconURL);
1.99 + }
1.100 +
1.101 + // Runs in background thread
1.102 + private void saveFaviconToDb(final byte[] encodedFavicon) {
1.103 + if (encodedFavicon == null) {
1.104 + return;
1.105 + }
1.106 +
1.107 + if ((flags & FLAG_PERSIST) == 0) {
1.108 + return;
1.109 + }
1.110 +
1.111 + ContentResolver resolver = context.getContentResolver();
1.112 + BrowserDB.updateFaviconForUrl(resolver, pageUrl, encodedFavicon, faviconURL);
1.113 + }
1.114 +
1.115 + /**
1.116 + * Helper method for trying the download request to grab a Favicon.
1.117 + * @param faviconURI URL of Favicon to try and download
1.118 + * @return The HttpResponse containing the downloaded Favicon if successful, null otherwise.
1.119 + */
1.120 + private HttpResponse tryDownload(URI faviconURI) throws URISyntaxException, IOException {
1.121 + HashSet<String> visitedLinkSet = new HashSet<String>();
1.122 + visitedLinkSet.add(faviconURI.toString());
1.123 + return tryDownloadRecurse(faviconURI, visitedLinkSet);
1.124 + }
1.125 + private HttpResponse tryDownloadRecurse(URI faviconURI, HashSet<String> visited) throws URISyntaxException, IOException {
1.126 + if (visited.size() == MAX_REDIRECTS_TO_FOLLOW) {
1.127 + return null;
1.128 + }
1.129 +
1.130 + HttpGet request = new HttpGet(faviconURI);
1.131 + HttpResponse response = httpClient.execute(request);
1.132 + if (response == null) {
1.133 + return null;
1.134 + }
1.135 +
1.136 + if (response.getStatusLine() != null) {
1.137 +
1.138 + // Was the response a failure?
1.139 + int status = response.getStatusLine().getStatusCode();
1.140 +
1.141 + // Handle HTTP status codes requesting a redirect.
1.142 + if (status >= 300 && status < 400) {
1.143 + Header header = response.getFirstHeader("Location");
1.144 +
1.145 + // Handle mad webservers.
1.146 + if (header == null) {
1.147 + return null;
1.148 + }
1.149 +
1.150 + String newURI = header.getValue();
1.151 + if (newURI == null || newURI.equals(faviconURI.toString())) {
1.152 + return null;
1.153 + }
1.154 +
1.155 + if (visited.contains(newURI)) {
1.156 + // Already been redirected here - abort.
1.157 + return null;
1.158 + }
1.159 +
1.160 + visited.add(newURI);
1.161 + return tryDownloadRecurse(new URI(newURI), visited);
1.162 + }
1.163 +
1.164 + if (status >= 400) {
1.165 + return null;
1.166 + }
1.167 + }
1.168 + return response;
1.169 + }
1.170 +
1.171 + /**
1.172 + * Retrieve the specified favicon from the JAR, returning null if it's not
1.173 + * a JAR URI.
1.174 + */
1.175 + private static Bitmap fetchJARFavicon(String uri) {
1.176 + if (uri == null) {
1.177 + return null;
1.178 + }
1.179 + if (uri.startsWith("jar:jar:")) {
1.180 + Log.d(LOGTAG, "Fetching favicon from JAR.");
1.181 + try {
1.182 + return GeckoJarReader.getBitmap(context.getResources(), uri);
1.183 + } catch (Exception e) {
1.184 + // Just about anything could happen here.
1.185 + Log.w(LOGTAG, "Error fetching favicon from JAR.", e);
1.186 + return null;
1.187 + }
1.188 + }
1.189 + return null;
1.190 + }
1.191 +
1.192 + // Runs in background thread.
1.193 + // Does not attempt to fetch from JARs.
1.194 + private LoadFaviconResult downloadFavicon(URI targetFaviconURI) {
1.195 + if (targetFaviconURI == null) {
1.196 + return null;
1.197 + }
1.198 +
1.199 + // Only get favicons for HTTP/HTTPS.
1.200 + String scheme = targetFaviconURI.getScheme();
1.201 + if (!"http".equals(scheme) && !"https".equals(scheme)) {
1.202 + return null;
1.203 + }
1.204 +
1.205 + LoadFaviconResult result = null;
1.206 +
1.207 + try {
1.208 + result = downloadAndDecodeImage(targetFaviconURI);
1.209 + } catch (Exception e) {
1.210 + Log.e(LOGTAG, "Error reading favicon", e);
1.211 + }
1.212 +
1.213 + return result;
1.214 + }
1.215 +
1.216 + /**
1.217 + * Download the Favicon from the given URL and pass it to the decoder function.
1.218 + *
1.219 + * @param targetFaviconURL URL of the favicon to download.
1.220 + * @return A LoadFaviconResult containing the bitmap(s) extracted from the downloaded file, or
1.221 + * null if no or corrupt data ware received.
1.222 + * @throws IOException If attempts to fully read the stream result in such an exception, such as
1.223 + * in the event of a transient connection failure.
1.224 + * @throws URISyntaxException If the underlying call to tryDownload retries and raises such an
1.225 + * exception trying a fallback URL.
1.226 + */
1.227 + private LoadFaviconResult downloadAndDecodeImage(URI targetFaviconURL) throws IOException, URISyntaxException {
1.228 + // Try the URL we were given.
1.229 + HttpResponse response = tryDownload(targetFaviconURL);
1.230 + if (response == null) {
1.231 + return null;
1.232 + }
1.233 +
1.234 + HttpEntity entity = response.getEntity();
1.235 + if (entity == null) {
1.236 + return null;
1.237 + }
1.238 +
1.239 + // This may not be provided, but if it is, it's useful.
1.240 + final long entityReportedLength = entity.getContentLength();
1.241 + int bufferSize;
1.242 + if (entityReportedLength > 0) {
1.243 + // The size was reported and sane, so let's use that.
1.244 + // Integer overflow should not be a problem for Favicon sizes...
1.245 + bufferSize = (int) entityReportedLength + 1;
1.246 + } else {
1.247 + // No declared size, so guess and reallocate later if it turns out to be too small.
1.248 + bufferSize = DEFAULT_FAVICON_BUFFER_SIZE;
1.249 + }
1.250 +
1.251 + // Allocate a buffer to hold the raw favicon data downloaded.
1.252 + byte[] buffer = new byte[bufferSize];
1.253 +
1.254 + // The offset of the start of the buffer's free space.
1.255 + int bPointer = 0;
1.256 +
1.257 + // The quantity of bytes the last call to read yielded.
1.258 + int lastRead = 0;
1.259 + InputStream contentStream = entity.getContent();
1.260 + try {
1.261 + // Fully read the entity into the buffer - decoding of streams is not supported
1.262 + // (and questionably pointful - what would one do with a half-decoded Favicon?)
1.263 + while (lastRead != -1) {
1.264 + // Read as many bytes as are currently available into the buffer.
1.265 + lastRead = contentStream.read(buffer, bPointer, buffer.length - bPointer);
1.266 + bPointer += lastRead;
1.267 +
1.268 + // If buffer has overflowed, double its size and carry on.
1.269 + if (bPointer == buffer.length) {
1.270 + bufferSize *= 2;
1.271 + byte[] newBuffer = new byte[bufferSize];
1.272 +
1.273 + // Copy the contents of the old buffer into the new buffer.
1.274 + System.arraycopy(buffer, 0, newBuffer, 0, buffer.length);
1.275 + buffer = newBuffer;
1.276 + }
1.277 + }
1.278 + } finally {
1.279 + contentStream.close();
1.280 + }
1.281 +
1.282 + // Having downloaded the image, decode it.
1.283 + return FaviconDecoder.decodeFavicon(buffer, 0, bPointer + 1);
1.284 + }
1.285 +
1.286 + @Override
1.287 + protected Bitmap doInBackground(Void... unused) {
1.288 + if (isCancelled()) {
1.289 + return null;
1.290 + }
1.291 +
1.292 + String storedFaviconUrl;
1.293 + boolean isUsingDefaultURL = false;
1.294 +
1.295 + // Handle the case of malformed favicon URL.
1.296 + // If favicon is empty, fall back to the stored one.
1.297 + if (TextUtils.isEmpty(faviconURL)) {
1.298 + // Try to get the favicon URL from the memory cache.
1.299 + storedFaviconUrl = Favicons.getFaviconURLForPageURLFromCache(pageUrl);
1.300 +
1.301 + // If that failed, try to get the URL from the database.
1.302 + if (storedFaviconUrl == null) {
1.303 + storedFaviconUrl = Favicons.getFaviconURLForPageURL(pageUrl);
1.304 + if (storedFaviconUrl != null) {
1.305 + // If that succeeded, cache the URL loaded from the database in memory.
1.306 + Favicons.putFaviconURLForPageURLInCache(pageUrl, storedFaviconUrl);
1.307 + }
1.308 + }
1.309 +
1.310 + // If we found a faviconURL - use it.
1.311 + if (storedFaviconUrl != null) {
1.312 + faviconURL = storedFaviconUrl;
1.313 + } else {
1.314 + // If we don't have a stored one, fall back to the default.
1.315 + faviconURL = Favicons.guessDefaultFaviconURL(pageUrl);
1.316 +
1.317 + if (TextUtils.isEmpty(faviconURL)) {
1.318 + return null;
1.319 + }
1.320 + isUsingDefaultURL = true;
1.321 + }
1.322 + }
1.323 +
1.324 + // Check if favicon has failed - if so, give up. We need this check because, sometimes, we
1.325 + // didn't know the real Favicon URL until we asked the database.
1.326 + if (Favicons.isFailedFavicon(faviconURL)) {
1.327 + return null;
1.328 + }
1.329 +
1.330 + if (isCancelled()) {
1.331 + return null;
1.332 + }
1.333 +
1.334 + Bitmap image;
1.335 + // Determine if there is already an ongoing task to fetch the Favicon we desire.
1.336 + // If there is, just join the queue and wait for it to finish. If not, we carry on.
1.337 + synchronized(loadsInFlight) {
1.338 + // Another load of the current Favicon is already underway
1.339 + LoadFaviconTask existingTask = loadsInFlight.get(faviconURL);
1.340 + if (existingTask != null && !existingTask.isCancelled()) {
1.341 + existingTask.chainTasks(this);
1.342 + isChaining = true;
1.343 +
1.344 + // If we are chaining, we want to keep the first task started to do this job as the one
1.345 + // in the hashmap so subsequent tasks will add themselves to its chaining list.
1.346 + return null;
1.347 + }
1.348 +
1.349 + // We do not want to update the hashmap if the task has chained - other tasks need to
1.350 + // chain onto the same parent task.
1.351 + loadsInFlight.put(faviconURL, this);
1.352 + }
1.353 +
1.354 + if (isCancelled()) {
1.355 + return null;
1.356 + }
1.357 +
1.358 + // If there are no valid bitmaps decoded, the returned LoadFaviconResult is null.
1.359 + LoadFaviconResult loadedBitmaps = loadFaviconFromDb();
1.360 + if (loadedBitmaps != null) {
1.361 + return pushToCacheAndGetResult(loadedBitmaps);
1.362 + }
1.363 +
1.364 + if (onlyFromLocal || isCancelled()) {
1.365 + return null;
1.366 + }
1.367 +
1.368 + // Let's see if it's in a JAR.
1.369 + image = fetchJARFavicon(faviconURL);
1.370 + if (imageIsValid(image)) {
1.371 + // We don't want to put this into the DB.
1.372 + Favicons.putFaviconInMemCache(faviconURL, image);
1.373 + return image;
1.374 + }
1.375 +
1.376 + try {
1.377 + loadedBitmaps = downloadFavicon(new URI(faviconURL));
1.378 + } catch (URISyntaxException e) {
1.379 + Log.e(LOGTAG, "The provided favicon URL is not valid");
1.380 + return null;
1.381 + } catch (Exception e) {
1.382 + Log.e(LOGTAG, "Couldn't download favicon.", e);
1.383 + }
1.384 +
1.385 + if (loadedBitmaps != null) {
1.386 + // Fetching bytes to store can fail. saveFaviconToDb will
1.387 + // do the right thing, but we still choose to cache the
1.388 + // downloaded icon in memory.
1.389 + saveFaviconToDb(loadedBitmaps.getBytesForDatabaseStorage());
1.390 + return pushToCacheAndGetResult(loadedBitmaps);
1.391 + }
1.392 +
1.393 + if (isUsingDefaultURL) {
1.394 + Favicons.putFaviconInFailedCache(faviconURL);
1.395 + return null;
1.396 + }
1.397 +
1.398 + if (isCancelled()) {
1.399 + return null;
1.400 + }
1.401 +
1.402 + // If we're not already trying the default URL, try it now.
1.403 + final String guessed = Favicons.guessDefaultFaviconURL(pageUrl);
1.404 + if (guessed == null) {
1.405 + Favicons.putFaviconInFailedCache(faviconURL);
1.406 + return null;
1.407 + }
1.408 +
1.409 + image = fetchJARFavicon(guessed);
1.410 + if (imageIsValid(image)) {
1.411 + // We don't want to put this into the DB.
1.412 + Favicons.putFaviconInMemCache(faviconURL, image);
1.413 + return image;
1.414 + }
1.415 +
1.416 + try {
1.417 + loadedBitmaps = downloadFavicon(new URI(guessed));
1.418 + } catch (Exception e) {
1.419 + // Not interesting. It was an educated guess, anyway.
1.420 + return null;
1.421 + }
1.422 +
1.423 + if (loadedBitmaps != null) {
1.424 + saveFaviconToDb(loadedBitmaps.getBytesForDatabaseStorage());
1.425 + return pushToCacheAndGetResult(loadedBitmaps);
1.426 + }
1.427 +
1.428 + return null;
1.429 + }
1.430 +
1.431 + /**
1.432 + * Helper method to put the result of a favicon load into the memory cache and then query the
1.433 + * cache for the particular bitmap we want for this request.
1.434 + * This call is certain to succeed, provided there was enough memory to decode this favicon.
1.435 + *
1.436 + * @param loadedBitmaps LoadFaviconResult to store.
1.437 + * @return The optimal favicon available to satisfy this LoadFaviconTask's request, or null if
1.438 + * we are under extreme memory pressure and find ourselves dropping the cache immediately.
1.439 + */
1.440 + private Bitmap pushToCacheAndGetResult(LoadFaviconResult loadedBitmaps) {
1.441 + Favicons.putFaviconsInMemCache(faviconURL, loadedBitmaps.getBitmaps());
1.442 + Bitmap result = Favicons.getSizedFaviconFromCache(faviconURL, targetWidth);
1.443 + return result;
1.444 + }
1.445 +
1.446 + private static boolean imageIsValid(final Bitmap image) {
1.447 + return image != null &&
1.448 + image.getWidth() > 0 &&
1.449 + image.getHeight() > 0;
1.450 + }
1.451 +
1.452 + @Override
1.453 + protected void onPostExecute(Bitmap image) {
1.454 + if (isChaining) {
1.455 + return;
1.456 + }
1.457 +
1.458 + // Process the result, scale for the listener, etc.
1.459 + processResult(image);
1.460 +
1.461 + synchronized (loadsInFlight) {
1.462 + // Prevent any other tasks from chaining on this one.
1.463 + loadsInFlight.remove(faviconURL);
1.464 + }
1.465 +
1.466 + // Since any update to chainees is done while holding the loadsInFlight lock, once we reach
1.467 + // this point no further updates to that list can possibly take place (As far as other tasks
1.468 + // are concerned, there is no longer a task to chain from. The above block will have waited
1.469 + // for any tasks that were adding themselves to the list before reaching this point.)
1.470 +
1.471 + // As such, I believe we're safe to do the following without holding the lock.
1.472 + // This is nice - we do not want to take the lock unless we have to anyway, and chaining rarely
1.473 + // actually happens outside of the strange situations unit tests create.
1.474 +
1.475 + // Share the result with all chained tasks.
1.476 + if (chainees != null) {
1.477 + for (LoadFaviconTask t : chainees) {
1.478 + // In the case that we just decoded multiple favicons, either we're passing the right
1.479 + // image now, or the call into the cache in processResult will fetch the right one.
1.480 + t.processResult(image);
1.481 + }
1.482 + }
1.483 + }
1.484 +
1.485 + private void processResult(Bitmap image) {
1.486 + Favicons.removeLoadTask(id);
1.487 + Bitmap scaled = image;
1.488 +
1.489 + // Notify listeners, scaling if required.
1.490 + if (targetWidth != -1 && image != null && image.getWidth() != targetWidth) {
1.491 + scaled = Favicons.getSizedFaviconFromCache(faviconURL, targetWidth);
1.492 + }
1.493 +
1.494 + Favicons.dispatchResult(pageUrl, faviconURL, scaled, listener);
1.495 + }
1.496 +
1.497 + @Override
1.498 + protected void onCancelled() {
1.499 + Favicons.removeLoadTask(id);
1.500 +
1.501 + synchronized(loadsInFlight) {
1.502 + // Only remove from the hashmap if the task there is the one that's being canceled.
1.503 + // Cancellation of a task that would have chained is not interesting to the hashmap.
1.504 + final LoadFaviconTask primary = loadsInFlight.get(faviconURL);
1.505 + if (primary == this) {
1.506 + loadsInFlight.remove(faviconURL);
1.507 + return;
1.508 + }
1.509 + if (primary == null) {
1.510 + // This shouldn't happen.
1.511 + return;
1.512 + }
1.513 + if (primary.chainees != null) {
1.514 + primary.chainees.remove(this);
1.515 + }
1.516 + }
1.517 +
1.518 + // Note that we don't call the listener callback if the
1.519 + // favicon load is cancelled.
1.520 + }
1.521 +
1.522 + /**
1.523 + * When the result of this job is ready, also notify the chainee of the result.
1.524 + * Used for aggregating concurrent requests for the same Favicon into a single actual request.
1.525 + * (Don't want to download a hundred instances of Google's Favicon at once, for example).
1.526 + * The loadsInFlight lock must be held when calling this function.
1.527 + *
1.528 + * @param aChainee LoadFaviconTask
1.529 + */
1.530 + private void chainTasks(LoadFaviconTask aChainee) {
1.531 + if (chainees == null) {
1.532 + chainees = new LinkedList<LoadFaviconTask>();
1.533 + }
1.534 +
1.535 + chainees.add(aChainee);
1.536 + }
1.537 +
1.538 + int getId() {
1.539 + return id;
1.540 + }
1.541 +
1.542 + static void closeHTTPClient() {
1.543 + // This work must be done on a background thread because it shuts down
1.544 + // the connection pool, which typically involves closing a connection --
1.545 + // which counts as network activity.
1.546 + if (ThreadUtils.isOnBackgroundThread()) {
1.547 + if (httpClient != null) {
1.548 + httpClient.close();
1.549 + }
1.550 + return;
1.551 + }
1.552 +
1.553 + ThreadUtils.postToBackgroundThread(new Runnable() {
1.554 + @Override
1.555 + public void run() {
1.556 + LoadFaviconTask.closeHTTPClient();
1.557 + }
1.558 + });
1.559 + }
1.560 +}
