The Tor Browser / file comparison
comparison: build/automationutils.py
build/automationutils.py
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
equal
deleted
inserted
replaced
| -1:000000000000 | 0:6a80d2f9a6c0 |
|---|---|
| 1 # | |
| 2 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 3 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 5 | |
| 6 from __future__ import with_statement | |
| 7 import glob, logging, os, platform, shutil, subprocess, sys, tempfile, urllib2, zipfile | |
| 8 import base64 | |
| 9 import re | |
| 10 import os | |
| 11 from urlparse import urlparse | |
| 12 from operator import itemgetter | |
| 13 import signal | |
| 14 | |
| 15 try: | |
| 16 import mozinfo | |
| 17 except ImportError: | |
| 18 # Stub out fake mozinfo since this is not importable on Android 4.0 Opt. | |
| 19 # This should be fixed; see | |
| 20 # https://bugzilla.mozilla.org/show_bug.cgi?id=650881 | |
| 21 mozinfo = type('mozinfo', (), dict(info={}))() | |
| 22 mozinfo.isWin = mozinfo.isLinux = mozinfo.isUnix = mozinfo.isMac = False | |
| 23 | |
| 24 # TODO! FILE: localautomation :/ | |
| 25 # mapping from would-be mozinfo attr <-> sys.platform | |
| 26 mapping = {'isMac': ['mac', 'darwin'], | |
| 27 'isLinux': ['linux', 'linux2'], | |
| 28 'isWin': ['win32', 'win64'], | |
| 29 } | |
| 30 mapping = dict(sum([[(value, key) for value in values] for key, values in mapping.items()], [])) | |
| 31 attr = mapping.get(sys.platform) | |
| 32 if attr: | |
| 33 setattr(mozinfo, attr, True) | |
| 34 if mozinfo.isLinux: | |
| 35 mozinfo.isUnix = True | |
| 36 | |
| 37 __all__ = [ | |
| 38 "ZipFileReader", | |
| 39 "addCommonOptions", | |
| 40 "dumpLeakLog", | |
| 41 "isURL", | |
| 42 "processLeakLog", | |
| 43 "getDebuggerInfo", | |
| 44 "DEBUGGER_INFO", | |
| 45 "replaceBackSlashes", | |
| 46 'KeyValueParseError', | |
| 47 'parseKeyValue', | |
| 48 'systemMemory', | |
| 49 'environment', | |
| 50 'dumpScreen', | |
| 51 "ShutdownLeaks" | |
| 52 ] | |
| 53 | |
| 54 # Map of debugging programs to information about them, like default arguments | |
| 55 # and whether or not they are interactive. | |
| 56 DEBUGGER_INFO = { | |
| 57 # gdb requires that you supply the '--args' flag in order to pass arguments | |
| 58 # after the executable name to the executable. | |
| 59 "gdb": { | |
| 60 "interactive": True, | |
| 61 "args": "-q --args" | |
| 62 }, | |
| 63 | |
| 64 "cgdb": { | |
| 65 "interactive": True, | |
| 66 "args": "-q --args" | |
| 67 }, | |
| 68 | |
| 69 "lldb": { | |
| 70 "interactive": True, | |
| 71 "args": "--", | |
| 72 "requiresEscapedArgs": True | |
| 73 }, | |
| 74 | |
| 75 # valgrind doesn't explain much about leaks unless you set the | |
| 76 # '--leak-check=full' flag. But there are a lot of objects that are | |
| 77 # semi-deliberately leaked, so we set '--show-possibly-lost=no' to avoid | |
| 78 # uninteresting output from those objects. We set '--smc-check==all-non-file' | |
| 79 # and '--vex-iropt-register-updates=allregs-at-mem-access' so that valgrind | |
| 80 # deals properly with JIT'd JavaScript code. | |
| 81 "valgrind": { | |
| 82 "interactive": False, | |
| 83 "args": " ".join(["--leak-check=full", | |
| 84 "--show-possibly-lost=no", | |
| 85 "--smc-check=all-non-file", | |
| 86 "--vex-iropt-register-updates=allregs-at-mem-access"]) | |
| 87 } | |
| 88 } | |
| 89 | |
| 90 class ZipFileReader(object): | |
| 91 """ | |
| 92 Class to read zip files in Python 2.5 and later. Limited to only what we | |
| 93 actually use. | |
| 94 """ | |
| 95 | |
| 96 def __init__(self, filename): | |
| 97 self._zipfile = zipfile.ZipFile(filename, "r") | |
| 98 | |
| 99 def __del__(self): | |
| 100 self._zipfile.close() | |
| 101 | |
| 102 def _getnormalizedpath(self, path): | |
| 103 """ | |
| 104 Gets a normalized path from 'path' (or the current working directory if | |
| 105 'path' is None). Also asserts that the path exists. | |
| 106 """ | |
| 107 if path is None: | |
| 108 path = os.curdir | |
| 109 path = os.path.normpath(os.path.expanduser(path)) | |
| 110 assert os.path.isdir(path) | |
| 111 return path | |
| 112 | |
| 113 def _extractname(self, name, path): | |
| 114 """ | |
| 115 Extracts a file with the given name from the zip file to the given path. | |
| 116 Also creates any directories needed along the way. | |
| 117 """ | |
| 118 filename = os.path.normpath(os.path.join(path, name)) | |
| 119 if name.endswith("/"): | |
| 120 os.makedirs(filename) | |
| 121 else: | |
| 122 path = os.path.split(filename)[0] | |
| 123 if not os.path.isdir(path): | |
| 124 os.makedirs(path) | |
| 125 with open(filename, "wb") as dest: | |
| 126 dest.write(self._zipfile.read(name)) | |
| 127 | |
| 128 def namelist(self): | |
| 129 return self._zipfile.namelist() | |
| 130 | |
| 131 def read(self, name): | |
| 132 return self._zipfile.read(name) | |
| 133 | |
| 134 def extract(self, name, path = None): | |
| 135 if hasattr(self._zipfile, "extract"): | |
| 136 return self._zipfile.extract(name, path) | |
| 137 | |
| 138 # This will throw if name is not part of the zip file. | |
| 139 self._zipfile.getinfo(name) | |
| 140 | |
| 141 self._extractname(name, self._getnormalizedpath(path)) | |
| 142 | |
| 143 def extractall(self, path = None): | |
| 144 if hasattr(self._zipfile, "extractall"): | |
| 145 return self._zipfile.extractall(path) | |
| 146 | |
| 147 path = self._getnormalizedpath(path) | |
| 148 | |
| 149 for name in self._zipfile.namelist(): | |
| 150 self._extractname(name, path) | |
| 151 | |
| 152 log = logging.getLogger() | |
| 153 | |
| 154 def isURL(thing): | |
| 155 """Return True if |thing| looks like a URL.""" | |
| 156 # We want to download URLs like http://... but not Windows paths like c:\... | |
| 157 return len(urlparse(thing).scheme) >= 2 | |
| 158 | |
| 159 # Python does not provide strsignal() even in the very latest 3.x. | |
| 160 # This is a reasonable fake. | |
| 161 def strsig(n): | |
| 162 # Signal numbers run 0 through NSIG-1; an array with NSIG members | |
| 163 # has exactly that many slots | |
| 164 _sigtbl = [None]*signal.NSIG | |
| 165 for k in dir(signal): | |
| 166 if k.startswith("SIG") and not k.startswith("SIG_") and k != "SIGCLD" and k != "SIGPOLL": | |
| 167 _sigtbl[getattr(signal, k)] = k | |
| 168 # Realtime signals mostly have no names | |
| 169 if hasattr(signal, "SIGRTMIN") and hasattr(signal, "SIGRTMAX"): | |
| 170 for r in range(signal.SIGRTMIN+1, signal.SIGRTMAX+1): | |
| 171 _sigtbl[r] = "SIGRTMIN+" + str(r - signal.SIGRTMIN) | |
| 172 # Fill in any remaining gaps | |
| 173 for i in range(signal.NSIG): | |
| 174 if _sigtbl[i] is None: | |
| 175 _sigtbl[i] = "unrecognized signal, number " + str(i) | |
| 176 if n < 0 or n >= signal.NSIG: | |
| 177 return "out-of-range signal, number "+str(n) | |
| 178 return _sigtbl[n] | |
| 179 | |
| 180 def printstatus(status, name = ""): | |
| 181 # 'status' is the exit status | |
| 182 if os.name != 'posix': | |
| 183 # Windows error codes are easier to look up if printed in hexadecimal | |
| 184 if status < 0: | |
| 185 status += 2**32 | |
| 186 print "TEST-INFO | %s: exit status %x\n" % (name, status) | |
| 187 elif os.WIFEXITED(status): | |
| 188 print "TEST-INFO | %s: exit %d\n" % (name, os.WEXITSTATUS(status)) | |
| 189 elif os.WIFSIGNALED(status): | |
| 190 # The python stdlib doesn't appear to have strsignal(), alas | |
| 191 print "TEST-INFO | {}: killed by {}".format(name,strsig(os.WTERMSIG(status))) | |
| 192 else: | |
| 193 # This is probably a can't-happen condition on Unix, but let's be defensive | |
| 194 print "TEST-INFO | %s: undecodable exit status %04x\n" % (name, status) | |
| 195 | |
| 196 def addCommonOptions(parser, defaults={}): | |
| 197 parser.add_option("--xre-path", | |
| 198 action = "store", type = "string", dest = "xrePath", | |
| 199 # individual scripts will set a sane default | |
| 200 default = None, | |
| 201 help = "absolute path to directory containing XRE (probably xulrunner)") | |
| 202 if 'SYMBOLS_PATH' not in defaults: | |
| 203 defaults['SYMBOLS_PATH'] = None | |
| 204 parser.add_option("--symbols-path", | |
| 205 action = "store", type = "string", dest = "symbolsPath", | |
| 206 default = defaults['SYMBOLS_PATH'], | |
| 207 help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols") | |
| 208 parser.add_option("--debugger", | |
| 209 action = "store", dest = "debugger", | |
| 210 help = "use the given debugger to launch the application") | |
| 211 parser.add_option("--debugger-args", | |
| 212 action = "store", dest = "debuggerArgs", | |
| 213 help = "pass the given args to the debugger _before_ " | |
| 214 "the application on the command line") | |
| 215 parser.add_option("--debugger-interactive", | |
| 216 action = "store_true", dest = "debuggerInteractive", | |
| 217 help = "prevents the test harness from redirecting " | |
| 218 "stdout and stderr for interactive debuggers") | |
| 219 | |
| 220 def getFullPath(directory, path): | |
| 221 "Get an absolute path relative to 'directory'." | |
| 222 return os.path.normpath(os.path.join(directory, os.path.expanduser(path))) | |
| 223 | |
| 224 def searchPath(directory, path): | |
| 225 "Go one step beyond getFullPath and try the various folders in PATH" | |
| 226 # Try looking in the current working directory first. | |
| 227 newpath = getFullPath(directory, path) | |
| 228 if os.path.isfile(newpath): | |
| 229 return newpath | |
| 230 | |
| 231 # At this point we have to fail if a directory was given (to prevent cases | |
| 232 # like './gdb' from matching '/usr/bin/./gdb'). | |
| 233 if not os.path.dirname(path): | |
| 234 for dir in os.environ['PATH'].split(os.pathsep): | |
| 235 newpath = os.path.join(dir, path) | |
| 236 if os.path.isfile(newpath): | |
| 237 return newpath | |
| 238 return None | |
| 239 | |
| 240 def getDebuggerInfo(directory, debugger, debuggerArgs, debuggerInteractive = False): | |
| 241 | |
| 242 debuggerInfo = None | |
| 243 | |
| 244 if debugger: | |
| 245 debuggerPath = searchPath(directory, debugger) | |
| 246 if not debuggerPath: | |
| 247 print "Error: Path %s doesn't exist." % debugger | |
| 248 sys.exit(1) | |
| 249 | |
| 250 debuggerName = os.path.basename(debuggerPath).lower() | |
| 251 | |
| 252 def getDebuggerInfo(type, default): | |
| 253 if debuggerName in DEBUGGER_INFO and type in DEBUGGER_INFO[debuggerName]: | |
| 254 return DEBUGGER_INFO[debuggerName][type] | |
| 255 return default | |
| 256 | |
| 257 debuggerInfo = { | |
| 258 "path": debuggerPath, | |
| 259 "interactive" : getDebuggerInfo("interactive", False), | |
| 260 "args": getDebuggerInfo("args", "").split(), | |
| 261 "requiresEscapedArgs": getDebuggerInfo("requiresEscapedArgs", False) | |
| 262 } | |
| 263 | |
| 264 if debuggerArgs: | |
| 265 debuggerInfo["args"] = debuggerArgs.split() | |
| 266 if debuggerInteractive: | |
| 267 debuggerInfo["interactive"] = debuggerInteractive | |
| 268 | |
| 269 return debuggerInfo | |
| 270 | |
| 271 | |
| 272 def dumpLeakLog(leakLogFile, filter = False): | |
| 273 """Process the leak log, without parsing it. | |
| 274 | |
| 275 Use this function if you want the raw log only. | |
| 276 Use it preferably with the |XPCOM_MEM_LEAK_LOG| environment variable. | |
| 277 """ | |
| 278 | |
| 279 # Don't warn (nor "info") if the log file is not there. | |
| 280 if not os.path.exists(leakLogFile): | |
| 281 return | |
| 282 | |
| 283 with open(leakLogFile, "r") as leaks: | |
| 284 leakReport = leaks.read() | |
| 285 | |
| 286 # Only |XPCOM_MEM_LEAK_LOG| reports can be actually filtered out. | |
| 287 # Only check whether an actual leak was reported. | |
| 288 if filter and not "0 TOTAL " in leakReport: | |
| 289 return | |
| 290 | |
| 291 # Simply copy the log. | |
| 292 log.info(leakReport.rstrip("\n")) | |
| 293 | |
| 294 def processSingleLeakFile(leakLogFileName, processType, leakThreshold): | |
| 295 """Process a single leak log. | |
| 296 """ | |
| 297 | |
| 298 # Per-Inst Leaked Total Rem ... | |
| 299 # 0 TOTAL 17 192 419115886 2 ... | |
| 300 # 833 nsTimerImpl 60 120 24726 2 ... | |
| 301 lineRe = re.compile(r"^\s*\d+\s+(?P<name>\S+)\s+" | |
| 302 r"(?P<size>-?\d+)\s+(?P<bytesLeaked>-?\d+)\s+" | |
| 303 r"-?\d+\s+(?P<numLeaked>-?\d+)") | |
| 304 | |
| 305 processString = "" | |
| 306 if processType: | |
| 307 # eg 'plugin' | |
| 308 processString = " %s process:" % processType | |
| 309 | |
| 310 crashedOnPurpose = False | |
| 311 totalBytesLeaked = None | |
| 312 leakAnalysis = [] | |
| 313 leakedObjectNames = [] | |
| 314 with open(leakLogFileName, "r") as leaks: | |
| 315 for line in leaks: | |
| 316 if line.find("purposefully crash") > -1: | |
| 317 crashedOnPurpose = True | |
| 318 matches = lineRe.match(line) | |
| 319 if not matches: | |
| 320 # eg: the leak table header row | |
| 321 log.info(line.rstrip()) | |
| 322 continue | |
| 323 name = matches.group("name") | |
| 324 size = int(matches.group("size")) | |
| 325 bytesLeaked = int(matches.group("bytesLeaked")) | |
| 326 numLeaked = int(matches.group("numLeaked")) | |
| 327 # Output the raw line from the leak log table if it is the TOTAL row, | |
| 328 # or is for an object row that has been leaked. | |
| 329 if numLeaked != 0 or name == "TOTAL": | |
| 330 log.info(line.rstrip()) | |
| 331 # Analyse the leak log, but output later or it will interrupt the leak table | |
| 332 if name == "TOTAL": | |
| 333 totalBytesLeaked = bytesLeaked | |
| 334 if size < 0 or bytesLeaked < 0 or numLeaked < 0: | |
| 335 leakAnalysis.append("TEST-UNEXPECTED-FAIL | leakcheck |%s negative leaks caught!" | |
| 336 % processString) | |
| 337 continue | |
| 338 if name != "TOTAL" and numLeaked != 0: | |
| 339 leakedObjectNames.append(name) | |
| 340 leakAnalysis.append("TEST-INFO | leakcheck |%s leaked %d %s (%s bytes)" | |
| 341 % (processString, numLeaked, name, bytesLeaked)) | |
| 342 log.info('\n'.join(leakAnalysis)) | |
| 343 | |
| 344 if totalBytesLeaked is None: | |
| 345 # We didn't see a line with name 'TOTAL' | |
| 346 if crashedOnPurpose: | |
| 347 log.info("TEST-INFO | leakcheck |%s deliberate crash and thus no leak log" | |
| 348 % processString) | |
| 349 else: | |
| 350 # TODO: This should be a TEST-UNEXPECTED-FAIL, but was changed to a warning | |
| 351 # due to too many intermittent failures (see bug 831223). | |
| 352 log.info("WARNING | leakcheck |%s missing output line for total leaks!" | |
| 353 % processString) | |
| 354 return | |
| 355 | |
| 356 if totalBytesLeaked == 0: | |
| 357 log.info("TEST-PASS | leakcheck |%s no leaks detected!" % processString) | |
| 358 return | |
| 359 | |
| 360 # totalBytesLeaked was seen and is non-zero. | |
| 361 if totalBytesLeaked > leakThreshold: | |
| 362 # Fail the run if we're over the threshold (which defaults to 0) | |
| 363 prefix = "TEST-UNEXPECTED-FAIL" | |
| 364 else: | |
| 365 prefix = "WARNING" | |
| 366 # Create a comma delimited string of the first N leaked objects found, | |
| 367 # to aid with bug summary matching in TBPL. Note: The order of the objects | |
| 368 # had no significance (they're sorted alphabetically). | |
| 369 maxSummaryObjects = 5 | |
| 370 leakedObjectSummary = ', '.join(leakedObjectNames[:maxSummaryObjects]) | |
| 371 if len(leakedObjectNames) > maxSummaryObjects: | |
| 372 leakedObjectSummary += ', ...' | |
| 373 log.info("%s | leakcheck |%s %d bytes leaked (%s)" | |
| 374 % (prefix, processString, totalBytesLeaked, leakedObjectSummary)) | |
| 375 | |
| 376 def processLeakLog(leakLogFile, leakThreshold = 0): | |
| 377 """Process the leak log, including separate leak logs created | |
| 378 by child processes. | |
| 379 | |
| 380 Use this function if you want an additional PASS/FAIL summary. | |
| 381 It must be used with the |XPCOM_MEM_BLOAT_LOG| environment variable. | |
| 382 """ | |
| 383 | |
| 384 if not os.path.exists(leakLogFile): | |
| 385 log.info("WARNING | leakcheck | refcount logging is off, so leaks can't be detected!") | |
| 386 return | |
| 387 | |
| 388 if leakThreshold != 0: | |
| 389 log.info("TEST-INFO | leakcheck | threshold set at %d bytes" % leakThreshold) | |
| 390 | |
| 391 (leakLogFileDir, leakFileBase) = os.path.split(leakLogFile) | |
| 392 fileNameRegExp = re.compile(r".*?_([a-z]*)_pid\d*$") | |
| 393 if leakFileBase[-4:] == ".log": | |
| 394 leakFileBase = leakFileBase[:-4] | |
| 395 fileNameRegExp = re.compile(r".*?_([a-z]*)_pid\d*.log$") | |
| 396 | |
| 397 for fileName in os.listdir(leakLogFileDir): | |
| 398 if fileName.find(leakFileBase) != -1: | |
| 399 thisFile = os.path.join(leakLogFileDir, fileName) | |
| 400 processType = None | |
| 401 m = fileNameRegExp.search(fileName) | |
| 402 if m: | |
| 403 processType = m.group(1) | |
| 404 processSingleLeakFile(thisFile, processType, leakThreshold) | |
| 405 | |
| 406 def replaceBackSlashes(input): | |
| 407 return input.replace('\\', '/') | |
| 408 | |
| 409 class KeyValueParseError(Exception): | |
| 410 """error when parsing strings of serialized key-values""" | |
| 411 def __init__(self, msg, errors=()): | |
| 412 self.errors = errors | |
| 413 Exception.__init__(self, msg) | |
| 414 | |
| 415 def parseKeyValue(strings, separator='=', context='key, value: '): | |
| 416 """ | |
| 417 parse string-serialized key-value pairs in the form of | |
| 418 `key = value`. Returns a list of 2-tuples. | |
| 419 Note that whitespace is not stripped. | |
| 420 """ | |
| 421 | |
| 422 # syntax check | |
| 423 missing = [string for string in strings if separator not in string] | |
| 424 if missing: | |
| 425 raise KeyValueParseError("Error: syntax error in %s" % (context, | |
| 426 ','.join(missing)), | |
| 427 errors=missing) | |
| 428 return [string.split(separator, 1) for string in strings] | |
| 429 | |
| 430 def systemMemory(): | |
| 431 """ | |
| 432 Returns total system memory in kilobytes. | |
| 433 Works only on unix-like platforms where `free` is in the path. | |
| 434 """ | |
| 435 return int(os.popen("free").readlines()[1].split()[1]) | |
| 436 | |
| 437 def environment(xrePath, env=None, crashreporter=True, debugger=False, dmdPath=None): | |
| 438 """populate OS environment variables for mochitest""" | |
| 439 | |
| 440 env = os.environ.copy() if env is None else env | |
| 441 | |
| 442 assert os.path.isabs(xrePath) | |
| 443 | |
| 444 ldLibraryPath = xrePath | |
| 445 | |
| 446 envVar = None | |
| 447 dmdLibrary = None | |
| 448 preloadEnvVar = None | |
| 449 if mozinfo.isUnix: | |
| 450 envVar = "LD_LIBRARY_PATH" | |
| 451 env['MOZILLA_FIVE_HOME'] = xrePath | |
| 452 dmdLibrary = "libdmd.so" | |
| 453 preloadEnvVar = "LD_PRELOAD" | |
| 454 elif mozinfo.isMac: | |
| 455 envVar = "DYLD_LIBRARY_PATH" | |
| 456 dmdLibrary = "libdmd.dylib" | |
| 457 preloadEnvVar = "DYLD_INSERT_LIBRARIES" | |
| 458 elif mozinfo.isWin: | |
| 459 envVar = "PATH" | |
| 460 dmdLibrary = "dmd.dll" | |
| 461 preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB" | |
| 462 if envVar: | |
| 463 envValue = ((env.get(envVar), str(ldLibraryPath)) | |
| 464 if mozinfo.isWin | |
| 465 else (ldLibraryPath, dmdPath, env.get(envVar))) | |
| 466 env[envVar] = os.path.pathsep.join([path for path in envValue if path]) | |
| 467 | |
| 468 if dmdPath and dmdLibrary and preloadEnvVar: | |
| 469 env['DMD'] = '1' | |
| 470 env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary) | |
| 471 | |
| 472 # crashreporter | |
| 473 env['GNOME_DISABLE_CRASH_DIALOG'] = '1' | |
| 474 env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1' | |
| 475 env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1' | |
| 476 | |
| 477 if crashreporter and not debugger: | |
| 478 env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' | |
| 479 env['MOZ_CRASHREPORTER'] = '1' | |
| 480 else: | |
| 481 env['MOZ_CRASHREPORTER_DISABLE'] = '1' | |
| 482 | |
| 483 # Crash on non-local network connections. | |
| 484 env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' | |
| 485 | |
| 486 # Set WebRTC logging in case it is not set yet | |
| 487 env.setdefault('NSPR_LOG_MODULES', 'signaling:5,mtransport:5,datachannel:5') | |
| 488 env.setdefault('R_LOG_LEVEL', '6') | |
| 489 env.setdefault('R_LOG_DESTINATION', 'stderr') | |
| 490 env.setdefault('R_LOG_VERBOSE', '1') | |
| 491 | |
| 492 # ASan specific environment stuff | |
| 493 asan = bool(mozinfo.info.get("asan")) | |
| 494 if asan and (mozinfo.isLinux or mozinfo.isMac): | |
| 495 try: | |
| 496 # Symbolizer support | |
| 497 llvmsym = os.path.join(xrePath, "llvm-symbolizer") | |
| 498 if os.path.isfile(llvmsym): | |
| 499 env["ASAN_SYMBOLIZER_PATH"] = llvmsym | |
| 500 log.info("ASan using symbolizer at %s", llvmsym) | |
| 501 | |
| 502 totalMemory = systemMemory() | |
| 503 | |
| 504 # Only 4 GB RAM or less available? Use custom ASan options to reduce | |
| 505 # the amount of resources required to do the tests. Standard options | |
| 506 # will otherwise lead to OOM conditions on the current test slaves. | |
| 507 message = "INFO | runtests.py | ASan running in %s configuration" | |
| 508 if totalMemory <= 1024 * 1024 * 4: | |
| 509 message = message % 'low-memory' | |
| 510 env["ASAN_OPTIONS"] = "quarantine_size=50331648:malloc_context_size=5" | |
| 511 else: | |
| 512 message = message % 'default memory' | |
| 513 except OSError,err: | |
| 514 log.info("Failed determine available memory, disabling ASan low-memory configuration: %s", err.strerror) | |
| 515 except: | |
| 516 log.info("Failed determine available memory, disabling ASan low-memory configuration") | |
| 517 else: | |
| 518 log.info(message) | |
| 519 | |
| 520 return env | |
| 521 | |
| 522 def dumpScreen(utilityPath): | |
| 523 """dumps a screenshot of the entire screen to a directory specified by | |
| 524 the MOZ_UPLOAD_DIR environment variable""" | |
| 525 import mozfile | |
| 526 | |
| 527 # Need to figure out which OS-dependent tool to use | |
| 528 if mozinfo.isUnix: | |
| 529 utility = [os.path.join(utilityPath, "screentopng")] | |
| 530 utilityname = "screentopng" | |
| 531 elif mozinfo.isMac: | |
| 532 utility = ['/usr/sbin/screencapture', '-C', '-x', '-t', 'png'] | |
| 533 utilityname = "screencapture" | |
| 534 elif mozinfo.isWin: | |
| 535 utility = [os.path.join(utilityPath, "screenshot.exe")] | |
| 536 utilityname = "screenshot" | |
| 537 | |
| 538 # Get dir where to write the screenshot file | |
| 539 parent_dir = os.environ.get('MOZ_UPLOAD_DIR', None) | |
| 540 if not parent_dir: | |
| 541 log.info('Failed to retrieve MOZ_UPLOAD_DIR env var') | |
| 542 return | |
| 543 | |
| 544 # Run the capture | |
| 545 try: | |
| 546 tmpfd, imgfilename = tempfile.mkstemp(prefix='mozilla-test-fail-screenshot_', suffix='.png', dir=parent_dir) | |
| 547 os.close(tmpfd) | |
| 548 returncode = subprocess.call(utility + [imgfilename]) | |
| 549 printstatus(returncode, utilityname) | |
| 550 except OSError, err: | |
| 551 log.info("Failed to start %s for screenshot: %s", | |
| 552 utility[0], err.strerror) | |
| 553 return | |
| 554 | |
| 555 class ShutdownLeaks(object): | |
| 556 """ | |
| 557 Parses the mochitest run log when running a debug build, assigns all leaked | |
| 558 DOM windows (that are still around after test suite shutdown, despite running | |
| 559 the GC) to the tests that created them and prints leak statistics. | |
| 560 """ | |
| 561 | |
| 562 def __init__(self, logger): | |
| 563 self.logger = logger | |
| 564 self.tests = [] | |
| 565 self.leakedWindows = {} | |
| 566 self.leakedDocShells = set() | |
| 567 self.currentTest = None | |
| 568 self.seenShutdown = False | |
| 569 | |
| 570 def log(self, line): | |
| 571 if line[2:11] == "DOMWINDOW": | |
| 572 self._logWindow(line) | |
| 573 elif line[2:10] == "DOCSHELL": | |
| 574 self._logDocShell(line) | |
| 575 elif line.startswith("TEST-START"): | |
| 576 fileName = line.split(" ")[-1].strip().replace("chrome://mochitests/content/browser/", "") | |
| 577 self.currentTest = {"fileName": fileName, "windows": set(), "docShells": set()} | |
| 578 elif line.startswith("INFO TEST-END"): | |
| 579 # don't track a test if no windows or docShells leaked | |
| 580 if self.currentTest and (self.currentTest["windows"] or self.currentTest["docShells"]): | |
| 581 self.tests.append(self.currentTest) | |
| 582 self.currentTest = None | |
| 583 elif line.startswith("INFO TEST-START | Shutdown"): | |
| 584 self.seenShutdown = True | |
| 585 | |
| 586 def process(self): | |
| 587 for test in self._parseLeakingTests(): | |
| 588 for url, count in self._zipLeakedWindows(test["leakedWindows"]): | |
| 589 self.logger("TEST-UNEXPECTED-FAIL | %s | leaked %d window(s) until shutdown [url = %s]", test["fileName"], count, url) | |
| 590 | |
| 591 if test["leakedDocShells"]: | |
| 592 self.logger("TEST-UNEXPECTED-FAIL | %s | leaked %d docShell(s) until shutdown", test["fileName"], len(test["leakedDocShells"])) | |
| 593 | |
| 594 def _logWindow(self, line): | |
| 595 created = line[:2] == "++" | |
| 596 pid = self._parseValue(line, "pid") | |
| 597 serial = self._parseValue(line, "serial") | |
| 598 | |
| 599 # log line has invalid format | |
| 600 if not pid or not serial: | |
| 601 self.logger("TEST-UNEXPECTED-FAIL | ShutdownLeaks | failed to parse line <%s>", line) | |
| 602 return | |
| 603 | |
| 604 key = pid + "." + serial | |
| 605 | |
| 606 if self.currentTest: | |
| 607 windows = self.currentTest["windows"] | |
| 608 if created: | |
| 609 windows.add(key) | |
| 610 else: | |
| 611 windows.discard(key) | |
| 612 elif self.seenShutdown and not created: | |
| 613 self.leakedWindows[key] = self._parseValue(line, "url") | |
| 614 | |
| 615 def _logDocShell(self, line): | |
| 616 created = line[:2] == "++" | |
| 617 pid = self._parseValue(line, "pid") | |
| 618 id = self._parseValue(line, "id") | |
| 619 | |
| 620 # log line has invalid format | |
| 621 if not pid or not id: | |
| 622 self.logger("TEST-UNEXPECTED-FAIL | ShutdownLeaks | failed to parse line <%s>", line) | |
| 623 return | |
| 624 | |
| 625 key = pid + "." + id | |
| 626 | |
| 627 if self.currentTest: | |
| 628 docShells = self.currentTest["docShells"] | |
| 629 if created: | |
| 630 docShells.add(key) | |
| 631 else: | |
| 632 docShells.discard(key) | |
| 633 elif self.seenShutdown and not created: | |
| 634 self.leakedDocShells.add(key) | |
| 635 | |
| 636 def _parseValue(self, line, name): | |
| 637 match = re.search("\[%s = (.+?)\]" % name, line) | |
| 638 if match: | |
| 639 return match.group(1) | |
| 640 return None | |
| 641 | |
| 642 def _parseLeakingTests(self): | |
| 643 leakingTests = [] | |
| 644 | |
| 645 for test in self.tests: | |
| 646 test["leakedWindows"] = [self.leakedWindows[id] for id in test["windows"] if id in self.leakedWindows] | |
| 647 test["leakedDocShells"] = [id for id in test["docShells"] if id in self.leakedDocShells] | |
| 648 test["leakCount"] = len(test["leakedWindows"]) + len(test["leakedDocShells"]) | |
| 649 | |
| 650 if test["leakCount"]: | |
| 651 leakingTests.append(test) | |
| 652 | |
| 653 return sorted(leakingTests, key=itemgetter("leakCount"), reverse=True) | |
| 654 | |
| 655 def _zipLeakedWindows(self, leakedWindows): | |
| 656 counts = [] | |
| 657 counted = set() | |
| 658 | |
| 659 for url in leakedWindows: | |
| 660 if not url in counted: | |
| 661 counts.append((url, leakedWindows.count(url))) | |
| 662 counted.add(url) | |
| 663 | |
| 664 return sorted(counts, key=itemgetter(1), reverse=True) |
