The Tor Browser / file comparison

comparison: layout/tools/reftest/runreftest.py

layout/tools/reftest/runreftest.py

branch
TOR_BUG_3246
changeset 6
8bccb770b82d
equal deleted inserted replaced
-1:000000000000 0:eb56a89e6b7c
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 """
6 Runs the reftest test harness.
7 """
8
9 from optparse import OptionParser
10 import collections
11 import json
12 import multiprocessing
13 import os
14 import re
15 import shutil
16 import subprocess
17 import sys
18 import threading
19
20 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
21 sys.path.insert(0, SCRIPT_DIRECTORY)
22
23 from automation import Automation
24 from automationutils import (
25 addCommonOptions,
26 getDebuggerInfo,
27 isURL,
28 processLeakLog
29 )
30 import mozprofile
31
32 def categoriesToRegex(categoryList):
33 return "\\(" + ', '.join(["(?P<%s>\\d+) %s" % c for c in categoryList]) + "\\)"
34 summaryLines = [('Successful', [('pass', 'pass'), ('loadOnly', 'load only')]),
35 ('Unexpected', [('fail', 'unexpected fail'),
36 ('pass', 'unexpected pass'),
37 ('asserts', 'unexpected asserts'),
38 ('fixedAsserts', 'unexpected fixed asserts'),
39 ('failedLoad', 'failed load'),
40 ('exception', 'exception')]),
41 ('Known problems', [('knownFail', 'known fail'),
42 ('knownAsserts', 'known asserts'),
43 ('random', 'random'),
44 ('skipped', 'skipped'),
45 ('slow', 'slow')])]
46
47 # Python's print is not threadsafe.
48 printLock = threading.Lock()
49
50 class ReftestThread(threading.Thread):
51 def __init__(self, cmdlineArgs):
52 threading.Thread.__init__(self)
53 self.cmdlineArgs = cmdlineArgs
54 self.summaryMatches = {}
55 self.retcode = -1
56 for text, _ in summaryLines:
57 self.summaryMatches[text] = None
58
59 def run(self):
60 with printLock:
61 print "Starting thread with", self.cmdlineArgs
62 sys.stdout.flush()
63 process = subprocess.Popen(self.cmdlineArgs, stdout=subprocess.PIPE)
64 for chunk in self.chunkForMergedOutput(process.stdout):
65 with printLock:
66 print chunk,
67 sys.stdout.flush()
68 self.retcode = process.wait()
69
70 def chunkForMergedOutput(self, logsource):
71 """Gather lines together that should be printed as one atomic unit.
72 Individual test results--anything between 'REFTEST TEST-START' and
73 'REFTEST TEST-END' lines--are an atomic unit. Lines with data from
74 summaries are parsed and the data stored for later aggregation.
75 Other lines are considered their own atomic units and are permitted
76 to intermix freely."""
77 testStartRegex = re.compile("^REFTEST TEST-START")
78 testEndRegex = re.compile("^REFTEST TEST-END")
79 summaryHeadRegex = re.compile("^REFTEST INFO \\| Result summary:")
80 summaryRegexFormatString = "^REFTEST INFO \\| (?P<message>{text}): (?P<total>\\d+) {regex}"
81 summaryRegexStrings = [summaryRegexFormatString.format(text=text,
82 regex=categoriesToRegex(categories))
83 for (text, categories) in summaryLines]
84 summaryRegexes = [re.compile(regex) for regex in summaryRegexStrings]
85
86 for line in logsource:
87 if testStartRegex.search(line) is not None:
88 chunkedLines = [line]
89 for lineToBeChunked in logsource:
90 chunkedLines.append(lineToBeChunked)
91 if testEndRegex.search(lineToBeChunked) is not None:
92 break
93 yield ''.join(chunkedLines)
94 continue
95
96 haveSuppressedSummaryLine = False
97 for regex in summaryRegexes:
98 match = regex.search(line)
99 if match is not None:
100 self.summaryMatches[match.group('message')] = match
101 haveSuppressedSummaryLine = True
102 break
103 if haveSuppressedSummaryLine:
104 continue
105
106 if summaryHeadRegex.search(line) is None:
107 yield line
108
109 class RefTest(object):
110
111 oldcwd = os.getcwd()
112
113 def __init__(self, automation=None):
114 self.automation = automation or Automation()
115
116 def getFullPath(self, path):
117 "Get an absolute path relative to self.oldcwd."
118 return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
119
120 def getManifestPath(self, path):
121 "Get the path of the manifest, and for remote testing this function is subclassed to point to remote manifest"
122 path = self.getFullPath(path)
123 if os.path.isdir(path):
124 defaultManifestPath = os.path.join(path, 'reftest.list')
125 if os.path.exists(defaultManifestPath):
126 path = defaultManifestPath
127 else:
128 defaultManifestPath = os.path.join(path, 'crashtests.list')
129 if os.path.exists(defaultManifestPath):
130 path = defaultManifestPath
131 return path
132
133 def makeJSString(self, s):
134 return '"%s"' % re.sub(r'([\\"])', r'\\\1', s)
135
136 def createReftestProfile(self, options, manifest, server='localhost',
137 special_powers=True, profile_to_clone=None):
138 """
139 Sets up a profile for reftest.
140 'manifest' is the path to the reftest.list file we want to test with. This is used in
141 the remote subclass in remotereftest.py so we can write it to a preference for the
142 bootstrap extension.
143 """
144
145 locations = mozprofile.permissions.ServerLocations()
146 locations.add_host(server, port=0)
147 locations.add_host('<file>', port=0)
148
149 # Set preferences for communication between our command line arguments
150 # and the reftest harness. Preferences that are required for reftest
151 # to work should instead be set in reftest-cmdline.js .
152 prefs = {}
153 prefs['reftest.timeout'] = options.timeout * 1000
154 if options.totalChunks:
155 prefs['reftest.totalChunks'] = options.totalChunks
156 if options.thisChunk:
157 prefs['reftest.thisChunk'] = options.thisChunk
158 if options.logFile:
159 prefs['reftest.logFile'] = options.logFile
160 if options.ignoreWindowSize:
161 prefs['reftest.ignoreWindowSize'] = True
162 if options.filter:
163 prefs['reftest.filter'] = options.filter
164 if options.shuffle:
165 prefs['reftest.shuffle'] = True
166 prefs['reftest.focusFilterMode'] = options.focusFilterMode
167
168 # Ensure that telemetry is disabled, so we don't connect to the telemetry
169 # server in the middle of the tests.
170 prefs['toolkit.telemetry.enabled'] = False
171 # Likewise for safebrowsing.
172 prefs['browser.safebrowsing.enabled'] = False
173 prefs['browser.safebrowsing.malware.enabled'] = False
174 # And for snippets.
175 prefs['browser.snippets.enabled'] = False
176 prefs['browser.snippets.syncPromo.enabled'] = False
177 # And for useragent updates.
178 prefs['general.useragent.updates.enabled'] = False
179 # And for webapp updates. Yes, it is supposed to be an integer.
180 prefs['browser.webapps.checkForUpdates'] = 0
181
182 if options.e10s:
183 prefs['browser.tabs.remote.autostart'] = True
184
185 for v in options.extraPrefs:
186 thispref = v.split('=')
187 if len(thispref) < 2:
188 print "Error: syntax error in --setpref=" + v
189 sys.exit(1)
190 prefs[thispref[0]] = mozprofile.Preferences.cast(thispref[1].strip())
191
192 # install the reftest extension bits into the profile
193 addons = []
194 addons.append(os.path.join(SCRIPT_DIRECTORY, "reftest"))
195
196 # I would prefer to use "--install-extension reftest/specialpowers", but that requires tight coordination with
197 # release engineering and landing on multiple branches at once.
198 if special_powers and (manifest.endswith('crashtests.list') or manifest.endswith('jstests.list')):
199 addons.append(os.path.join(SCRIPT_DIRECTORY, 'specialpowers'))
200
201 # Install distributed extensions, if application has any.
202 distExtDir = os.path.join(options.app[ : options.app.rfind(os.sep)], "distribution", "extensions")
203 if os.path.isdir(distExtDir):
204 for f in os.listdir(distExtDir):
205 addons.append(os.path.join(distExtDir, f))
206
207 # Install custom extensions.
208 for f in options.extensionsToInstall:
209 addons.append(self.getFullPath(f))
210
211 kwargs = { 'addons': addons,
212 'preferences': prefs,
213 'locations': locations }
214 if profile_to_clone:
215 profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)
216 else:
217 profile = mozprofile.Profile(**kwargs)
218
219 self.copyExtraFilesToProfile(options, profile)
220 return profile
221
222 def buildBrowserEnv(self, options, profileDir):
223 browserEnv = self.automation.environment(xrePath = options.xrePath, debugger=options.debugger)
224 browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
225
226 for v in options.environment:
227 ix = v.find("=")
228 if ix <= 0:
229 print "Error: syntax error in --setenv=" + v
230 return None
231 browserEnv[v[:ix]] = v[ix + 1:]
232
233 # Enable leaks detection to its own log file.
234 self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log")
235 browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile
236 return browserEnv
237
238 def cleanup(self, profileDir):
239 if profileDir:
240 shutil.rmtree(profileDir, True)
241
242 def runTests(self, testPath, options, cmdlineArgs = None):
243 if not options.runTestsInParallel:
244 return self.runSerialTests(testPath, options, cmdlineArgs)
245
246 cpuCount = multiprocessing.cpu_count()
247
248 # We have the directive, technology, and machine to run multiple test instances.
249 # Experimentation says that reftests are not overly CPU-intensive, so we can run
250 # multiple jobs per CPU core.
251 #
252 # Our Windows machines in automation seem to get upset when we run a lot of
253 # simultaneous tests on them, so tone things down there.
254 if sys.platform == 'win32':
255 jobsWithoutFocus = cpuCount
256 else:
257 jobsWithoutFocus = 2 * cpuCount
258
259 totalJobs = jobsWithoutFocus + 1
260 perProcessArgs = [sys.argv[:] for i in range(0, totalJobs)]
261
262 # First job is only needs-focus tests. Remaining jobs are non-needs-focus and chunked.
263 perProcessArgs[0].insert(-1, "--focus-filter-mode=needs-focus")
264 for (chunkNumber, jobArgs) in enumerate(perProcessArgs[1:], start=1):
265 jobArgs[-1:-1] = ["--focus-filter-mode=non-needs-focus",
266 "--total-chunks=%d" % jobsWithoutFocus,
267 "--this-chunk=%d" % chunkNumber]
268
269 for jobArgs in perProcessArgs:
270 try:
271 jobArgs.remove("--run-tests-in-parallel")
272 except:
273 pass
274 jobArgs.insert(-1, "--no-run-tests-in-parallel")
275 jobArgs[0:0] = [sys.executable, "-u"]
276
277 threads = [ReftestThread(args) for args in perProcessArgs[1:]]
278 for t in threads:
279 t.start()
280
281 while True:
282 # The test harness in each individual thread will be doing timeout
283 # handling on its own, so we shouldn't need to worry about any of
284 # the threads hanging for arbitrarily long.
285 for t in threads:
286 t.join(10)
287 if not any(t.is_alive() for t in threads):
288 break
289
290 # Run the needs-focus tests serially after the other ones, so we don't
291 # have to worry about races between the needs-focus tests *actually*
292 # needing focus and the dummy windows in the non-needs-focus tests
293 # trying to focus themselves.
294 focusThread = ReftestThread(perProcessArgs[0])
295 focusThread.start()
296 focusThread.join()
297
298 # Output the summaries that the ReftestThread filters suppressed.
299 summaryObjects = [collections.defaultdict(int) for s in summaryLines]
300 for t in threads:
301 for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
302 threadMatches = t.summaryMatches[text]
303 for (attribute, description) in categories:
304 amount = int(threadMatches.group(attribute) if threadMatches else 0)
305 summaryObj[attribute] += amount
306 amount = int(threadMatches.group('total') if threadMatches else 0)
307 summaryObj['total'] += amount
308
309 print 'REFTEST INFO | Result summary:'
310 for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
311 details = ', '.join(["%d %s" % (summaryObj[attribute], description) for (attribute, description) in categories])
312 print 'REFTEST INFO | ' + text + ': ' + str(summaryObj['total']) + ' (' + details + ')'
313
314 return int(any(t.retcode != 0 for t in threads))
315
316 def runSerialTests(self, testPath, options, cmdlineArgs = None):
317 debuggerInfo = getDebuggerInfo(self.oldcwd, options.debugger, options.debuggerArgs,
318 options.debuggerInteractive);
319
320 profileDir = None
321 try:
322 reftestlist = self.getManifestPath(testPath)
323 if cmdlineArgs == None:
324 cmdlineArgs = ['-reftest', reftestlist]
325 profile = self.createReftestProfile(options, reftestlist)
326 profileDir = profile.profile # name makes more sense
327
328 # browser environment
329 browserEnv = self.buildBrowserEnv(options, profileDir)
330
331 self.automation.log.info("REFTEST INFO | runreftest.py | Running tests: start.\n")
332 status = self.automation.runApp(None, browserEnv, options.app, profileDir,
333 cmdlineArgs,
334 utilityPath = options.utilityPath,
335 xrePath=options.xrePath,
336 debuggerInfo=debuggerInfo,
337 symbolsPath=options.symbolsPath,
338 # give the JS harness 30 seconds to deal
339 # with its own timeouts
340 timeout=options.timeout + 30.0)
341 processLeakLog(self.leakLogFile, options.leakThreshold)
342 self.automation.log.info("\nREFTEST INFO | runreftest.py | Running tests: end.")
343 finally:
344 self.cleanup(profileDir)
345 return status
346
347 def copyExtraFilesToProfile(self, options, profile):
348 "Copy extra files or dirs specified on the command line to the testing profile."
349 profileDir = profile.profile
350 for f in options.extraProfileFiles:
351 abspath = self.getFullPath(f)
352 if os.path.isfile(abspath):
353 if os.path.basename(abspath) == 'user.js':
354 extra_prefs = mozprofile.Preferences.read_prefs(abspath)
355 profile.set_preferences(extra_prefs)
356 else:
357 shutil.copy2(abspath, profileDir)
358 elif os.path.isdir(abspath):
359 dest = os.path.join(profileDir, os.path.basename(abspath))
360 shutil.copytree(abspath, dest)
361 else:
362 self.automation.log.warning("WARNING | runreftest.py | Failed to copy %s to profile", abspath)
363 continue
364
365
366 class ReftestOptions(OptionParser):
367
368 def __init__(self, automation=None):
369 self.automation = automation or Automation()
370 OptionParser.__init__(self)
371 defaults = {}
372
373 # we want to pass down everything from automation.__all__
374 addCommonOptions(self,
375 defaults=dict(zip(self.automation.__all__,
376 [getattr(self.automation, x) for x in self.automation.__all__])))
377 self.automation.addCommonOptions(self)
378 self.add_option("--appname",
379 action = "store", type = "string", dest = "app",
380 default = os.path.join(SCRIPT_DIRECTORY, automation.DEFAULT_APP),
381 help = "absolute path to application, overriding default")
382 self.add_option("--extra-profile-file",
383 action = "append", dest = "extraProfileFiles",
384 default = [],
385 help = "copy specified files/dirs to testing profile")
386 self.add_option("--timeout",
387 action = "store", dest = "timeout", type = "int",
388 default = 5 * 60, # 5 minutes per bug 479518
389 help = "reftest will timeout in specified number of seconds. [default %default s].")
390 self.add_option("--leak-threshold",
391 action = "store", type = "int", dest = "leakThreshold",
392 default = 0,
393 help = "fail if the number of bytes leaked through "
394 "refcounted objects (or bytes in classes with "
395 "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "
396 "than the given number")
397 self.add_option("--utility-path",
398 action = "store", type = "string", dest = "utilityPath",
399 default = self.automation.DIST_BIN,
400 help = "absolute path to directory containing utility "
401 "programs (xpcshell, ssltunnel, certutil)")
402 defaults["utilityPath"] = self.automation.DIST_BIN
403
404 self.add_option("--total-chunks",
405 type = "int", dest = "totalChunks",
406 help = "how many chunks to split the tests up into")
407 defaults["totalChunks"] = None
408
409 self.add_option("--this-chunk",
410 type = "int", dest = "thisChunk",
411 help = "which chunk to run between 1 and --total-chunks")
412 defaults["thisChunk"] = None
413
414 self.add_option("--log-file",
415 action = "store", type = "string", dest = "logFile",
416 default = None,
417 help = "file to log output to in addition to stdout")
418 defaults["logFile"] = None
419
420 self.add_option("--skip-slow-tests",
421 dest = "skipSlowTests", action = "store_true",
422 help = "skip tests marked as slow when running")
423 defaults["skipSlowTests"] = False
424
425 self.add_option("--ignore-window-size",
426 dest = "ignoreWindowSize", action = "store_true",
427 help = "ignore the window size, which may cause spurious failures and passes")
428 defaults["ignoreWindowSize"] = False
429
430 self.add_option("--install-extension",
431 action = "append", dest = "extensionsToInstall",
432 help = "install the specified extension in the testing profile. "
433 "The extension file's name should be <id>.xpi where <id> is "
434 "the extension's id as indicated in its install.rdf. "
435 "An optional path can be specified too.")
436 defaults["extensionsToInstall"] = []
437
438 self.add_option("--run-tests-in-parallel",
439 action = "store_true", dest = "runTestsInParallel",
440 help = "run tests in parallel if possible")
441 self.add_option("--no-run-tests-in-parallel",
442 action = "store_false", dest = "runTestsInParallel",
443 help = "do not run tests in parallel")
444 defaults["runTestsInParallel"] = False
445
446 self.add_option("--setenv",
447 action = "append", type = "string",
448 dest = "environment", metavar = "NAME=VALUE",
449 help = "sets the given variable in the application's "
450 "environment")
451 defaults["environment"] = []
452
453 self.add_option("--filter",
454 action = "store", type="string", dest = "filter",
455 help = "specifies a regular expression (as could be passed to the JS "
456 "RegExp constructor) to test against URLs in the reftest manifest; "
457 "only test items that have a matching test URL will be run.")
458 defaults["filter"] = None
459
460 self.add_option("--shuffle",
461 action = "store_true", dest = "shuffle",
462 help = "run reftests in random order")
463 defaults["shuffle"] = False
464
465 self.add_option("--focus-filter-mode",
466 action = "store", type = "string", dest = "focusFilterMode",
467 help = "filters tests to run by whether they require focus. "
468 "Valid values are `all', `needs-focus', or `non-needs-focus'. "
469 "Defaults to `all'.")
470 defaults["focusFilterMode"] = "all"
471
472 self.add_option("--e10s",
473 action = "store_true",
474 dest = "e10s",
475 help = "enables content processes")
476 defaults["e10s"] = False
477
478 self.set_defaults(**defaults)
479
480 def verifyCommonOptions(self, options, reftest):
481 if options.totalChunks is not None and options.thisChunk is None:
482 self.error("thisChunk must be specified when totalChunks is specified")
483
484 if options.totalChunks:
485 if not 1 <= options.thisChunk <= options.totalChunks:
486 self.error("thisChunk must be between 1 and totalChunks")
487
488 if options.logFile:
489 options.logFile = reftest.getFullPath(options.logFile)
490
491 if options.xrePath is not None:
492 if not os.access(options.xrePath, os.F_OK):
493 self.error("--xre-path '%s' not found" % options.xrePath)
494 if not os.path.isdir(options.xrePath):
495 self.error("--xre-path '%s' is not a directory" % options.xrePath)
496 options.xrePath = reftest.getFullPath(options.xrePath)
497
498 if options.runTestsInParallel:
499 if options.logFile is not None:
500 self.error("cannot specify logfile with parallel tests")
501 if options.totalChunks is not None and options.thisChunk is None:
502 self.error("cannot specify thisChunk or totalChunks with parallel tests")
503 if options.focusFilterMode != "all":
504 self.error("cannot specify focusFilterMode with parallel tests")
505 if options.debugger is not None:
506 self.error("cannot specify a debugger with parallel tests")
507
508 return options
509
510 def main():
511 automation = Automation()
512 parser = ReftestOptions(automation)
513 reftest = RefTest(automation)
514
515 options, args = parser.parse_args()
516 if len(args) != 1:
517 print >>sys.stderr, "No reftest.list specified."
518 sys.exit(1)
519
520 options = parser.verifyCommonOptions(options, reftest)
521
522 options.app = reftest.getFullPath(options.app)
523 if not os.path.exists(options.app):
524 print """Error: Path %(app)s doesn't exist.
525 Are you executing $objdir/_tests/reftest/runreftest.py?""" \
526 % {"app": options.app}
527 sys.exit(1)
528
529 if options.xrePath is None:
530 options.xrePath = os.path.dirname(options.app)
531
532 if options.symbolsPath and not isURL(options.symbolsPath):
533 options.symbolsPath = reftest.getFullPath(options.symbolsPath)
534 options.utilityPath = reftest.getFullPath(options.utilityPath)
535
536 sys.exit(reftest.runTests(args[0], options))
537
538 if __name__ == "__main__":
539 main()

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial