testing/xpcshell/runxpcshelltests.py@7e26c7da4463
testing/xpcshell/runxpcshelltests.py
Wed, 31 Dec 2014 06:55:50 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:55:50 +0100
- changeset 2
- 7e26c7da4463
- permissions
- -rwxr-xr-x
Added tag UPSTREAM_283F7C6 for changeset ca08bd8f51b2
1 #!/usr/bin/env python
2 #
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 import copy
8 import json
9 import math
10 import os
11 import os.path
12 import random
13 import re
14 import shutil
15 import signal
16 import socket
17 import sys
18 import time
19 import traceback
20 import xml.dom.minidom
21 from collections import deque
22 from distutils import dir_util
23 from multiprocessing import cpu_count
24 from optparse import OptionParser
25 from subprocess import Popen, PIPE, STDOUT
26 from tempfile import mkdtemp, gettempdir
27 from threading import Timer, Thread, Event, RLock
29 try:
30 import psutil
31 HAVE_PSUTIL = True
32 except ImportError:
33 HAVE_PSUTIL = False
35 from automation import Automation, getGlobalLog, resetGlobalLog
36 from automationutils import *
38 # Printing buffered output in case of a failure or verbose mode will result
39 # in buffered output interleaved with other threads' output.
40 # To prevent his, each call to the logger as well as any blocks of output that
41 # are intended to be continuous are protected by the same lock.
42 LOG_MUTEX = RLock()
44 HARNESS_TIMEOUT = 5 * 60
46 # benchmarking on tbpl revealed that this works best for now
47 NUM_THREADS = int(cpu_count() * 4)
49 FAILURE_ACTIONS = set(['test_unexpected_fail',
50 'test_unexpected_pass',
51 'javascript_error'])
52 ACTION_STRINGS = {
53 "test_unexpected_fail": "TEST-UNEXPECTED-FAIL",
54 "test_known_fail": "TEST-KNOWN-FAIL",
55 "test_unexpected_pass": "TEST-UNEXPECTED-PASS",
56 "javascript_error": "TEST-UNEXPECTED-FAIL",
57 "test_pass": "TEST-PASS",
58 "test_info": "TEST-INFO"
59 }
61 # --------------------------------------------------------------
62 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
63 #
64 here = os.path.dirname(__file__)
65 mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
67 if os.path.isdir(mozbase):
68 for package in os.listdir(mozbase):
69 sys.path.append(os.path.join(mozbase, package))
71 import manifestparser
72 import mozcrash
73 import mozinfo
75 # --------------------------------------------------------------
77 # TODO: perhaps this should be in a more generally shared location?
78 # This regex matches all of the C0 and C1 control characters
79 # (U+0000 through U+001F; U+007F; U+0080 through U+009F),
80 # except TAB (U+0009), CR (U+000D), LF (U+000A) and backslash (U+005C).
81 # A raw string is deliberately not used.
82 _cleanup_encoding_re = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]')
83 def _cleanup_encoding_repl(m):
84 c = m.group(0)
85 return '\\\\' if c == '\\' else '\\x{0:02X}'.format(ord(c))
86 def cleanup_encoding(s):
87 """S is either a byte or unicode string. Either way it may
88 contain control characters, unpaired surrogates, reserved code
89 points, etc. If it is a byte string, it is assumed to be
90 UTF-8, but it may not be *correct* UTF-8. Produce a byte
91 string that can safely be dumped into a (generally UTF-8-coded)
92 logfile."""
93 if not isinstance(s, unicode):
94 s = s.decode('utf-8', 'replace')
95 if s.endswith('\n'):
96 # A new line is always added by head.js to delimit messages,
97 # however consumers will want to supply their own.
98 s = s[:-1]
99 # Replace all C0 and C1 control characters with \xNN escapes.
100 s = _cleanup_encoding_re.sub(_cleanup_encoding_repl, s)
101 return s.encode('utf-8', 'backslashreplace')
103 """ Control-C handling """
104 gotSIGINT = False
105 def markGotSIGINT(signum, stackFrame):
106 global gotSIGINT
107 gotSIGINT = True
109 class XPCShellTestThread(Thread):
110 def __init__(self, test_object, event, cleanup_dir_list, retry=True,
111 tests_root_dir=None, app_dir_key=None, interactive=False,
112 verbose=False, pStdout=None, pStderr=None, keep_going=False,
113 log=None, **kwargs):
114 Thread.__init__(self)
115 self.daemon = True
117 self.test_object = test_object
118 self.cleanup_dir_list = cleanup_dir_list
119 self.retry = retry
121 self.appPath = kwargs.get('appPath')
122 self.xrePath = kwargs.get('xrePath')
123 self.testingModulesDir = kwargs.get('testingModulesDir')
124 self.debuggerInfo = kwargs.get('debuggerInfo')
125 self.pluginsPath = kwargs.get('pluginsPath')
126 self.httpdManifest = kwargs.get('httpdManifest')
127 self.httpdJSPath = kwargs.get('httpdJSPath')
128 self.headJSPath = kwargs.get('headJSPath')
129 self.testharnessdir = kwargs.get('testharnessdir')
130 self.profileName = kwargs.get('profileName')
131 self.singleFile = kwargs.get('singleFile')
132 self.env = copy.deepcopy(kwargs.get('env'))
133 self.symbolsPath = kwargs.get('symbolsPath')
134 self.logfiles = kwargs.get('logfiles')
135 self.xpcshell = kwargs.get('xpcshell')
136 self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
137 self.failureManifest = kwargs.get('failureManifest')
138 self.on_message = kwargs.get('on_message')
140 self.tests_root_dir = tests_root_dir
141 self.app_dir_key = app_dir_key
142 self.interactive = interactive
143 self.verbose = verbose
144 self.pStdout = pStdout
145 self.pStderr = pStderr
146 self.keep_going = keep_going
147 self.log = log
149 # only one of these will be set to 1. adding them to the totals in
150 # the harness
151 self.passCount = 0
152 self.todoCount = 0
153 self.failCount = 0
155 self.output_lines = []
156 self.has_failure_output = False
157 self.saw_proc_start = False
158 self.saw_proc_end = False
160 # event from main thread to signal work done
161 self.event = event
162 self.done = False # explicitly set flag so we don't rely on thread.isAlive
164 def run(self):
165 try:
166 self.run_test()
167 except Exception as e:
168 self.exception = e
169 self.traceback = traceback.format_exc()
170 else:
171 self.exception = None
172 self.traceback = None
173 if self.retry:
174 self.log.info("TEST-INFO | %s | Test failed or timed out, will retry."
175 % self.test_object['name'])
176 self.done = True
177 self.event.set()
179 def kill(self, proc):
180 """
181 Simple wrapper to kill a process.
182 On a remote system, this is overloaded to handle remote process communication.
183 """
184 return proc.kill()
186 def removeDir(self, dirname):
187 """
188 Simple wrapper to remove (recursively) a given directory.
189 On a remote system, we need to overload this to work on the remote filesystem.
190 """
191 shutil.rmtree(dirname)
193 def poll(self, proc):
194 """
195 Simple wrapper to check if a process has terminated.
196 On a remote system, this is overloaded to handle remote process communication.
197 """
198 return proc.poll()
200 def createLogFile(self, test_file, stdout):
201 """
202 For a given test file and stdout buffer, create a log file.
203 On a remote system we have to fix the test name since it can contain directories.
204 """
205 with open(test_file + ".log", "w") as f:
206 f.write(stdout)
208 def getReturnCode(self, proc):
209 """
210 Simple wrapper to get the return code for a given process.
211 On a remote system we overload this to work with the remote process management.
212 """
213 return proc.returncode
215 def communicate(self, proc):
216 """
217 Simple wrapper to communicate with a process.
218 On a remote system, this is overloaded to handle remote process communication.
219 """
220 # Processing of incremental output put here to
221 # sidestep issues on remote platforms, where what we know
222 # as proc is a file pulled off of a device.
223 if proc.stdout:
224 while True:
225 line = proc.stdout.readline()
226 if not line:
227 break
228 self.process_line(line)
230 if self.saw_proc_start and not self.saw_proc_end:
231 self.has_failure_output = True
233 return proc.communicate()
235 def launchProcess(self, cmd, stdout, stderr, env, cwd):
236 """
237 Simple wrapper to launch a process.
238 On a remote system, this is more complex and we need to overload this function.
239 """
240 if HAVE_PSUTIL:
241 popen_func = psutil.Popen
242 else:
243 popen_func = Popen
244 proc = popen_func(cmd, stdout=stdout, stderr=stderr,
245 env=env, cwd=cwd)
246 return proc
248 def checkForCrashes(self,
249 dump_directory,
250 symbols_path,
251 test_name=None):
252 """
253 Simple wrapper to check for crashes.
254 On a remote system, this is more complex and we need to overload this function.
255 """
256 return mozcrash.check_for_crashes(dump_directory, symbols_path, test_name=test_name)
258 def logCommand(self, name, completeCmd, testdir):
259 self.log.info("TEST-INFO | %s | full command: %r" % (name, completeCmd))
260 self.log.info("TEST-INFO | %s | current directory: %r" % (name, testdir))
261 # Show only those environment variables that are changed from
262 # the ambient environment.
263 changedEnv = (set("%s=%s" % i for i in self.env.iteritems())
264 - set("%s=%s" % i for i in os.environ.iteritems()))
265 self.log.info("TEST-INFO | %s | environment: %s" % (name, list(changedEnv)))
267 def testTimeout(self, test_file, proc):
268 if not self.retry:
269 self.log.error("TEST-UNEXPECTED-FAIL | %s | Test timed out" % test_file)
270 self.done = True
271 Automation().killAndGetStackNoScreenshot(proc.pid, self.appPath, self.debuggerInfo)
273 def buildCmdTestFile(self, name):
274 """
275 Build the command line arguments for the test file.
276 On a remote system, this may be overloaded to use a remote path structure.
277 """
278 return ['-e', 'const _TEST_FILE = ["%s"];' %
279 replaceBackSlashes(name)]
281 def setupTempDir(self):
282 tempDir = mkdtemp()
283 self.env["XPCSHELL_TEST_TEMP_DIR"] = tempDir
284 if self.interactive:
285 self.log.info("TEST-INFO | temp dir is %s" % tempDir)
286 return tempDir
288 def setupPluginsDir(self):
289 if not os.path.isdir(self.pluginsPath):
290 return None
292 pluginsDir = mkdtemp()
293 # shutil.copytree requires dst to not exist. Deleting the tempdir
294 # would make a race condition possible in a concurrent environment,
295 # so we are using dir_utils.copy_tree which accepts an existing dst
296 dir_util.copy_tree(self.pluginsPath, pluginsDir)
297 if self.interactive:
298 self.log.info("TEST-INFO | plugins dir is %s" % pluginsDir)
299 return pluginsDir
301 def setupProfileDir(self):
302 """
303 Create a temporary folder for the profile and set appropriate environment variables.
304 When running check-interactive and check-one, the directory is well-defined and
305 retained for inspection once the tests complete.
307 On a remote system, this may be overloaded to use a remote path structure.
308 """
309 if self.interactive or self.singleFile:
310 profileDir = os.path.join(gettempdir(), self.profileName, "xpcshellprofile")
311 try:
312 # This could be left over from previous runs
313 self.removeDir(profileDir)
314 except:
315 pass
316 os.makedirs(profileDir)
317 else:
318 profileDir = mkdtemp()
319 self.env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
320 if self.interactive or self.singleFile:
321 self.log.info("TEST-INFO | profile dir is %s" % profileDir)
322 return profileDir
324 def buildCmdHead(self, headfiles, tailfiles, xpcscmd):
325 """
326 Build the command line arguments for the head and tail files,
327 along with the address of the webserver which some tests require.
329 On a remote system, this is overloaded to resolve quoting issues over a secondary command line.
330 """
331 cmdH = ", ".join(['"' + replaceBackSlashes(f) + '"'
332 for f in headfiles])
333 cmdT = ", ".join(['"' + replaceBackSlashes(f) + '"'
334 for f in tailfiles])
335 return xpcscmd + \
336 ['-e', 'const _SERVER_ADDR = "localhost"',
337 '-e', 'const _HEAD_FILES = [%s];' % cmdH,
338 '-e', 'const _TAIL_FILES = [%s];' % cmdT]
340 def getHeadAndTailFiles(self, test_object):
341 """Obtain the list of head and tail files.
343 Returns a 2-tuple. The first element is a list of head files. The second
344 is a list of tail files.
345 """
346 def sanitize_list(s, kind):
347 for f in s.strip().split(' '):
348 f = f.strip()
349 if len(f) < 1:
350 continue
352 path = os.path.normpath(os.path.join(test_object['here'], f))
353 if not os.path.exists(path):
354 raise Exception('%s file does not exist: %s' % (kind, path))
356 if not os.path.isfile(path):
357 raise Exception('%s file is not a file: %s' % (kind, path))
359 yield path
361 return (list(sanitize_list(test_object['head'], 'head')),
362 list(sanitize_list(test_object['tail'], 'tail')))
364 def buildXpcsCmd(self, testdir):
365 """
366 Load the root head.js file as the first file in our test path, before other head, test, and tail files.
367 On a remote system, we overload this to add additional command line arguments, so this gets overloaded.
368 """
369 # - NOTE: if you rename/add any of the constants set here, update
370 # do_load_child_test_harness() in head.js
371 if not self.appPath:
372 self.appPath = self.xrePath
374 self.xpcsCmd = [
375 self.xpcshell,
376 '-g', self.xrePath,
377 '-a', self.appPath,
378 '-r', self.httpdManifest,
379 '-m',
380 '-s',
381 '-e', 'const _HTTPD_JS_PATH = "%s";' % self.httpdJSPath,
382 '-e', 'const _HEAD_JS_PATH = "%s";' % self.headJSPath
383 ]
385 if self.testingModulesDir:
386 # Escape backslashes in string literal.
387 sanitized = self.testingModulesDir.replace('\\', '\\\\')
388 self.xpcsCmd.extend([
389 '-e',
390 'const _TESTING_MODULES_DIR = "%s";' % sanitized
391 ])
393 self.xpcsCmd.extend(['-f', os.path.join(self.testharnessdir, 'head.js')])
395 if self.debuggerInfo:
396 self.xpcsCmd = [self.debuggerInfo["path"]] + self.debuggerInfo["args"] + self.xpcsCmd
398 # Automation doesn't specify a pluginsPath and xpcshell defaults to
399 # $APPDIR/plugins. We do the same here so we can carry on with
400 # setting up every test with its own plugins directory.
401 if not self.pluginsPath:
402 self.pluginsPath = os.path.join(self.appPath, 'plugins')
404 self.pluginsDir = self.setupPluginsDir()
405 if self.pluginsDir:
406 self.xpcsCmd.extend(['-p', self.pluginsDir])
408 def cleanupDir(self, directory, name, xunit_result):
409 if not os.path.exists(directory):
410 return
412 TRY_LIMIT = 25 # up to TRY_LIMIT attempts (one every second), because
413 # the Windows filesystem is slow to react to the changes
414 try_count = 0
415 while try_count < TRY_LIMIT:
416 try:
417 self.removeDir(directory)
418 except OSError:
419 self.log.info("TEST-INFO | Failed to remove directory: %s. Waiting." % directory)
420 # We suspect the filesystem may still be making changes. Wait a
421 # little bit and try again.
422 time.sleep(1)
423 try_count += 1
424 else:
425 # removed fine
426 return
428 # we try cleaning up again later at the end of the run
429 self.cleanup_dir_list.append(directory)
431 def clean_temp_dirs(self, name, stdout):
432 # We don't want to delete the profile when running check-interactive
433 # or check-one.
434 if self.profileDir and not self.interactive and not self.singleFile:
435 self.cleanupDir(self.profileDir, name, self.xunit_result)
437 self.cleanupDir(self.tempDir, name, self.xunit_result)
439 if self.pluginsDir:
440 self.cleanupDir(self.pluginsDir, name, self.xunit_result)
442 def message_from_line(self, line):
443 """ Given a line of raw output, convert to a string message. """
444 if isinstance(line, basestring):
445 # This function has received unstructured output.
446 if line:
447 if 'TEST-UNEXPECTED-' in line:
448 self.has_failure_output = True
449 return line
451 msg = ['%s: ' % line['process'] if 'process' in line else '']
453 # Each call to the logger in head.js either specified '_message'
454 # or both 'source_file' and 'diagnostic'. If either of these are
455 # missing, they ended up being undefined as a result of the way
456 # the test was run.
457 if '_message' in line:
458 msg.append(line['_message'])
459 if 'diagnostic' in line:
460 msg.append('\nDiagnostic: %s' % line['diagnostic'])
461 else:
462 msg.append('%s | %s | %s' % (ACTION_STRINGS[line['action']],
463 line.get('source_file', 'undefined'),
464 line.get('diagnostic', 'undefined')))
466 msg.append('\n%s' % line['stack'] if 'stack' in line else '')
467 return ''.join(msg)
469 def parse_output(self, output):
470 """Parses process output for structured messages and saves output as it is
471 read. Sets self.has_failure_output in case of evidence of a failure"""
472 for line_string in output.splitlines():
473 self.process_line(line_string)
475 if self.saw_proc_start and not self.saw_proc_end:
476 self.has_failure_output = True
478 def report_message(self, line):
479 """ Reports a message to a consumer, both as a strucutured and
480 human-readable log message. """
482 message = cleanup_encoding(self.message_from_line(line))
483 if message.endswith('\n'):
484 # A new line is always added by head.js to delimit messages,
485 # however consumers will want to supply their own.
486 message = message[:-1]
488 if self.on_message:
489 self.on_message(line, message)
490 else:
491 self.output_lines.append(message)
493 def process_line(self, line_string):
494 """ Parses a single line of output, determining its significance and
495 reporting a message.
496 """
497 try:
498 line_object = json.loads(line_string)
499 if not isinstance(line_object, dict):
500 self.report_message(line_string)
501 return
502 except ValueError:
503 self.report_message(line_string)
504 return
506 if 'action' not in line_object:
507 # In case a test outputs something that happens to be valid
508 # JSON.
509 self.report_message(line_string)
510 return
512 action = line_object['action']
513 self.report_message(line_object)
515 if action in FAILURE_ACTIONS:
516 self.has_failure_output = True
517 elif action == 'child_test_start':
518 self.saw_proc_start = True
519 elif action == 'child_test_end':
520 self.saw_proc_end = True
522 def log_output(self, output):
523 """Prints given output line-by-line to avoid overflowing buffers."""
524 self.log.info(">>>>>>>")
525 if output:
526 if isinstance(output, basestring):
527 output = output.splitlines()
528 for part in output:
529 # For multi-line output, such as a stack trace
530 for line in part.splitlines():
531 try:
532 line = line.decode('utf-8')
533 except UnicodeDecodeError:
534 self.log.info("TEST-INFO | %s | Detected non UTF-8 output."\
535 " Please modify the test to only print UTF-8." %
536 self.test_object['name'])
537 # add '?' instead of funky bytes
538 line = line.decode('utf-8', 'replace')
539 self.log.info(line)
540 self.log.info("<<<<<<<")
542 def run_test(self):
543 """Run an individual xpcshell test."""
544 global gotSIGINT
546 name = self.test_object['path']
548 self.xunit_result = {'name': self.test_object['name'], 'classname': 'xpcshell'}
550 # The xUnit package is defined as the path component between the root
551 # dir and the test with path characters replaced with '.' (using Java
552 # class notation).
553 if self.tests_root_dir is not None:
554 self.tests_root_dir = os.path.normpath(self.tests_root_dir)
555 if os.path.normpath(self.test_object['here']).find(self.tests_root_dir) != 0:
556 raise Exception('tests_root_dir is not a parent path of %s' %
557 self.test_object['here'])
558 relpath = self.test_object['here'][len(self.tests_root_dir):].lstrip('/\\')
559 self.xunit_result['classname'] = relpath.replace('/', '.').replace('\\', '.')
561 # Check for skipped tests
562 if 'disabled' in self.test_object:
563 self.log.info('TEST-INFO | skipping %s | %s' %
564 (name, self.test_object['disabled']))
566 self.xunit_result['skipped'] = True
567 self.retry = False
569 self.keep_going = True
570 return
572 # Check for known-fail tests
573 expected = self.test_object['expected'] == 'pass'
575 # By default self.appPath will equal the gre dir. If specified in the
576 # xpcshell.ini file, set a different app dir for this test.
577 if self.app_dir_key and self.app_dir_key in self.test_object:
578 rel_app_dir = self.test_object[self.app_dir_key]
579 rel_app_dir = os.path.join(self.xrePath, rel_app_dir)
580 self.appPath = os.path.abspath(rel_app_dir)
581 else:
582 self.appPath = None
584 test_dir = os.path.dirname(name)
585 self.buildXpcsCmd(test_dir)
586 head_files, tail_files = self.getHeadAndTailFiles(self.test_object)
587 cmdH = self.buildCmdHead(head_files, tail_files, self.xpcsCmd)
589 # Create a profile and a temp dir that the JS harness can stick
590 # a profile and temporary data in
591 self.profileDir = self.setupProfileDir()
592 self.tempDir = self.setupTempDir()
594 # The test file will have to be loaded after the head files.
595 cmdT = self.buildCmdTestFile(name)
597 args = self.xpcsRunArgs[:]
598 if 'debug' in self.test_object:
599 args.insert(0, '-d')
601 completeCmd = cmdH + cmdT + args
603 testTimeoutInterval = HARNESS_TIMEOUT
604 # Allow a test to request a multiple of the timeout if it is expected to take long
605 if 'requesttimeoutfactor' in self.test_object:
606 testTimeoutInterval *= int(self.test_object['requesttimeoutfactor'])
608 testTimer = None
609 if not self.interactive and not self.debuggerInfo:
610 testTimer = Timer(testTimeoutInterval, lambda: self.testTimeout(name, proc))
611 testTimer.start()
613 proc = None
614 stdout = None
615 stderr = None
617 try:
618 self.log.info("TEST-INFO | %s | running test ..." % name)
619 if self.verbose:
620 self.logCommand(name, completeCmd, test_dir)
622 startTime = time.time()
623 proc = self.launchProcess(completeCmd,
624 stdout=self.pStdout, stderr=self.pStderr, env=self.env, cwd=test_dir)
626 if self.interactive:
627 self.log.info("TEST-INFO | %s | Process ID: %d" % (name, proc.pid))
629 stdout, stderr = self.communicate(proc)
631 if self.interactive:
632 # Not sure what else to do here...
633 self.keep_going = True
634 return
636 if testTimer:
637 testTimer.cancel()
639 if stdout:
640 self.parse_output(stdout)
641 result = not (self.has_failure_output or
642 (self.getReturnCode(proc) != 0))
644 if result != expected:
645 if self.retry:
646 self.clean_temp_dirs(name, stdout)
647 return
649 failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS")
650 message = "%s | %s | test failed (with xpcshell return code: %d)" % (
651 failureType, name, self.getReturnCode(proc))
652 if self.output_lines:
653 message += ", see following log:"
655 with LOG_MUTEX:
656 self.log.error(message)
657 self.log_output(self.output_lines)
659 self.failCount += 1
660 self.xunit_result["passed"] = False
662 self.xunit_result["failure"] = {
663 "type": failureType,
664 "message": message,
665 "text": stdout
666 }
668 if self.failureManifest:
669 with open(self.failureManifest, 'a') as f:
670 f.write('[%s]\n' % self.test_object['path'])
671 for k, v in self.test_object.items():
672 f.write('%s = %s\n' % (k, v))
674 else:
675 now = time.time()
676 timeTaken = (now - startTime) * 1000
677 self.xunit_result["time"] = now - startTime
679 with LOG_MUTEX:
680 self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken))
681 if self.verbose:
682 self.log_output(self.output_lines)
684 self.xunit_result["passed"] = True
685 self.retry = False
687 if expected:
688 self.passCount = 1
689 else:
690 self.todoCount = 1
691 self.xunit_result["todo"] = True
693 if self.checkForCrashes(self.tempDir, self.symbolsPath, test_name=name):
694 if self.retry:
695 self.clean_temp_dirs(name, stdout)
696 return
698 message = "PROCESS-CRASH | %s | application crashed" % name
699 self.failCount = 1
700 self.xunit_result["passed"] = False
701 self.xunit_result["failure"] = {
702 "type": "PROCESS-CRASH",
703 "message": message,
704 "text": stdout
705 }
707 if self.logfiles and stdout:
708 self.createLogFile(name, stdout)
710 finally:
711 # We can sometimes get here before the process has terminated, which would
712 # cause removeDir() to fail - so check for the process & kill it it needed.
713 if proc and self.poll(proc) is None:
714 self.kill(proc)
716 if self.retry:
717 self.clean_temp_dirs(name, stdout)
718 return
720 with LOG_MUTEX:
721 message = "TEST-UNEXPECTED-FAIL | %s | Process still running after test!" % name
722 self.log.error(message)
723 self.log_output(self.output_lines)
725 self.failCount = 1
726 self.xunit_result["passed"] = False
727 self.xunit_result["failure"] = {
728 "type": "TEST-UNEXPECTED-FAIL",
729 "message": message,
730 "text": stdout
731 }
733 self.clean_temp_dirs(name, stdout)
735 if gotSIGINT:
736 self.xunit_result["passed"] = False
737 self.xunit_result["time"] = "0.0"
738 self.xunit_result["failure"] = {
739 "type": "SIGINT",
740 "message": "Received SIGINT",
741 "text": "Received SIGINT (control-C) during test execution."
742 }
744 self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution")
745 if self.keep_going:
746 gotSIGINT = False
747 else:
748 self.keep_going = False
749 return
751 self.keep_going = True
753 class XPCShellTests(object):
755 log = getGlobalLog()
756 oldcwd = os.getcwd()
758 def __init__(self, log=None):
759 """ Init logging and node status """
760 if log:
761 resetGlobalLog(log)
763 # Each method of the underlying logger must acquire the log
764 # mutex before writing to stdout.
765 log_funs = ['debug', 'info', 'warning', 'error', 'critical', 'log']
766 for fun_name in log_funs:
767 unwrapped = getattr(self.log, fun_name, None)
768 if unwrapped:
769 def wrap(fn):
770 def wrapped(*args, **kwargs):
771 with LOG_MUTEX:
772 fn(*args, **kwargs)
773 return wrapped
774 setattr(self.log, fun_name, wrap(unwrapped))
776 self.nodeProc = {}
778 def buildTestList(self):
779 """
780 read the xpcshell.ini manifest and set self.alltests to be
781 an array of test objects.
783 if we are chunking tests, it will be done here as well
784 """
785 if isinstance(self.manifest, manifestparser.TestManifest):
786 mp = self.manifest
787 else:
788 mp = manifestparser.TestManifest(strict=False)
789 if self.manifest is None:
790 for testdir in self.testdirs:
791 if testdir:
792 mp.read(os.path.join(testdir, 'xpcshell.ini'))
793 else:
794 mp.read(self.manifest)
796 self.buildTestPath()
798 try:
799 self.alltests = mp.active_tests(**mozinfo.info)
800 except TypeError:
801 sys.stderr.write("*** offending mozinfo.info: %s\n" % repr(mozinfo.info))
802 raise
804 if self.singleFile is None and self.totalChunks > 1:
805 self.chunkTests()
807 def chunkTests(self):
808 """
809 Split the list of tests up into [totalChunks] pieces and filter the
810 self.alltests based on thisChunk, so we only run a subset.
811 """
812 totalTests = len(self.alltests)
813 testsPerChunk = math.ceil(totalTests / float(self.totalChunks))
814 start = int(round((self.thisChunk-1) * testsPerChunk))
815 end = int(start + testsPerChunk)
816 if end > totalTests:
817 end = totalTests
818 self.log.info("Running tests %d-%d/%d", start+1, end, totalTests)
819 self.alltests = self.alltests[start:end]
821 def setAbsPath(self):
822 """
823 Set the absolute path for xpcshell, httpdjspath and xrepath.
824 These 3 variables depend on input from the command line and we need to allow for absolute paths.
825 This function is overloaded for a remote solution as os.path* won't work remotely.
826 """
827 self.testharnessdir = os.path.dirname(os.path.abspath(__file__))
828 self.headJSPath = self.testharnessdir.replace("\\", "/") + "/head.js"
829 self.xpcshell = os.path.abspath(self.xpcshell)
831 # we assume that httpd.js lives in components/ relative to xpcshell
832 self.httpdJSPath = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.js')
833 self.httpdJSPath = replaceBackSlashes(self.httpdJSPath)
835 self.httpdManifest = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.manifest')
836 self.httpdManifest = replaceBackSlashes(self.httpdManifest)
838 if self.xrePath is None:
839 self.xrePath = os.path.dirname(self.xpcshell)
840 else:
841 self.xrePath = os.path.abspath(self.xrePath)
843 if self.mozInfo is None:
844 self.mozInfo = os.path.join(self.testharnessdir, "mozinfo.json")
846 def buildCoreEnvironment(self):
847 """
848 Add environment variables likely to be used across all platforms, including remote systems.
849 """
850 # Make assertions fatal
851 self.env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
852 # Crash reporting interferes with debugging
853 if not self.debuggerInfo:
854 self.env["MOZ_CRASHREPORTER"] = "1"
855 # Don't launch the crash reporter client
856 self.env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
857 # Capturing backtraces is very slow on some platforms, and it's
858 # disabled by automation.py too
859 self.env["NS_TRACE_MALLOC_DISABLE_STACKS"] = "1"
860 # Don't permit remote connections.
861 self.env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
863 def buildEnvironment(self):
864 """
865 Create and returns a dictionary of self.env to include all the appropriate env variables and values.
866 On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
867 """
868 self.env = dict(os.environ)
869 self.buildCoreEnvironment()
870 if sys.platform == 'win32':
871 self.env["PATH"] = self.env["PATH"] + ";" + self.xrePath
872 elif sys.platform in ('os2emx', 'os2knix'):
873 os.environ["BEGINLIBPATH"] = self.xrePath + ";" + self.env["BEGINLIBPATH"]
874 os.environ["LIBPATHSTRICT"] = "T"
875 elif sys.platform == 'osx' or sys.platform == "darwin":
876 self.env["DYLD_LIBRARY_PATH"] = self.xrePath
877 else: # unix or linux?
878 if not "LD_LIBRARY_PATH" in self.env or self.env["LD_LIBRARY_PATH"] is None:
879 self.env["LD_LIBRARY_PATH"] = self.xrePath
880 else:
881 self.env["LD_LIBRARY_PATH"] = ":".join([self.xrePath, self.env["LD_LIBRARY_PATH"]])
883 if "asan" in self.mozInfo and self.mozInfo["asan"]:
884 # ASan symbolizer support
885 llvmsym = os.path.join(self.xrePath, "llvm-symbolizer")
886 if os.path.isfile(llvmsym):
887 self.env["ASAN_SYMBOLIZER_PATH"] = llvmsym
888 self.log.info("INFO | runxpcshelltests.py | ASan using symbolizer at %s", llvmsym)
889 else:
890 self.log.info("INFO | runxpcshelltests.py | ASan symbolizer binary not found: %s", llvmsym)
892 return self.env
894 def getPipes(self):
895 """
896 Determine the value of the stdout and stderr for the test.
897 Return value is a list (pStdout, pStderr).
898 """
899 if self.interactive:
900 pStdout = None
901 pStderr = None
902 else:
903 if (self.debuggerInfo and self.debuggerInfo["interactive"]):
904 pStdout = None
905 pStderr = None
906 else:
907 if sys.platform == 'os2emx':
908 pStdout = None
909 else:
910 pStdout = PIPE
911 pStderr = STDOUT
912 return pStdout, pStderr
914 def buildTestPath(self):
915 """
916 If we specifiy a testpath, set the self.testPath variable to be the given directory or file.
918 |testPath| will be the optional path only, or |None|.
919 |singleFile| will be the optional test only, or |None|.
920 """
921 self.singleFile = None
922 if self.testPath is not None:
923 if self.testPath.endswith('.js'):
924 # Split into path and file.
925 if self.testPath.find('/') == -1:
926 # Test only.
927 self.singleFile = self.testPath
928 else:
929 # Both path and test.
930 # Reuse |testPath| temporarily.
931 self.testPath = self.testPath.rsplit('/', 1)
932 self.singleFile = self.testPath[1]
933 self.testPath = self.testPath[0]
934 else:
935 # Path only.
936 # Simply remove optional ending separator.
937 self.testPath = self.testPath.rstrip("/")
939 def verifyDirPath(self, dirname):
940 """
941 Simple wrapper to get the absolute path for a given directory name.
942 On a remote system, we need to overload this to work on the remote filesystem.
943 """
944 return os.path.abspath(dirname)
946 def trySetupNode(self):
947 """
948 Run node for SPDY tests, if available, and updates mozinfo as appropriate.
949 """
950 nodeMozInfo = {'hasNode': False} # Assume the worst
951 nodeBin = None
953 # We try to find the node executable in the path given to us by the user in
954 # the MOZ_NODE_PATH environment variable
955 localPath = os.getenv('MOZ_NODE_PATH', None)
956 if localPath and os.path.exists(localPath) and os.path.isfile(localPath):
957 nodeBin = localPath
959 if nodeBin:
960 self.log.info('Found node at %s' % (nodeBin,))
962 def startServer(name, serverJs):
963 if os.path.exists(serverJs):
964 # OK, we found our SPDY server, let's try to get it running
965 self.log.info('Found %s at %s' % (name, serverJs))
966 try:
967 # We pipe stdin to node because the spdy server will exit when its
968 # stdin reaches EOF
969 process = Popen([nodeBin, serverJs], stdin=PIPE, stdout=PIPE,
970 stderr=STDOUT, env=self.env, cwd=os.getcwd())
971 self.nodeProc[name] = process
973 # Check to make sure the server starts properly by waiting for it to
974 # tell us it's started
975 msg = process.stdout.readline()
976 if 'server listening' in msg:
977 nodeMozInfo['hasNode'] = True
978 except OSError, e:
979 # This occurs if the subprocess couldn't be started
980 self.log.error('Could not run %s server: %s' % (name, str(e)))
982 myDir = os.path.split(os.path.abspath(__file__))[0]
983 startServer('moz-spdy', os.path.join(myDir, 'moz-spdy', 'moz-spdy.js'))
984 startServer('moz-http2', os.path.join(myDir, 'moz-http2', 'moz-http2.js'))
986 mozinfo.update(nodeMozInfo)
988 def shutdownNode(self):
989 """
990 Shut down our node process, if it exists
991 """
992 for name, proc in self.nodeProc.iteritems():
993 self.log.info('Node %s server shutting down ...' % name)
994 proc.terminate()
996 def writeXunitResults(self, results, name=None, filename=None, fh=None):
997 """
998 Write Xunit XML from results.
1000 The function receives an iterable of results dicts. Each dict must have
1001 the following keys:
1003 classname - The "class" name of the test.
1004 name - The simple name of the test.
1006 In addition, it must have one of the following saying how the test
1007 executed:
1009 passed - Boolean indicating whether the test passed. False if it
1010 failed.
1011 skipped - True if the test was skipped.
1013 The following keys are optional:
1015 time - Execution time of the test in decimal seconds.
1016 failure - Dict describing test failure. Requires keys:
1017 type - String type of failure.
1018 message - String describing basic failure.
1019 text - Verbose string describing failure.
1021 Arguments:
1023 |name|, Name of the test suite. Many tools expect Java class dot notation
1024 e.g. dom.simple.foo. A directory with '/' converted to '.' is a good
1025 choice.
1026 |fh|, File handle to write XML to.
1027 |filename|, File name to write XML to.
1028 |results|, Iterable of tuples describing the results.
1029 """
1030 if filename is None and fh is None:
1031 raise Exception("One of filename or fh must be defined.")
1033 if name is None:
1034 name = "xpcshell"
1035 else:
1036 assert isinstance(name, basestring)
1038 if filename is not None:
1039 fh = open(filename, 'wb')
1041 doc = xml.dom.minidom.Document()
1042 testsuite = doc.createElement("testsuite")
1043 testsuite.setAttribute("name", name)
1044 doc.appendChild(testsuite)
1046 total = 0
1047 passed = 0
1048 failed = 0
1049 skipped = 0
1051 for result in results:
1052 total += 1
1054 if result.get("skipped", None):
1055 skipped += 1
1056 elif result["passed"]:
1057 passed += 1
1058 else:
1059 failed += 1
1061 testcase = doc.createElement("testcase")
1062 testcase.setAttribute("classname", result["classname"])
1063 testcase.setAttribute("name", result["name"])
1065 if "time" in result:
1066 testcase.setAttribute("time", str(result["time"]))
1067 else:
1068 # It appears most tools expect the time attribute to be present.
1069 testcase.setAttribute("time", "0")
1071 if "failure" in result:
1072 failure = doc.createElement("failure")
1073 failure.setAttribute("type", str(result["failure"]["type"]))
1074 failure.setAttribute("message", result["failure"]["message"])
1076 # Lossy translation but required to not break CDATA. Also, text could
1077 # be None and Python 2.5's minidom doesn't accept None. Later versions
1078 # do, however.
1079 cdata = result["failure"]["text"]
1080 if not isinstance(cdata, str):
1081 cdata = ""
1083 cdata = cdata.replace("]]>", "]] >")
1084 text = doc.createCDATASection(cdata)
1085 failure.appendChild(text)
1086 testcase.appendChild(failure)
1088 if result.get("skipped", None):
1089 e = doc.createElement("skipped")
1090 testcase.appendChild(e)
1092 testsuite.appendChild(testcase)
1094 testsuite.setAttribute("tests", str(total))
1095 testsuite.setAttribute("failures", str(failed))
1096 testsuite.setAttribute("skip", str(skipped))
1098 doc.writexml(fh, addindent=" ", newl="\n", encoding="utf-8")
1100 def post_to_autolog(self, results, name):
1101 from moztest.results import TestContext, TestResult, TestResultCollection
1102 from moztest.output.autolog import AutologOutput
1104 context = TestContext(
1105 testgroup='b2g xpcshell testsuite',
1106 operating_system='android',
1107 arch='emulator',
1108 harness='xpcshell',
1109 hostname=socket.gethostname(),
1110 tree='b2g',
1111 buildtype='opt',
1112 )
1114 collection = TestResultCollection('b2g emulator testsuite')
1116 for result in results:
1117 duration = result.get('time', 0)
1119 if 'skipped' in result:
1120 outcome = 'SKIPPED'
1121 elif 'todo' in result:
1122 outcome = 'KNOWN-FAIL'
1123 elif result['passed']:
1124 outcome = 'PASS'
1125 else:
1126 outcome = 'UNEXPECTED-FAIL'
1128 output = None
1129 if 'failure' in result:
1130 output = result['failure']['text']
1132 t = TestResult(name=result['name'], test_class=name,
1133 time_start=0, context=context)
1134 t.finish(result=outcome, time_end=duration, output=output)
1136 collection.append(t)
1137 collection.time_taken += duration
1139 out = AutologOutput()
1140 out.post(out.make_testgroups(collection))
1142 def buildXpcsRunArgs(self):
1143 """
1144 Add arguments to run the test or make it interactive.
1145 """
1146 if self.interactive:
1147 self.xpcsRunArgs = [
1148 '-e', 'print("To start the test, type |_execute_test();|.");',
1149 '-i']
1150 else:
1151 self.xpcsRunArgs = ['-e', '_execute_test(); quit(0);']
1153 def addTestResults(self, test):
1154 self.passCount += test.passCount
1155 self.failCount += test.failCount
1156 self.todoCount += test.todoCount
1157 self.xunitResults.append(test.xunit_result)
1159 def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None,
1160 manifest=None, testdirs=None, testPath=None, mobileArgs=None,
1161 interactive=False, verbose=False, keepGoing=False, logfiles=True,
1162 thisChunk=1, totalChunks=1, debugger=None,
1163 debuggerArgs=None, debuggerInteractive=False,
1164 profileName=None, mozInfo=None, sequential=False, shuffle=False,
1165 testsRootDir=None, xunitFilename=None, xunitName=None,
1166 testingModulesDir=None, autolog=False, pluginsPath=None,
1167 testClass=XPCShellTestThread, failureManifest=None,
1168 on_message=None, **otherOptions):
1169 """Run xpcshell tests.
1171 |xpcshell|, is the xpcshell executable to use to run the tests.
1172 |xrePath|, if provided, is the path to the XRE to use.
1173 |appPath|, if provided, is the path to an application directory.
1174 |symbolsPath|, if provided is the path to a directory containing
1175 breakpad symbols for processing crashes in tests.
1176 |manifest|, if provided, is a file containing a list of
1177 test directories to run.
1178 |testdirs|, if provided, is a list of absolute paths of test directories.
1179 No-manifest only option.
1180 |testPath|, if provided, indicates a single path and/or test to run.
1181 |pluginsPath|, if provided, custom plugins directory to be returned from
1182 the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST.
1183 |interactive|, if set to True, indicates to provide an xpcshell prompt
1184 instead of automatically executing the test.
1185 |verbose|, if set to True, will cause stdout/stderr from tests to
1186 be printed always
1187 |logfiles|, if set to False, indicates not to save output to log files.
1188 Non-interactive only option.
1189 |debuggerInfo|, if set, specifies the debugger and debugger arguments
1190 that will be used to launch xpcshell.
1191 |profileName|, if set, specifies the name of the application for the profile
1192 directory if running only a subset of tests.
1193 |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict.
1194 |shuffle|, if True, execute tests in random order.
1195 |testsRootDir|, absolute path to root directory of all tests. This is used
1196 by xUnit generation to determine the package name of the tests.
1197 |xunitFilename|, if set, specifies the filename to which to write xUnit XML
1198 results.
1199 |xunitName|, if outputting an xUnit XML file, the str value to use for the
1200 testsuite name.
1201 |testingModulesDir|, if provided, specifies where JS modules reside.
1202 xpcshell will register a resource handler mapping this path.
1203 |otherOptions| may be present for the convenience of subclasses
1204 """
1206 global gotSIGINT
1208 if testdirs is None:
1209 testdirs = []
1211 if xunitFilename is not None or xunitName is not None:
1212 if not isinstance(testsRootDir, basestring):
1213 raise Exception("testsRootDir must be a str when outputting xUnit.")
1215 if not os.path.isabs(testsRootDir):
1216 testsRootDir = os.path.abspath(testsRootDir)
1218 if not os.path.exists(testsRootDir):
1219 raise Exception("testsRootDir path does not exists: %s" %
1220 testsRootDir)
1222 # Try to guess modules directory.
1223 # This somewhat grotesque hack allows the buildbot machines to find the
1224 # modules directory without having to configure the buildbot hosts. This
1225 # code path should never be executed in local runs because the build system
1226 # should always set this argument.
1227 if not testingModulesDir:
1228 ourDir = os.path.dirname(__file__)
1229 possible = os.path.join(ourDir, os.path.pardir, 'modules')
1231 if os.path.isdir(possible):
1232 testingModulesDir = possible
1234 if testingModulesDir:
1235 # The resource loader expects native paths. Depending on how we were
1236 # invoked, a UNIX style path may sneak in on Windows. We try to
1237 # normalize that.
1238 testingModulesDir = os.path.normpath(testingModulesDir)
1240 if not os.path.isabs(testingModulesDir):
1241 testingModulesDir = os.path.abspath(testingModulesDir)
1243 if not testingModulesDir.endswith(os.path.sep):
1244 testingModulesDir += os.path.sep
1246 self.xpcshell = xpcshell
1247 self.xrePath = xrePath
1248 self.appPath = appPath
1249 self.symbolsPath = symbolsPath
1250 self.manifest = manifest
1251 self.testdirs = testdirs
1252 self.testPath = testPath
1253 self.interactive = interactive
1254 self.verbose = verbose
1255 self.keepGoing = keepGoing
1256 self.logfiles = logfiles
1257 self.on_message = on_message
1258 self.totalChunks = totalChunks
1259 self.thisChunk = thisChunk
1260 self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive)
1261 self.profileName = profileName or "xpcshell"
1262 self.mozInfo = mozInfo
1263 self.testingModulesDir = testingModulesDir
1264 self.pluginsPath = pluginsPath
1265 self.sequential = sequential
1267 if not testdirs and not manifest:
1268 # nothing to test!
1269 self.log.error("Error: No test dirs or test manifest specified!")
1270 return False
1272 self.testCount = 0
1273 self.passCount = 0
1274 self.failCount = 0
1275 self.todoCount = 0
1277 self.setAbsPath()
1278 self.buildXpcsRunArgs()
1280 self.event = Event()
1282 # Handle filenames in mozInfo
1283 if not isinstance(self.mozInfo, dict):
1284 mozInfoFile = self.mozInfo
1285 if not os.path.isfile(mozInfoFile):
1286 self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile)
1287 return False
1288 self.mozInfo = json.load(open(mozInfoFile))
1290 # mozinfo.info is used as kwargs. Some builds are done with
1291 # an older Python that can't handle Unicode keys in kwargs.
1292 # All of the keys in question should be ASCII.
1293 fixedInfo = {}
1294 for k, v in self.mozInfo.items():
1295 if isinstance(k, unicode):
1296 k = k.encode('ascii')
1297 fixedInfo[k] = v
1298 self.mozInfo = fixedInfo
1300 mozinfo.update(self.mozInfo)
1302 # buildEnvironment() needs mozInfo, so we call it after mozInfo is initialized.
1303 self.buildEnvironment()
1305 # The appDirKey is a optional entry in either the default or individual test
1306 # sections that defines a relative application directory for test runs. If
1307 # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell
1308 # test harness.
1309 appDirKey = None
1310 if "appname" in self.mozInfo:
1311 appDirKey = self.mozInfo["appname"] + "-appdir"
1313 # We have to do this before we build the test list so we know whether or
1314 # not to run tests that depend on having the node spdy server
1315 self.trySetupNode()
1317 pStdout, pStderr = self.getPipes()
1319 self.buildTestList()
1320 if self.singleFile:
1321 self.sequential = True
1323 if shuffle:
1324 random.shuffle(self.alltests)
1326 self.xunitResults = []
1327 self.cleanup_dir_list = []
1328 self.try_again_list = []
1330 kwargs = {
1331 'appPath': self.appPath,
1332 'xrePath': self.xrePath,
1333 'testingModulesDir': self.testingModulesDir,
1334 'debuggerInfo': self.debuggerInfo,
1335 'pluginsPath': self.pluginsPath,
1336 'httpdManifest': self.httpdManifest,
1337 'httpdJSPath': self.httpdJSPath,
1338 'headJSPath': self.headJSPath,
1339 'testharnessdir': self.testharnessdir,
1340 'profileName': self.profileName,
1341 'singleFile': self.singleFile,
1342 'env': self.env, # making a copy of this in the testthreads
1343 'symbolsPath': self.symbolsPath,
1344 'logfiles': self.logfiles,
1345 'xpcshell': self.xpcshell,
1346 'xpcsRunArgs': self.xpcsRunArgs,
1347 'failureManifest': failureManifest,
1348 'on_message': self.on_message,
1349 }
1351 if self.sequential:
1352 # Allow user to kill hung xpcshell subprocess with SIGINT
1353 # when we are only running tests sequentially.
1354 signal.signal(signal.SIGINT, markGotSIGINT)
1356 if self.debuggerInfo:
1357 # Force a sequential run
1358 self.sequential = True
1360 # If we have an interactive debugger, disable SIGINT entirely.
1361 if self.debuggerInfo["interactive"]:
1362 signal.signal(signal.SIGINT, lambda signum, frame: None)
1364 # create a queue of all tests that will run
1365 tests_queue = deque()
1366 # also a list for the tests that need to be run sequentially
1367 sequential_tests = []
1368 for test_object in self.alltests:
1369 name = test_object['path']
1370 if self.singleFile and not name.endswith(self.singleFile):
1371 continue
1373 if self.testPath and name.find(self.testPath) == -1:
1374 continue
1376 self.testCount += 1
1378 test = testClass(test_object, self.event, self.cleanup_dir_list,
1379 tests_root_dir=testsRootDir, app_dir_key=appDirKey,
1380 interactive=interactive, verbose=verbose, pStdout=pStdout,
1381 pStderr=pStderr, keep_going=keepGoing, log=self.log,
1382 mobileArgs=mobileArgs, **kwargs)
1383 if 'run-sequentially' in test_object or self.sequential:
1384 sequential_tests.append(test)
1385 else:
1386 tests_queue.append(test)
1388 if self.sequential:
1389 self.log.info("INFO | Running tests sequentially.")
1390 else:
1391 self.log.info("INFO | Using at most %d threads." % NUM_THREADS)
1393 # keep a set of NUM_THREADS running tests and start running the
1394 # tests in the queue at most NUM_THREADS at a time
1395 running_tests = set()
1396 keep_going = True
1397 exceptions = []
1398 tracebacks = []
1399 while tests_queue or running_tests:
1400 # if we're not supposed to continue and all of the running tests
1401 # are done, stop
1402 if not keep_going and not running_tests:
1403 break
1405 # if there's room to run more tests, start running them
1406 while keep_going and tests_queue and (len(running_tests) < NUM_THREADS):
1407 test = tests_queue.popleft()
1408 running_tests.add(test)
1409 test.start()
1411 # queue is full (for now) or no more new tests,
1412 # process the finished tests so far
1414 # wait for at least one of the tests to finish
1415 self.event.wait(1)
1416 self.event.clear()
1418 # find what tests are done (might be more than 1)
1419 done_tests = set()
1420 for test in running_tests:
1421 if test.done:
1422 done_tests.add(test)
1423 test.join(1) # join with timeout so we don't hang on blocked threads
1424 # if the test had trouble, we will try running it again
1425 # at the end of the run
1426 if test.retry or test.is_alive():
1427 # if the join call timed out, test.is_alive => True
1428 self.try_again_list.append(test.test_object)
1429 continue
1430 # did the test encounter any exception?
1431 if test.exception:
1432 exceptions.append(test.exception)
1433 tracebacks.append(test.traceback)
1434 # we won't add any more tests, will just wait for
1435 # the currently running ones to finish
1436 keep_going = False
1437 keep_going = keep_going and test.keep_going
1438 self.addTestResults(test)
1440 # make room for new tests to run
1441 running_tests.difference_update(done_tests)
1443 if keep_going:
1444 # run the other tests sequentially
1445 for test in sequential_tests:
1446 if not keep_going:
1447 self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \
1448 "(Use --keep-going to keep running tests after killing one with SIGINT)")
1449 break
1450 # we don't want to retry these tests
1451 test.retry = False
1452 test.start()
1453 test.join()
1454 self.addTestResults(test)
1455 # did the test encounter any exception?
1456 if test.exception:
1457 exceptions.append(test.exception)
1458 tracebacks.append(test.traceback)
1459 break
1460 keep_going = test.keep_going
1462 # retry tests that failed when run in parallel
1463 if self.try_again_list:
1464 self.log.info("Retrying tests that failed when run in parallel.")
1465 for test_object in self.try_again_list:
1466 test = testClass(test_object, self.event, self.cleanup_dir_list,
1467 retry=False, tests_root_dir=testsRootDir,
1468 app_dir_key=appDirKey, interactive=interactive,
1469 verbose=verbose, pStdout=pStdout, pStderr=pStderr,
1470 keep_going=keepGoing, log=self.log, mobileArgs=mobileArgs,
1471 **kwargs)
1472 test.start()
1473 test.join()
1474 self.addTestResults(test)
1475 # did the test encounter any exception?
1476 if test.exception:
1477 exceptions.append(test.exception)
1478 tracebacks.append(test.traceback)
1479 break
1480 keep_going = test.keep_going
1482 # restore default SIGINT behaviour
1483 signal.signal(signal.SIGINT, signal.SIG_DFL)
1485 self.shutdownNode()
1486 # Clean up any slacker directories that might be lying around
1487 # Some might fail because of windows taking too long to unlock them.
1488 # We don't do anything if this fails because the test slaves will have
1489 # their $TEMP dirs cleaned up on reboot anyway.
1490 for directory in self.cleanup_dir_list:
1491 try:
1492 shutil.rmtree(directory)
1493 except:
1494 self.log.info("INFO | %s could not be cleaned up." % directory)
1496 if exceptions:
1497 self.log.info("INFO | Following exceptions were raised:")
1498 for t in tracebacks:
1499 self.log.error(t)
1500 raise exceptions[0]
1502 if self.testCount == 0:
1503 self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?")
1504 self.failCount = 1
1506 self.log.info("INFO | Result summary:")
1507 self.log.info("INFO | Passed: %d" % self.passCount)
1508 self.log.info("INFO | Failed: %d" % self.failCount)
1509 self.log.info("INFO | Todo: %d" % self.todoCount)
1510 self.log.info("INFO | Retried: %d" % len(self.try_again_list))
1512 if autolog:
1513 self.post_to_autolog(self.xunitResults, xunitName)
1515 if xunitFilename is not None:
1516 self.writeXunitResults(filename=xunitFilename, results=self.xunitResults,
1517 name=xunitName)
1519 if gotSIGINT and not keepGoing:
1520 self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \
1521 "(Use --keep-going to keep running tests after killing one with SIGINT)")
1522 return False
1524 return self.failCount == 0
1526 class XPCShellOptions(OptionParser):
1527 def __init__(self):
1528 """Process command line arguments and call runTests() to do the real work."""
1529 OptionParser.__init__(self)
1531 addCommonOptions(self)
1532 self.add_option("--app-path",
1533 type="string", dest="appPath", default=None,
1534 help="application directory (as opposed to XRE directory)")
1535 self.add_option("--autolog",
1536 action="store_true", dest="autolog", default=False,
1537 help="post to autolog")
1538 self.add_option("--interactive",
1539 action="store_true", dest="interactive", default=False,
1540 help="don't automatically run tests, drop to an xpcshell prompt")
1541 self.add_option("--verbose",
1542 action="store_true", dest="verbose", default=False,
1543 help="always print stdout and stderr from tests")
1544 self.add_option("--keep-going",
1545 action="store_true", dest="keepGoing", default=False,
1546 help="continue running tests after test killed with control-C (SIGINT)")
1547 self.add_option("--logfiles",
1548 action="store_true", dest="logfiles", default=True,
1549 help="create log files (default, only used to override --no-logfiles)")
1550 self.add_option("--manifest",
1551 type="string", dest="manifest", default=None,
1552 help="Manifest of test directories to use")
1553 self.add_option("--no-logfiles",
1554 action="store_false", dest="logfiles",
1555 help="don't create log files")
1556 self.add_option("--sequential",
1557 action="store_true", dest="sequential", default=False,
1558 help="Run all tests sequentially")
1559 self.add_option("--test-path",
1560 type="string", dest="testPath", default=None,
1561 help="single path and/or test filename to test")
1562 self.add_option("--tests-root-dir",
1563 type="string", dest="testsRootDir", default=None,
1564 help="absolute path to directory where all tests are located. this is typically $(objdir)/_tests")
1565 self.add_option("--testing-modules-dir",
1566 dest="testingModulesDir", default=None,
1567 help="Directory where testing modules are located.")
1568 self.add_option("--test-plugin-path",
1569 type="string", dest="pluginsPath", default=None,
1570 help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. "
1571 "By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory "
1572 "to return for NS_APP_PLUGINS_DIR_LIST when queried.")
1573 self.add_option("--total-chunks",
1574 type = "int", dest = "totalChunks", default=1,
1575 help = "how many chunks to split the tests up into")
1576 self.add_option("--this-chunk",
1577 type = "int", dest = "thisChunk", default=1,
1578 help = "which chunk to run between 1 and --total-chunks")
1579 self.add_option("--profile-name",
1580 type = "string", dest="profileName", default=None,
1581 help="name of application profile being tested")
1582 self.add_option("--build-info-json",
1583 type = "string", dest="mozInfo", default=None,
1584 help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
1585 self.add_option("--shuffle",
1586 action="store_true", dest="shuffle", default=False,
1587 help="Execute tests in random order")
1588 self.add_option("--xunit-file", dest="xunitFilename",
1589 help="path to file where xUnit results will be written.")
1590 self.add_option("--xunit-suite-name", dest="xunitName",
1591 help="name to record for this xUnit test suite. Many "
1592 "tools expect Java class notation, e.g. "
1593 "dom.basic.foo")
1594 self.add_option("--failure-manifest", dest="failureManifest",
1595 action="store",
1596 help="path to file where failure manifest will be written.")
1598 def main():
1599 parser = XPCShellOptions()
1600 options, args = parser.parse_args()
1602 if len(args) < 2 and options.manifest is None or \
1603 (len(args) < 1 and options.manifest is not None):
1604 print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
1605 or: %s --manifest=test.manifest <path to xpcshell>""" % (sys.argv[0],
1606 sys.argv[0])
1607 sys.exit(1)
1609 xpcsh = XPCShellTests()
1611 if options.interactive and not options.testPath:
1612 print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
1613 sys.exit(1)
1615 if not xpcsh.runTests(args[0], testdirs=args[1:], **options.__dict__):
1616 sys.exit(1)
1618 if __name__ == '__main__':
1619 main()
