The Tor Browser / file comparison

comparison: testing/xpcshell/runxpcshelltests.py

testing/xpcshell/runxpcshelltests.py

changeset 1
ca08bd8f51b2
equal deleted inserted replaced
-1:000000000000 0:3a064aa4f9eb
1 #!/usr/bin/env python
2 #
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this
5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7 import copy
8 import json
9 import math
10 import os
11 import os.path
12 import random
13 import re
14 import shutil
15 import signal
16 import socket
17 import sys
18 import time
19 import traceback
20 import xml.dom.minidom
21 from collections import deque
22 from distutils import dir_util
23 from multiprocessing import cpu_count
24 from optparse import OptionParser
25 from subprocess import Popen, PIPE, STDOUT
26 from tempfile import mkdtemp, gettempdir
27 from threading import Timer, Thread, Event, RLock
28
29 try:
30 import psutil
31 HAVE_PSUTIL = True
32 except ImportError:
33 HAVE_PSUTIL = False
34
35 from automation import Automation, getGlobalLog, resetGlobalLog
36 from automationutils import *
37
38 # Printing buffered output in case of a failure or verbose mode will result
39 # in buffered output interleaved with other threads' output.
40 # To prevent his, each call to the logger as well as any blocks of output that
41 # are intended to be continuous are protected by the same lock.
42 LOG_MUTEX = RLock()
43
44 HARNESS_TIMEOUT = 5 * 60
45
46 # benchmarking on tbpl revealed that this works best for now
47 NUM_THREADS = int(cpu_count() * 4)
48
49 FAILURE_ACTIONS = set(['test_unexpected_fail',
50 'test_unexpected_pass',
51 'javascript_error'])
52 ACTION_STRINGS = {
53 "test_unexpected_fail": "TEST-UNEXPECTED-FAIL",
54 "test_known_fail": "TEST-KNOWN-FAIL",
55 "test_unexpected_pass": "TEST-UNEXPECTED-PASS",
56 "javascript_error": "TEST-UNEXPECTED-FAIL",
57 "test_pass": "TEST-PASS",
58 "test_info": "TEST-INFO"
59 }
60
61 # --------------------------------------------------------------
62 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
63 #
64 here = os.path.dirname(__file__)
65 mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
66
67 if os.path.isdir(mozbase):
68 for package in os.listdir(mozbase):
69 sys.path.append(os.path.join(mozbase, package))
70
71 import manifestparser
72 import mozcrash
73 import mozinfo
74
75 # --------------------------------------------------------------
76
77 # TODO: perhaps this should be in a more generally shared location?
78 # This regex matches all of the C0 and C1 control characters
79 # (U+0000 through U+001F; U+007F; U+0080 through U+009F),
80 # except TAB (U+0009), CR (U+000D), LF (U+000A) and backslash (U+005C).
81 # A raw string is deliberately not used.
82 _cleanup_encoding_re = re.compile(u'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\\\\]')
83 def _cleanup_encoding_repl(m):
84 c = m.group(0)
85 return '\\\\' if c == '\\' else '\\x{0:02X}'.format(ord(c))
86 def cleanup_encoding(s):
87 """S is either a byte or unicode string. Either way it may
88 contain control characters, unpaired surrogates, reserved code
89 points, etc. If it is a byte string, it is assumed to be
90 UTF-8, but it may not be *correct* UTF-8. Produce a byte
91 string that can safely be dumped into a (generally UTF-8-coded)
92 logfile."""
93 if not isinstance(s, unicode):
94 s = s.decode('utf-8', 'replace')
95 if s.endswith('\n'):
96 # A new line is always added by head.js to delimit messages,
97 # however consumers will want to supply their own.
98 s = s[:-1]
99 # Replace all C0 and C1 control characters with \xNN escapes.
100 s = _cleanup_encoding_re.sub(_cleanup_encoding_repl, s)
101 return s.encode('utf-8', 'backslashreplace')
102
103 """ Control-C handling """
104 gotSIGINT = False
105 def markGotSIGINT(signum, stackFrame):
106 global gotSIGINT
107 gotSIGINT = True
108
109 class XPCShellTestThread(Thread):
110 def __init__(self, test_object, event, cleanup_dir_list, retry=True,
111 tests_root_dir=None, app_dir_key=None, interactive=False,
112 verbose=False, pStdout=None, pStderr=None, keep_going=False,
113 log=None, **kwargs):
114 Thread.__init__(self)
115 self.daemon = True
116
117 self.test_object = test_object
118 self.cleanup_dir_list = cleanup_dir_list
119 self.retry = retry
120
121 self.appPath = kwargs.get('appPath')
122 self.xrePath = kwargs.get('xrePath')
123 self.testingModulesDir = kwargs.get('testingModulesDir')
124 self.debuggerInfo = kwargs.get('debuggerInfo')
125 self.pluginsPath = kwargs.get('pluginsPath')
126 self.httpdManifest = kwargs.get('httpdManifest')
127 self.httpdJSPath = kwargs.get('httpdJSPath')
128 self.headJSPath = kwargs.get('headJSPath')
129 self.testharnessdir = kwargs.get('testharnessdir')
130 self.profileName = kwargs.get('profileName')
131 self.singleFile = kwargs.get('singleFile')
132 self.env = copy.deepcopy(kwargs.get('env'))
133 self.symbolsPath = kwargs.get('symbolsPath')
134 self.logfiles = kwargs.get('logfiles')
135 self.xpcshell = kwargs.get('xpcshell')
136 self.xpcsRunArgs = kwargs.get('xpcsRunArgs')
137 self.failureManifest = kwargs.get('failureManifest')
138 self.on_message = kwargs.get('on_message')
139
140 self.tests_root_dir = tests_root_dir
141 self.app_dir_key = app_dir_key
142 self.interactive = interactive
143 self.verbose = verbose
144 self.pStdout = pStdout
145 self.pStderr = pStderr
146 self.keep_going = keep_going
147 self.log = log
148
149 # only one of these will be set to 1. adding them to the totals in
150 # the harness
151 self.passCount = 0
152 self.todoCount = 0
153 self.failCount = 0
154
155 self.output_lines = []
156 self.has_failure_output = False
157 self.saw_proc_start = False
158 self.saw_proc_end = False
159
160 # event from main thread to signal work done
161 self.event = event
162 self.done = False # explicitly set flag so we don't rely on thread.isAlive
163
164 def run(self):
165 try:
166 self.run_test()
167 except Exception as e:
168 self.exception = e
169 self.traceback = traceback.format_exc()
170 else:
171 self.exception = None
172 self.traceback = None
173 if self.retry:
174 self.log.info("TEST-INFO | %s | Test failed or timed out, will retry."
175 % self.test_object['name'])
176 self.done = True
177 self.event.set()
178
179 def kill(self, proc):
180 """
181 Simple wrapper to kill a process.
182 On a remote system, this is overloaded to handle remote process communication.
183 """
184 return proc.kill()
185
186 def removeDir(self, dirname):
187 """
188 Simple wrapper to remove (recursively) a given directory.
189 On a remote system, we need to overload this to work on the remote filesystem.
190 """
191 shutil.rmtree(dirname)
192
193 def poll(self, proc):
194 """
195 Simple wrapper to check if a process has terminated.
196 On a remote system, this is overloaded to handle remote process communication.
197 """
198 return proc.poll()
199
200 def createLogFile(self, test_file, stdout):
201 """
202 For a given test file and stdout buffer, create a log file.
203 On a remote system we have to fix the test name since it can contain directories.
204 """
205 with open(test_file + ".log", "w") as f:
206 f.write(stdout)
207
208 def getReturnCode(self, proc):
209 """
210 Simple wrapper to get the return code for a given process.
211 On a remote system we overload this to work with the remote process management.
212 """
213 return proc.returncode
214
215 def communicate(self, proc):
216 """
217 Simple wrapper to communicate with a process.
218 On a remote system, this is overloaded to handle remote process communication.
219 """
220 # Processing of incremental output put here to
221 # sidestep issues on remote platforms, where what we know
222 # as proc is a file pulled off of a device.
223 if proc.stdout:
224 while True:
225 line = proc.stdout.readline()
226 if not line:
227 break
228 self.process_line(line)
229
230 if self.saw_proc_start and not self.saw_proc_end:
231 self.has_failure_output = True
232
233 return proc.communicate()
234
235 def launchProcess(self, cmd, stdout, stderr, env, cwd):
236 """
237 Simple wrapper to launch a process.
238 On a remote system, this is more complex and we need to overload this function.
239 """
240 if HAVE_PSUTIL:
241 popen_func = psutil.Popen
242 else:
243 popen_func = Popen
244 proc = popen_func(cmd, stdout=stdout, stderr=stderr,
245 env=env, cwd=cwd)
246 return proc
247
248 def checkForCrashes(self,
249 dump_directory,
250 symbols_path,
251 test_name=None):
252 """
253 Simple wrapper to check for crashes.
254 On a remote system, this is more complex and we need to overload this function.
255 """
256 return mozcrash.check_for_crashes(dump_directory, symbols_path, test_name=test_name)
257
258 def logCommand(self, name, completeCmd, testdir):
259 self.log.info("TEST-INFO | %s | full command: %r" % (name, completeCmd))
260 self.log.info("TEST-INFO | %s | current directory: %r" % (name, testdir))
261 # Show only those environment variables that are changed from
262 # the ambient environment.
263 changedEnv = (set("%s=%s" % i for i in self.env.iteritems())
264 - set("%s=%s" % i for i in os.environ.iteritems()))
265 self.log.info("TEST-INFO | %s | environment: %s" % (name, list(changedEnv)))
266
267 def testTimeout(self, test_file, proc):
268 if not self.retry:
269 self.log.error("TEST-UNEXPECTED-FAIL | %s | Test timed out" % test_file)
270 self.done = True
271 Automation().killAndGetStackNoScreenshot(proc.pid, self.appPath, self.debuggerInfo)
272
273 def buildCmdTestFile(self, name):
274 """
275 Build the command line arguments for the test file.
276 On a remote system, this may be overloaded to use a remote path structure.
277 """
278 return ['-e', 'const _TEST_FILE = ["%s"];' %
279 replaceBackSlashes(name)]
280
281 def setupTempDir(self):
282 tempDir = mkdtemp()
283 self.env["XPCSHELL_TEST_TEMP_DIR"] = tempDir
284 if self.interactive:
285 self.log.info("TEST-INFO | temp dir is %s" % tempDir)
286 return tempDir
287
288 def setupPluginsDir(self):
289 if not os.path.isdir(self.pluginsPath):
290 return None
291
292 pluginsDir = mkdtemp()
293 # shutil.copytree requires dst to not exist. Deleting the tempdir
294 # would make a race condition possible in a concurrent environment,
295 # so we are using dir_utils.copy_tree which accepts an existing dst
296 dir_util.copy_tree(self.pluginsPath, pluginsDir)
297 if self.interactive:
298 self.log.info("TEST-INFO | plugins dir is %s" % pluginsDir)
299 return pluginsDir
300
301 def setupProfileDir(self):
302 """
303 Create a temporary folder for the profile and set appropriate environment variables.
304 When running check-interactive and check-one, the directory is well-defined and
305 retained for inspection once the tests complete.
306
307 On a remote system, this may be overloaded to use a remote path structure.
308 """
309 if self.interactive or self.singleFile:
310 profileDir = os.path.join(gettempdir(), self.profileName, "xpcshellprofile")
311 try:
312 # This could be left over from previous runs
313 self.removeDir(profileDir)
314 except:
315 pass
316 os.makedirs(profileDir)
317 else:
318 profileDir = mkdtemp()
319 self.env["XPCSHELL_TEST_PROFILE_DIR"] = profileDir
320 if self.interactive or self.singleFile:
321 self.log.info("TEST-INFO | profile dir is %s" % profileDir)
322 return profileDir
323
324 def buildCmdHead(self, headfiles, tailfiles, xpcscmd):
325 """
326 Build the command line arguments for the head and tail files,
327 along with the address of the webserver which some tests require.
328
329 On a remote system, this is overloaded to resolve quoting issues over a secondary command line.
330 """
331 cmdH = ", ".join(['"' + replaceBackSlashes(f) + '"'
332 for f in headfiles])
333 cmdT = ", ".join(['"' + replaceBackSlashes(f) + '"'
334 for f in tailfiles])
335 return xpcscmd + \
336 ['-e', 'const _SERVER_ADDR = "localhost"',
337 '-e', 'const _HEAD_FILES = [%s];' % cmdH,
338 '-e', 'const _TAIL_FILES = [%s];' % cmdT]
339
340 def getHeadAndTailFiles(self, test_object):
341 """Obtain the list of head and tail files.
342
343 Returns a 2-tuple. The first element is a list of head files. The second
344 is a list of tail files.
345 """
346 def sanitize_list(s, kind):
347 for f in s.strip().split(' '):
348 f = f.strip()
349 if len(f) < 1:
350 continue
351
352 path = os.path.normpath(os.path.join(test_object['here'], f))
353 if not os.path.exists(path):
354 raise Exception('%s file does not exist: %s' % (kind, path))
355
356 if not os.path.isfile(path):
357 raise Exception('%s file is not a file: %s' % (kind, path))
358
359 yield path
360
361 return (list(sanitize_list(test_object['head'], 'head')),
362 list(sanitize_list(test_object['tail'], 'tail')))
363
364 def buildXpcsCmd(self, testdir):
365 """
366 Load the root head.js file as the first file in our test path, before other head, test, and tail files.
367 On a remote system, we overload this to add additional command line arguments, so this gets overloaded.
368 """
369 # - NOTE: if you rename/add any of the constants set here, update
370 # do_load_child_test_harness() in head.js
371 if not self.appPath:
372 self.appPath = self.xrePath
373
374 self.xpcsCmd = [
375 self.xpcshell,
376 '-g', self.xrePath,
377 '-a', self.appPath,
378 '-r', self.httpdManifest,
379 '-m',
380 '-s',
381 '-e', 'const _HTTPD_JS_PATH = "%s";' % self.httpdJSPath,
382 '-e', 'const _HEAD_JS_PATH = "%s";' % self.headJSPath
383 ]
384
385 if self.testingModulesDir:
386 # Escape backslashes in string literal.
387 sanitized = self.testingModulesDir.replace('\\', '\\\\')
388 self.xpcsCmd.extend([
389 '-e',
390 'const _TESTING_MODULES_DIR = "%s";' % sanitized
391 ])
392
393 self.xpcsCmd.extend(['-f', os.path.join(self.testharnessdir, 'head.js')])
394
395 if self.debuggerInfo:
396 self.xpcsCmd = [self.debuggerInfo["path"]] + self.debuggerInfo["args"] + self.xpcsCmd
397
398 # Automation doesn't specify a pluginsPath and xpcshell defaults to
399 # $APPDIR/plugins. We do the same here so we can carry on with
400 # setting up every test with its own plugins directory.
401 if not self.pluginsPath:
402 self.pluginsPath = os.path.join(self.appPath, 'plugins')
403
404 self.pluginsDir = self.setupPluginsDir()
405 if self.pluginsDir:
406 self.xpcsCmd.extend(['-p', self.pluginsDir])
407
408 def cleanupDir(self, directory, name, xunit_result):
409 if not os.path.exists(directory):
410 return
411
412 TRY_LIMIT = 25 # up to TRY_LIMIT attempts (one every second), because
413 # the Windows filesystem is slow to react to the changes
414 try_count = 0
415 while try_count < TRY_LIMIT:
416 try:
417 self.removeDir(directory)
418 except OSError:
419 self.log.info("TEST-INFO | Failed to remove directory: %s. Waiting." % directory)
420 # We suspect the filesystem may still be making changes. Wait a
421 # little bit and try again.
422 time.sleep(1)
423 try_count += 1
424 else:
425 # removed fine
426 return
427
428 # we try cleaning up again later at the end of the run
429 self.cleanup_dir_list.append(directory)
430
431 def clean_temp_dirs(self, name, stdout):
432 # We don't want to delete the profile when running check-interactive
433 # or check-one.
434 if self.profileDir and not self.interactive and not self.singleFile:
435 self.cleanupDir(self.profileDir, name, self.xunit_result)
436
437 self.cleanupDir(self.tempDir, name, self.xunit_result)
438
439 if self.pluginsDir:
440 self.cleanupDir(self.pluginsDir, name, self.xunit_result)
441
442 def message_from_line(self, line):
443 """ Given a line of raw output, convert to a string message. """
444 if isinstance(line, basestring):
445 # This function has received unstructured output.
446 if line:
447 if 'TEST-UNEXPECTED-' in line:
448 self.has_failure_output = True
449 return line
450
451 msg = ['%s: ' % line['process'] if 'process' in line else '']
452
453 # Each call to the logger in head.js either specified '_message'
454 # or both 'source_file' and 'diagnostic'. If either of these are
455 # missing, they ended up being undefined as a result of the way
456 # the test was run.
457 if '_message' in line:
458 msg.append(line['_message'])
459 if 'diagnostic' in line:
460 msg.append('\nDiagnostic: %s' % line['diagnostic'])
461 else:
462 msg.append('%s | %s | %s' % (ACTION_STRINGS[line['action']],
463 line.get('source_file', 'undefined'),
464 line.get('diagnostic', 'undefined')))
465
466 msg.append('\n%s' % line['stack'] if 'stack' in line else '')
467 return ''.join(msg)
468
469 def parse_output(self, output):
470 """Parses process output for structured messages and saves output as it is
471 read. Sets self.has_failure_output in case of evidence of a failure"""
472 for line_string in output.splitlines():
473 self.process_line(line_string)
474
475 if self.saw_proc_start and not self.saw_proc_end:
476 self.has_failure_output = True
477
478 def report_message(self, line):
479 """ Reports a message to a consumer, both as a strucutured and
480 human-readable log message. """
481
482 message = cleanup_encoding(self.message_from_line(line))
483 if message.endswith('\n'):
484 # A new line is always added by head.js to delimit messages,
485 # however consumers will want to supply their own.
486 message = message[:-1]
487
488 if self.on_message:
489 self.on_message(line, message)
490 else:
491 self.output_lines.append(message)
492
493 def process_line(self, line_string):
494 """ Parses a single line of output, determining its significance and
495 reporting a message.
496 """
497 try:
498 line_object = json.loads(line_string)
499 if not isinstance(line_object, dict):
500 self.report_message(line_string)
501 return
502 except ValueError:
503 self.report_message(line_string)
504 return
505
506 if 'action' not in line_object:
507 # In case a test outputs something that happens to be valid
508 # JSON.
509 self.report_message(line_string)
510 return
511
512 action = line_object['action']
513 self.report_message(line_object)
514
515 if action in FAILURE_ACTIONS:
516 self.has_failure_output = True
517 elif action == 'child_test_start':
518 self.saw_proc_start = True
519 elif action == 'child_test_end':
520 self.saw_proc_end = True
521
522 def log_output(self, output):
523 """Prints given output line-by-line to avoid overflowing buffers."""
524 self.log.info(">>>>>>>")
525 if output:
526 if isinstance(output, basestring):
527 output = output.splitlines()
528 for part in output:
529 # For multi-line output, such as a stack trace
530 for line in part.splitlines():
531 try:
532 line = line.decode('utf-8')
533 except UnicodeDecodeError:
534 self.log.info("TEST-INFO | %s | Detected non UTF-8 output."\
535 " Please modify the test to only print UTF-8." %
536 self.test_object['name'])
537 # add '?' instead of funky bytes
538 line = line.decode('utf-8', 'replace')
539 self.log.info(line)
540 self.log.info("<<<<<<<")
541
542 def run_test(self):
543 """Run an individual xpcshell test."""
544 global gotSIGINT
545
546 name = self.test_object['path']
547
548 self.xunit_result = {'name': self.test_object['name'], 'classname': 'xpcshell'}
549
550 # The xUnit package is defined as the path component between the root
551 # dir and the test with path characters replaced with '.' (using Java
552 # class notation).
553 if self.tests_root_dir is not None:
554 self.tests_root_dir = os.path.normpath(self.tests_root_dir)
555 if os.path.normpath(self.test_object['here']).find(self.tests_root_dir) != 0:
556 raise Exception('tests_root_dir is not a parent path of %s' %
557 self.test_object['here'])
558 relpath = self.test_object['here'][len(self.tests_root_dir):].lstrip('/\\')
559 self.xunit_result['classname'] = relpath.replace('/', '.').replace('\\', '.')
560
561 # Check for skipped tests
562 if 'disabled' in self.test_object:
563 self.log.info('TEST-INFO | skipping %s | %s' %
564 (name, self.test_object['disabled']))
565
566 self.xunit_result['skipped'] = True
567 self.retry = False
568
569 self.keep_going = True
570 return
571
572 # Check for known-fail tests
573 expected = self.test_object['expected'] == 'pass'
574
575 # By default self.appPath will equal the gre dir. If specified in the
576 # xpcshell.ini file, set a different app dir for this test.
577 if self.app_dir_key and self.app_dir_key in self.test_object:
578 rel_app_dir = self.test_object[self.app_dir_key]
579 rel_app_dir = os.path.join(self.xrePath, rel_app_dir)
580 self.appPath = os.path.abspath(rel_app_dir)
581 else:
582 self.appPath = None
583
584 test_dir = os.path.dirname(name)
585 self.buildXpcsCmd(test_dir)
586 head_files, tail_files = self.getHeadAndTailFiles(self.test_object)
587 cmdH = self.buildCmdHead(head_files, tail_files, self.xpcsCmd)
588
589 # Create a profile and a temp dir that the JS harness can stick
590 # a profile and temporary data in
591 self.profileDir = self.setupProfileDir()
592 self.tempDir = self.setupTempDir()
593
594 # The test file will have to be loaded after the head files.
595 cmdT = self.buildCmdTestFile(name)
596
597 args = self.xpcsRunArgs[:]
598 if 'debug' in self.test_object:
599 args.insert(0, '-d')
600
601 completeCmd = cmdH + cmdT + args
602
603 testTimeoutInterval = HARNESS_TIMEOUT
604 # Allow a test to request a multiple of the timeout if it is expected to take long
605 if 'requesttimeoutfactor' in self.test_object:
606 testTimeoutInterval *= int(self.test_object['requesttimeoutfactor'])
607
608 testTimer = None
609 if not self.interactive and not self.debuggerInfo:
610 testTimer = Timer(testTimeoutInterval, lambda: self.testTimeout(name, proc))
611 testTimer.start()
612
613 proc = None
614 stdout = None
615 stderr = None
616
617 try:
618 self.log.info("TEST-INFO | %s | running test ..." % name)
619 if self.verbose:
620 self.logCommand(name, completeCmd, test_dir)
621
622 startTime = time.time()
623 proc = self.launchProcess(completeCmd,
624 stdout=self.pStdout, stderr=self.pStderr, env=self.env, cwd=test_dir)
625
626 if self.interactive:
627 self.log.info("TEST-INFO | %s | Process ID: %d" % (name, proc.pid))
628
629 stdout, stderr = self.communicate(proc)
630
631 if self.interactive:
632 # Not sure what else to do here...
633 self.keep_going = True
634 return
635
636 if testTimer:
637 testTimer.cancel()
638
639 if stdout:
640 self.parse_output(stdout)
641 result = not (self.has_failure_output or
642 (self.getReturnCode(proc) != 0))
643
644 if result != expected:
645 if self.retry:
646 self.clean_temp_dirs(name, stdout)
647 return
648
649 failureType = "TEST-UNEXPECTED-%s" % ("FAIL" if expected else "PASS")
650 message = "%s | %s | test failed (with xpcshell return code: %d)" % (
651 failureType, name, self.getReturnCode(proc))
652 if self.output_lines:
653 message += ", see following log:"
654
655 with LOG_MUTEX:
656 self.log.error(message)
657 self.log_output(self.output_lines)
658
659 self.failCount += 1
660 self.xunit_result["passed"] = False
661
662 self.xunit_result["failure"] = {
663 "type": failureType,
664 "message": message,
665 "text": stdout
666 }
667
668 if self.failureManifest:
669 with open(self.failureManifest, 'a') as f:
670 f.write('[%s]\n' % self.test_object['path'])
671 for k, v in self.test_object.items():
672 f.write('%s = %s\n' % (k, v))
673
674 else:
675 now = time.time()
676 timeTaken = (now - startTime) * 1000
677 self.xunit_result["time"] = now - startTime
678
679 with LOG_MUTEX:
680 self.log.info("TEST-%s | %s | test passed (time: %.3fms)" % ("PASS" if expected else "KNOWN-FAIL", name, timeTaken))
681 if self.verbose:
682 self.log_output(self.output_lines)
683
684 self.xunit_result["passed"] = True
685 self.retry = False
686
687 if expected:
688 self.passCount = 1
689 else:
690 self.todoCount = 1
691 self.xunit_result["todo"] = True
692
693 if self.checkForCrashes(self.tempDir, self.symbolsPath, test_name=name):
694 if self.retry:
695 self.clean_temp_dirs(name, stdout)
696 return
697
698 message = "PROCESS-CRASH | %s | application crashed" % name
699 self.failCount = 1
700 self.xunit_result["passed"] = False
701 self.xunit_result["failure"] = {
702 "type": "PROCESS-CRASH",
703 "message": message,
704 "text": stdout
705 }
706
707 if self.logfiles and stdout:
708 self.createLogFile(name, stdout)
709
710 finally:
711 # We can sometimes get here before the process has terminated, which would
712 # cause removeDir() to fail - so check for the process & kill it it needed.
713 if proc and self.poll(proc) is None:
714 self.kill(proc)
715
716 if self.retry:
717 self.clean_temp_dirs(name, stdout)
718 return
719
720 with LOG_MUTEX:
721 message = "TEST-UNEXPECTED-FAIL | %s | Process still running after test!" % name
722 self.log.error(message)
723 self.log_output(self.output_lines)
724
725 self.failCount = 1
726 self.xunit_result["passed"] = False
727 self.xunit_result["failure"] = {
728 "type": "TEST-UNEXPECTED-FAIL",
729 "message": message,
730 "text": stdout
731 }
732
733 self.clean_temp_dirs(name, stdout)
734
735 if gotSIGINT:
736 self.xunit_result["passed"] = False
737 self.xunit_result["time"] = "0.0"
738 self.xunit_result["failure"] = {
739 "type": "SIGINT",
740 "message": "Received SIGINT",
741 "text": "Received SIGINT (control-C) during test execution."
742 }
743
744 self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C) during test execution")
745 if self.keep_going:
746 gotSIGINT = False
747 else:
748 self.keep_going = False
749 return
750
751 self.keep_going = True
752
753 class XPCShellTests(object):
754
755 log = getGlobalLog()
756 oldcwd = os.getcwd()
757
758 def __init__(self, log=None):
759 """ Init logging and node status """
760 if log:
761 resetGlobalLog(log)
762
763 # Each method of the underlying logger must acquire the log
764 # mutex before writing to stdout.
765 log_funs = ['debug', 'info', 'warning', 'error', 'critical', 'log']
766 for fun_name in log_funs:
767 unwrapped = getattr(self.log, fun_name, None)
768 if unwrapped:
769 def wrap(fn):
770 def wrapped(*args, **kwargs):
771 with LOG_MUTEX:
772 fn(*args, **kwargs)
773 return wrapped
774 setattr(self.log, fun_name, wrap(unwrapped))
775
776 self.nodeProc = {}
777
778 def buildTestList(self):
779 """
780 read the xpcshell.ini manifest and set self.alltests to be
781 an array of test objects.
782
783 if we are chunking tests, it will be done here as well
784 """
785 if isinstance(self.manifest, manifestparser.TestManifest):
786 mp = self.manifest
787 else:
788 mp = manifestparser.TestManifest(strict=False)
789 if self.manifest is None:
790 for testdir in self.testdirs:
791 if testdir:
792 mp.read(os.path.join(testdir, 'xpcshell.ini'))
793 else:
794 mp.read(self.manifest)
795
796 self.buildTestPath()
797
798 try:
799 self.alltests = mp.active_tests(**mozinfo.info)
800 except TypeError:
801 sys.stderr.write("*** offending mozinfo.info: %s\n" % repr(mozinfo.info))
802 raise
803
804 if self.singleFile is None and self.totalChunks > 1:
805 self.chunkTests()
806
807 def chunkTests(self):
808 """
809 Split the list of tests up into [totalChunks] pieces and filter the
810 self.alltests based on thisChunk, so we only run a subset.
811 """
812 totalTests = len(self.alltests)
813 testsPerChunk = math.ceil(totalTests / float(self.totalChunks))
814 start = int(round((self.thisChunk-1) * testsPerChunk))
815 end = int(start + testsPerChunk)
816 if end > totalTests:
817 end = totalTests
818 self.log.info("Running tests %d-%d/%d", start+1, end, totalTests)
819 self.alltests = self.alltests[start:end]
820
821 def setAbsPath(self):
822 """
823 Set the absolute path for xpcshell, httpdjspath and xrepath.
824 These 3 variables depend on input from the command line and we need to allow for absolute paths.
825 This function is overloaded for a remote solution as os.path* won't work remotely.
826 """
827 self.testharnessdir = os.path.dirname(os.path.abspath(__file__))
828 self.headJSPath = self.testharnessdir.replace("\\", "/") + "/head.js"
829 self.xpcshell = os.path.abspath(self.xpcshell)
830
831 # we assume that httpd.js lives in components/ relative to xpcshell
832 self.httpdJSPath = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.js')
833 self.httpdJSPath = replaceBackSlashes(self.httpdJSPath)
834
835 self.httpdManifest = os.path.join(os.path.dirname(self.xpcshell), 'components', 'httpd.manifest')
836 self.httpdManifest = replaceBackSlashes(self.httpdManifest)
837
838 if self.xrePath is None:
839 self.xrePath = os.path.dirname(self.xpcshell)
840 else:
841 self.xrePath = os.path.abspath(self.xrePath)
842
843 if self.mozInfo is None:
844 self.mozInfo = os.path.join(self.testharnessdir, "mozinfo.json")
845
846 def buildCoreEnvironment(self):
847 """
848 Add environment variables likely to be used across all platforms, including remote systems.
849 """
850 # Make assertions fatal
851 self.env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
852 # Crash reporting interferes with debugging
853 if not self.debuggerInfo:
854 self.env["MOZ_CRASHREPORTER"] = "1"
855 # Don't launch the crash reporter client
856 self.env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
857 # Capturing backtraces is very slow on some platforms, and it's
858 # disabled by automation.py too
859 self.env["NS_TRACE_MALLOC_DISABLE_STACKS"] = "1"
860 # Don't permit remote connections.
861 self.env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
862
863 def buildEnvironment(self):
864 """
865 Create and returns a dictionary of self.env to include all the appropriate env variables and values.
866 On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
867 """
868 self.env = dict(os.environ)
869 self.buildCoreEnvironment()
870 if sys.platform == 'win32':
871 self.env["PATH"] = self.env["PATH"] + ";" + self.xrePath
872 elif sys.platform in ('os2emx', 'os2knix'):
873 os.environ["BEGINLIBPATH"] = self.xrePath + ";" + self.env["BEGINLIBPATH"]
874 os.environ["LIBPATHSTRICT"] = "T"
875 elif sys.platform == 'osx' or sys.platform == "darwin":
876 self.env["DYLD_LIBRARY_PATH"] = self.xrePath
877 else: # unix or linux?
878 if not "LD_LIBRARY_PATH" in self.env or self.env["LD_LIBRARY_PATH"] is None:
879 self.env["LD_LIBRARY_PATH"] = self.xrePath
880 else:
881 self.env["LD_LIBRARY_PATH"] = ":".join([self.xrePath, self.env["LD_LIBRARY_PATH"]])
882
883 if "asan" in self.mozInfo and self.mozInfo["asan"]:
884 # ASan symbolizer support
885 llvmsym = os.path.join(self.xrePath, "llvm-symbolizer")
886 if os.path.isfile(llvmsym):
887 self.env["ASAN_SYMBOLIZER_PATH"] = llvmsym
888 self.log.info("INFO | runxpcshelltests.py | ASan using symbolizer at %s", llvmsym)
889 else:
890 self.log.info("INFO | runxpcshelltests.py | ASan symbolizer binary not found: %s", llvmsym)
891
892 return self.env
893
894 def getPipes(self):
895 """
896 Determine the value of the stdout and stderr for the test.
897 Return value is a list (pStdout, pStderr).
898 """
899 if self.interactive:
900 pStdout = None
901 pStderr = None
902 else:
903 if (self.debuggerInfo and self.debuggerInfo["interactive"]):
904 pStdout = None
905 pStderr = None
906 else:
907 if sys.platform == 'os2emx':
908 pStdout = None
909 else:
910 pStdout = PIPE
911 pStderr = STDOUT
912 return pStdout, pStderr
913
914 def buildTestPath(self):
915 """
916 If we specifiy a testpath, set the self.testPath variable to be the given directory or file.
917
918 |testPath| will be the optional path only, or |None|.
919 |singleFile| will be the optional test only, or |None|.
920 """
921 self.singleFile = None
922 if self.testPath is not None:
923 if self.testPath.endswith('.js'):
924 # Split into path and file.
925 if self.testPath.find('/') == -1:
926 # Test only.
927 self.singleFile = self.testPath
928 else:
929 # Both path and test.
930 # Reuse |testPath| temporarily.
931 self.testPath = self.testPath.rsplit('/', 1)
932 self.singleFile = self.testPath[1]
933 self.testPath = self.testPath[0]
934 else:
935 # Path only.
936 # Simply remove optional ending separator.
937 self.testPath = self.testPath.rstrip("/")
938
939 def verifyDirPath(self, dirname):
940 """
941 Simple wrapper to get the absolute path for a given directory name.
942 On a remote system, we need to overload this to work on the remote filesystem.
943 """
944 return os.path.abspath(dirname)
945
946 def trySetupNode(self):
947 """
948 Run node for SPDY tests, if available, and updates mozinfo as appropriate.
949 """
950 nodeMozInfo = {'hasNode': False} # Assume the worst
951 nodeBin = None
952
953 # We try to find the node executable in the path given to us by the user in
954 # the MOZ_NODE_PATH environment variable
955 localPath = os.getenv('MOZ_NODE_PATH', None)
956 if localPath and os.path.exists(localPath) and os.path.isfile(localPath):
957 nodeBin = localPath
958
959 if nodeBin:
960 self.log.info('Found node at %s' % (nodeBin,))
961
962 def startServer(name, serverJs):
963 if os.path.exists(serverJs):
964 # OK, we found our SPDY server, let's try to get it running
965 self.log.info('Found %s at %s' % (name, serverJs))
966 try:
967 # We pipe stdin to node because the spdy server will exit when its
968 # stdin reaches EOF
969 process = Popen([nodeBin, serverJs], stdin=PIPE, stdout=PIPE,
970 stderr=STDOUT, env=self.env, cwd=os.getcwd())
971 self.nodeProc[name] = process
972
973 # Check to make sure the server starts properly by waiting for it to
974 # tell us it's started
975 msg = process.stdout.readline()
976 if 'server listening' in msg:
977 nodeMozInfo['hasNode'] = True
978 except OSError, e:
979 # This occurs if the subprocess couldn't be started
980 self.log.error('Could not run %s server: %s' % (name, str(e)))
981
982 myDir = os.path.split(os.path.abspath(__file__))[0]
983 startServer('moz-spdy', os.path.join(myDir, 'moz-spdy', 'moz-spdy.js'))
984 startServer('moz-http2', os.path.join(myDir, 'moz-http2', 'moz-http2.js'))
985
986 mozinfo.update(nodeMozInfo)
987
988 def shutdownNode(self):
989 """
990 Shut down our node process, if it exists
991 """
992 for name, proc in self.nodeProc.iteritems():
993 self.log.info('Node %s server shutting down ...' % name)
994 proc.terminate()
995
996 def writeXunitResults(self, results, name=None, filename=None, fh=None):
997 """
998 Write Xunit XML from results.
999
1000 The function receives an iterable of results dicts. Each dict must have
1001 the following keys:
1002
1003 classname - The "class" name of the test.
1004 name - The simple name of the test.
1005
1006 In addition, it must have one of the following saying how the test
1007 executed:
1008
1009 passed - Boolean indicating whether the test passed. False if it
1010 failed.
1011 skipped - True if the test was skipped.
1012
1013 The following keys are optional:
1014
1015 time - Execution time of the test in decimal seconds.
1016 failure - Dict describing test failure. Requires keys:
1017 type - String type of failure.
1018 message - String describing basic failure.
1019 text - Verbose string describing failure.
1020
1021 Arguments:
1022
1023 |name|, Name of the test suite. Many tools expect Java class dot notation
1024 e.g. dom.simple.foo. A directory with '/' converted to '.' is a good
1025 choice.
1026 |fh|, File handle to write XML to.
1027 |filename|, File name to write XML to.
1028 |results|, Iterable of tuples describing the results.
1029 """
1030 if filename is None and fh is None:
1031 raise Exception("One of filename or fh must be defined.")
1032
1033 if name is None:
1034 name = "xpcshell"
1035 else:
1036 assert isinstance(name, basestring)
1037
1038 if filename is not None:
1039 fh = open(filename, 'wb')
1040
1041 doc = xml.dom.minidom.Document()
1042 testsuite = doc.createElement("testsuite")
1043 testsuite.setAttribute("name", name)
1044 doc.appendChild(testsuite)
1045
1046 total = 0
1047 passed = 0
1048 failed = 0
1049 skipped = 0
1050
1051 for result in results:
1052 total += 1
1053
1054 if result.get("skipped", None):
1055 skipped += 1
1056 elif result["passed"]:
1057 passed += 1
1058 else:
1059 failed += 1
1060
1061 testcase = doc.createElement("testcase")
1062 testcase.setAttribute("classname", result["classname"])
1063 testcase.setAttribute("name", result["name"])
1064
1065 if "time" in result:
1066 testcase.setAttribute("time", str(result["time"]))
1067 else:
1068 # It appears most tools expect the time attribute to be present.
1069 testcase.setAttribute("time", "0")
1070
1071 if "failure" in result:
1072 failure = doc.createElement("failure")
1073 failure.setAttribute("type", str(result["failure"]["type"]))
1074 failure.setAttribute("message", result["failure"]["message"])
1075
1076 # Lossy translation but required to not break CDATA. Also, text could
1077 # be None and Python 2.5's minidom doesn't accept None. Later versions
1078 # do, however.
1079 cdata = result["failure"]["text"]
1080 if not isinstance(cdata, str):
1081 cdata = ""
1082
1083 cdata = cdata.replace("]]>", "]] >")
1084 text = doc.createCDATASection(cdata)
1085 failure.appendChild(text)
1086 testcase.appendChild(failure)
1087
1088 if result.get("skipped", None):
1089 e = doc.createElement("skipped")
1090 testcase.appendChild(e)
1091
1092 testsuite.appendChild(testcase)
1093
1094 testsuite.setAttribute("tests", str(total))
1095 testsuite.setAttribute("failures", str(failed))
1096 testsuite.setAttribute("skip", str(skipped))
1097
1098 doc.writexml(fh, addindent=" ", newl="\n", encoding="utf-8")
1099
1100 def post_to_autolog(self, results, name):
1101 from moztest.results import TestContext, TestResult, TestResultCollection
1102 from moztest.output.autolog import AutologOutput
1103
1104 context = TestContext(
1105 testgroup='b2g xpcshell testsuite',
1106 operating_system='android',
1107 arch='emulator',
1108 harness='xpcshell',
1109 hostname=socket.gethostname(),
1110 tree='b2g',
1111 buildtype='opt',
1112 )
1113
1114 collection = TestResultCollection('b2g emulator testsuite')
1115
1116 for result in results:
1117 duration = result.get('time', 0)
1118
1119 if 'skipped' in result:
1120 outcome = 'SKIPPED'
1121 elif 'todo' in result:
1122 outcome = 'KNOWN-FAIL'
1123 elif result['passed']:
1124 outcome = 'PASS'
1125 else:
1126 outcome = 'UNEXPECTED-FAIL'
1127
1128 output = None
1129 if 'failure' in result:
1130 output = result['failure']['text']
1131
1132 t = TestResult(name=result['name'], test_class=name,
1133 time_start=0, context=context)
1134 t.finish(result=outcome, time_end=duration, output=output)
1135
1136 collection.append(t)
1137 collection.time_taken += duration
1138
1139 out = AutologOutput()
1140 out.post(out.make_testgroups(collection))
1141
1142 def buildXpcsRunArgs(self):
1143 """
1144 Add arguments to run the test or make it interactive.
1145 """
1146 if self.interactive:
1147 self.xpcsRunArgs = [
1148 '-e', 'print("To start the test, type |_execute_test();|.");',
1149 '-i']
1150 else:
1151 self.xpcsRunArgs = ['-e', '_execute_test(); quit(0);']
1152
1153 def addTestResults(self, test):
1154 self.passCount += test.passCount
1155 self.failCount += test.failCount
1156 self.todoCount += test.todoCount
1157 self.xunitResults.append(test.xunit_result)
1158
1159 def runTests(self, xpcshell, xrePath=None, appPath=None, symbolsPath=None,
1160 manifest=None, testdirs=None, testPath=None, mobileArgs=None,
1161 interactive=False, verbose=False, keepGoing=False, logfiles=True,
1162 thisChunk=1, totalChunks=1, debugger=None,
1163 debuggerArgs=None, debuggerInteractive=False,
1164 profileName=None, mozInfo=None, sequential=False, shuffle=False,
1165 testsRootDir=None, xunitFilename=None, xunitName=None,
1166 testingModulesDir=None, autolog=False, pluginsPath=None,
1167 testClass=XPCShellTestThread, failureManifest=None,
1168 on_message=None, **otherOptions):
1169 """Run xpcshell tests.
1170
1171 |xpcshell|, is the xpcshell executable to use to run the tests.
1172 |xrePath|, if provided, is the path to the XRE to use.
1173 |appPath|, if provided, is the path to an application directory.
1174 |symbolsPath|, if provided is the path to a directory containing
1175 breakpad symbols for processing crashes in tests.
1176 |manifest|, if provided, is a file containing a list of
1177 test directories to run.
1178 |testdirs|, if provided, is a list of absolute paths of test directories.
1179 No-manifest only option.
1180 |testPath|, if provided, indicates a single path and/or test to run.
1181 |pluginsPath|, if provided, custom plugins directory to be returned from
1182 the xpcshell dir svc provider for NS_APP_PLUGINS_DIR_LIST.
1183 |interactive|, if set to True, indicates to provide an xpcshell prompt
1184 instead of automatically executing the test.
1185 |verbose|, if set to True, will cause stdout/stderr from tests to
1186 be printed always
1187 |logfiles|, if set to False, indicates not to save output to log files.
1188 Non-interactive only option.
1189 |debuggerInfo|, if set, specifies the debugger and debugger arguments
1190 that will be used to launch xpcshell.
1191 |profileName|, if set, specifies the name of the application for the profile
1192 directory if running only a subset of tests.
1193 |mozInfo|, if set, specifies specifies build configuration information, either as a filename containing JSON, or a dict.
1194 |shuffle|, if True, execute tests in random order.
1195 |testsRootDir|, absolute path to root directory of all tests. This is used
1196 by xUnit generation to determine the package name of the tests.
1197 |xunitFilename|, if set, specifies the filename to which to write xUnit XML
1198 results.
1199 |xunitName|, if outputting an xUnit XML file, the str value to use for the
1200 testsuite name.
1201 |testingModulesDir|, if provided, specifies where JS modules reside.
1202 xpcshell will register a resource handler mapping this path.
1203 |otherOptions| may be present for the convenience of subclasses
1204 """
1205
1206 global gotSIGINT
1207
1208 if testdirs is None:
1209 testdirs = []
1210
1211 if xunitFilename is not None or xunitName is not None:
1212 if not isinstance(testsRootDir, basestring):
1213 raise Exception("testsRootDir must be a str when outputting xUnit.")
1214
1215 if not os.path.isabs(testsRootDir):
1216 testsRootDir = os.path.abspath(testsRootDir)
1217
1218 if not os.path.exists(testsRootDir):
1219 raise Exception("testsRootDir path does not exists: %s" %
1220 testsRootDir)
1221
1222 # Try to guess modules directory.
1223 # This somewhat grotesque hack allows the buildbot machines to find the
1224 # modules directory without having to configure the buildbot hosts. This
1225 # code path should never be executed in local runs because the build system
1226 # should always set this argument.
1227 if not testingModulesDir:
1228 ourDir = os.path.dirname(__file__)
1229 possible = os.path.join(ourDir, os.path.pardir, 'modules')
1230
1231 if os.path.isdir(possible):
1232 testingModulesDir = possible
1233
1234 if testingModulesDir:
1235 # The resource loader expects native paths. Depending on how we were
1236 # invoked, a UNIX style path may sneak in on Windows. We try to
1237 # normalize that.
1238 testingModulesDir = os.path.normpath(testingModulesDir)
1239
1240 if not os.path.isabs(testingModulesDir):
1241 testingModulesDir = os.path.abspath(testingModulesDir)
1242
1243 if not testingModulesDir.endswith(os.path.sep):
1244 testingModulesDir += os.path.sep
1245
1246 self.xpcshell = xpcshell
1247 self.xrePath = xrePath
1248 self.appPath = appPath
1249 self.symbolsPath = symbolsPath
1250 self.manifest = manifest
1251 self.testdirs = testdirs
1252 self.testPath = testPath
1253 self.interactive = interactive
1254 self.verbose = verbose
1255 self.keepGoing = keepGoing
1256 self.logfiles = logfiles
1257 self.on_message = on_message
1258 self.totalChunks = totalChunks
1259 self.thisChunk = thisChunk
1260 self.debuggerInfo = getDebuggerInfo(self.oldcwd, debugger, debuggerArgs, debuggerInteractive)
1261 self.profileName = profileName or "xpcshell"
1262 self.mozInfo = mozInfo
1263 self.testingModulesDir = testingModulesDir
1264 self.pluginsPath = pluginsPath
1265 self.sequential = sequential
1266
1267 if not testdirs and not manifest:
1268 # nothing to test!
1269 self.log.error("Error: No test dirs or test manifest specified!")
1270 return False
1271
1272 self.testCount = 0
1273 self.passCount = 0
1274 self.failCount = 0
1275 self.todoCount = 0
1276
1277 self.setAbsPath()
1278 self.buildXpcsRunArgs()
1279
1280 self.event = Event()
1281
1282 # Handle filenames in mozInfo
1283 if not isinstance(self.mozInfo, dict):
1284 mozInfoFile = self.mozInfo
1285 if not os.path.isfile(mozInfoFile):
1286 self.log.error("Error: couldn't find mozinfo.json at '%s'. Perhaps you need to use --build-info-json?" % mozInfoFile)
1287 return False
1288 self.mozInfo = json.load(open(mozInfoFile))
1289
1290 # mozinfo.info is used as kwargs. Some builds are done with
1291 # an older Python that can't handle Unicode keys in kwargs.
1292 # All of the keys in question should be ASCII.
1293 fixedInfo = {}
1294 for k, v in self.mozInfo.items():
1295 if isinstance(k, unicode):
1296 k = k.encode('ascii')
1297 fixedInfo[k] = v
1298 self.mozInfo = fixedInfo
1299
1300 mozinfo.update(self.mozInfo)
1301
1302 # buildEnvironment() needs mozInfo, so we call it after mozInfo is initialized.
1303 self.buildEnvironment()
1304
1305 # The appDirKey is a optional entry in either the default or individual test
1306 # sections that defines a relative application directory for test runs. If
1307 # defined we pass 'grePath/$appDirKey' for the -a parameter of the xpcshell
1308 # test harness.
1309 appDirKey = None
1310 if "appname" in self.mozInfo:
1311 appDirKey = self.mozInfo["appname"] + "-appdir"
1312
1313 # We have to do this before we build the test list so we know whether or
1314 # not to run tests that depend on having the node spdy server
1315 self.trySetupNode()
1316
1317 pStdout, pStderr = self.getPipes()
1318
1319 self.buildTestList()
1320 if self.singleFile:
1321 self.sequential = True
1322
1323 if shuffle:
1324 random.shuffle(self.alltests)
1325
1326 self.xunitResults = []
1327 self.cleanup_dir_list = []
1328 self.try_again_list = []
1329
1330 kwargs = {
1331 'appPath': self.appPath,
1332 'xrePath': self.xrePath,
1333 'testingModulesDir': self.testingModulesDir,
1334 'debuggerInfo': self.debuggerInfo,
1335 'pluginsPath': self.pluginsPath,
1336 'httpdManifest': self.httpdManifest,
1337 'httpdJSPath': self.httpdJSPath,
1338 'headJSPath': self.headJSPath,
1339 'testharnessdir': self.testharnessdir,
1340 'profileName': self.profileName,
1341 'singleFile': self.singleFile,
1342 'env': self.env, # making a copy of this in the testthreads
1343 'symbolsPath': self.symbolsPath,
1344 'logfiles': self.logfiles,
1345 'xpcshell': self.xpcshell,
1346 'xpcsRunArgs': self.xpcsRunArgs,
1347 'failureManifest': failureManifest,
1348 'on_message': self.on_message,
1349 }
1350
1351 if self.sequential:
1352 # Allow user to kill hung xpcshell subprocess with SIGINT
1353 # when we are only running tests sequentially.
1354 signal.signal(signal.SIGINT, markGotSIGINT)
1355
1356 if self.debuggerInfo:
1357 # Force a sequential run
1358 self.sequential = True
1359
1360 # If we have an interactive debugger, disable SIGINT entirely.
1361 if self.debuggerInfo["interactive"]:
1362 signal.signal(signal.SIGINT, lambda signum, frame: None)
1363
1364 # create a queue of all tests that will run
1365 tests_queue = deque()
1366 # also a list for the tests that need to be run sequentially
1367 sequential_tests = []
1368 for test_object in self.alltests:
1369 name = test_object['path']
1370 if self.singleFile and not name.endswith(self.singleFile):
1371 continue
1372
1373 if self.testPath and name.find(self.testPath) == -1:
1374 continue
1375
1376 self.testCount += 1
1377
1378 test = testClass(test_object, self.event, self.cleanup_dir_list,
1379 tests_root_dir=testsRootDir, app_dir_key=appDirKey,
1380 interactive=interactive, verbose=verbose, pStdout=pStdout,
1381 pStderr=pStderr, keep_going=keepGoing, log=self.log,
1382 mobileArgs=mobileArgs, **kwargs)
1383 if 'run-sequentially' in test_object or self.sequential:
1384 sequential_tests.append(test)
1385 else:
1386 tests_queue.append(test)
1387
1388 if self.sequential:
1389 self.log.info("INFO | Running tests sequentially.")
1390 else:
1391 self.log.info("INFO | Using at most %d threads." % NUM_THREADS)
1392
1393 # keep a set of NUM_THREADS running tests and start running the
1394 # tests in the queue at most NUM_THREADS at a time
1395 running_tests = set()
1396 keep_going = True
1397 exceptions = []
1398 tracebacks = []
1399 while tests_queue or running_tests:
1400 # if we're not supposed to continue and all of the running tests
1401 # are done, stop
1402 if not keep_going and not running_tests:
1403 break
1404
1405 # if there's room to run more tests, start running them
1406 while keep_going and tests_queue and (len(running_tests) < NUM_THREADS):
1407 test = tests_queue.popleft()
1408 running_tests.add(test)
1409 test.start()
1410
1411 # queue is full (for now) or no more new tests,
1412 # process the finished tests so far
1413
1414 # wait for at least one of the tests to finish
1415 self.event.wait(1)
1416 self.event.clear()
1417
1418 # find what tests are done (might be more than 1)
1419 done_tests = set()
1420 for test in running_tests:
1421 if test.done:
1422 done_tests.add(test)
1423 test.join(1) # join with timeout so we don't hang on blocked threads
1424 # if the test had trouble, we will try running it again
1425 # at the end of the run
1426 if test.retry or test.is_alive():
1427 # if the join call timed out, test.is_alive => True
1428 self.try_again_list.append(test.test_object)
1429 continue
1430 # did the test encounter any exception?
1431 if test.exception:
1432 exceptions.append(test.exception)
1433 tracebacks.append(test.traceback)
1434 # we won't add any more tests, will just wait for
1435 # the currently running ones to finish
1436 keep_going = False
1437 keep_going = keep_going and test.keep_going
1438 self.addTestResults(test)
1439
1440 # make room for new tests to run
1441 running_tests.difference_update(done_tests)
1442
1443 if keep_going:
1444 # run the other tests sequentially
1445 for test in sequential_tests:
1446 if not keep_going:
1447 self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \
1448 "(Use --keep-going to keep running tests after killing one with SIGINT)")
1449 break
1450 # we don't want to retry these tests
1451 test.retry = False
1452 test.start()
1453 test.join()
1454 self.addTestResults(test)
1455 # did the test encounter any exception?
1456 if test.exception:
1457 exceptions.append(test.exception)
1458 tracebacks.append(test.traceback)
1459 break
1460 keep_going = test.keep_going
1461
1462 # retry tests that failed when run in parallel
1463 if self.try_again_list:
1464 self.log.info("Retrying tests that failed when run in parallel.")
1465 for test_object in self.try_again_list:
1466 test = testClass(test_object, self.event, self.cleanup_dir_list,
1467 retry=False, tests_root_dir=testsRootDir,
1468 app_dir_key=appDirKey, interactive=interactive,
1469 verbose=verbose, pStdout=pStdout, pStderr=pStderr,
1470 keep_going=keepGoing, log=self.log, mobileArgs=mobileArgs,
1471 **kwargs)
1472 test.start()
1473 test.join()
1474 self.addTestResults(test)
1475 # did the test encounter any exception?
1476 if test.exception:
1477 exceptions.append(test.exception)
1478 tracebacks.append(test.traceback)
1479 break
1480 keep_going = test.keep_going
1481
1482 # restore default SIGINT behaviour
1483 signal.signal(signal.SIGINT, signal.SIG_DFL)
1484
1485 self.shutdownNode()
1486 # Clean up any slacker directories that might be lying around
1487 # Some might fail because of windows taking too long to unlock them.
1488 # We don't do anything if this fails because the test slaves will have
1489 # their $TEMP dirs cleaned up on reboot anyway.
1490 for directory in self.cleanup_dir_list:
1491 try:
1492 shutil.rmtree(directory)
1493 except:
1494 self.log.info("INFO | %s could not be cleaned up." % directory)
1495
1496 if exceptions:
1497 self.log.info("INFO | Following exceptions were raised:")
1498 for t in tracebacks:
1499 self.log.error(t)
1500 raise exceptions[0]
1501
1502 if self.testCount == 0:
1503 self.log.error("TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?")
1504 self.failCount = 1
1505
1506 self.log.info("INFO | Result summary:")
1507 self.log.info("INFO | Passed: %d" % self.passCount)
1508 self.log.info("INFO | Failed: %d" % self.failCount)
1509 self.log.info("INFO | Todo: %d" % self.todoCount)
1510 self.log.info("INFO | Retried: %d" % len(self.try_again_list))
1511
1512 if autolog:
1513 self.post_to_autolog(self.xunitResults, xunitName)
1514
1515 if xunitFilename is not None:
1516 self.writeXunitResults(filename=xunitFilename, results=self.xunitResults,
1517 name=xunitName)
1518
1519 if gotSIGINT and not keepGoing:
1520 self.log.error("TEST-UNEXPECTED-FAIL | Received SIGINT (control-C), so stopped run. " \
1521 "(Use --keep-going to keep running tests after killing one with SIGINT)")
1522 return False
1523
1524 return self.failCount == 0
1525
1526 class XPCShellOptions(OptionParser):
1527 def __init__(self):
1528 """Process command line arguments and call runTests() to do the real work."""
1529 OptionParser.__init__(self)
1530
1531 addCommonOptions(self)
1532 self.add_option("--app-path",
1533 type="string", dest="appPath", default=None,
1534 help="application directory (as opposed to XRE directory)")
1535 self.add_option("--autolog",
1536 action="store_true", dest="autolog", default=False,
1537 help="post to autolog")
1538 self.add_option("--interactive",
1539 action="store_true", dest="interactive", default=False,
1540 help="don't automatically run tests, drop to an xpcshell prompt")
1541 self.add_option("--verbose",
1542 action="store_true", dest="verbose", default=False,
1543 help="always print stdout and stderr from tests")
1544 self.add_option("--keep-going",
1545 action="store_true", dest="keepGoing", default=False,
1546 help="continue running tests after test killed with control-C (SIGINT)")
1547 self.add_option("--logfiles",
1548 action="store_true", dest="logfiles", default=True,
1549 help="create log files (default, only used to override --no-logfiles)")
1550 self.add_option("--manifest",
1551 type="string", dest="manifest", default=None,
1552 help="Manifest of test directories to use")
1553 self.add_option("--no-logfiles",
1554 action="store_false", dest="logfiles",
1555 help="don't create log files")
1556 self.add_option("--sequential",
1557 action="store_true", dest="sequential", default=False,
1558 help="Run all tests sequentially")
1559 self.add_option("--test-path",
1560 type="string", dest="testPath", default=None,
1561 help="single path and/or test filename to test")
1562 self.add_option("--tests-root-dir",
1563 type="string", dest="testsRootDir", default=None,
1564 help="absolute path to directory where all tests are located. this is typically $(objdir)/_tests")
1565 self.add_option("--testing-modules-dir",
1566 dest="testingModulesDir", default=None,
1567 help="Directory where testing modules are located.")
1568 self.add_option("--test-plugin-path",
1569 type="string", dest="pluginsPath", default=None,
1570 help="Path to the location of a plugins directory containing the test plugin or plugins required for tests. "
1571 "By default xpcshell's dir svc provider returns gre/plugins. Use test-plugin-path to add a directory "
1572 "to return for NS_APP_PLUGINS_DIR_LIST when queried.")
1573 self.add_option("--total-chunks",
1574 type = "int", dest = "totalChunks", default=1,
1575 help = "how many chunks to split the tests up into")
1576 self.add_option("--this-chunk",
1577 type = "int", dest = "thisChunk", default=1,
1578 help = "which chunk to run between 1 and --total-chunks")
1579 self.add_option("--profile-name",
1580 type = "string", dest="profileName", default=None,
1581 help="name of application profile being tested")
1582 self.add_option("--build-info-json",
1583 type = "string", dest="mozInfo", default=None,
1584 help="path to a mozinfo.json including information about the build configuration. defaults to looking for mozinfo.json next to the script.")
1585 self.add_option("--shuffle",
1586 action="store_true", dest="shuffle", default=False,
1587 help="Execute tests in random order")
1588 self.add_option("--xunit-file", dest="xunitFilename",
1589 help="path to file where xUnit results will be written.")
1590 self.add_option("--xunit-suite-name", dest="xunitName",
1591 help="name to record for this xUnit test suite. Many "
1592 "tools expect Java class notation, e.g. "
1593 "dom.basic.foo")
1594 self.add_option("--failure-manifest", dest="failureManifest",
1595 action="store",
1596 help="path to file where failure manifest will be written.")
1597
1598 def main():
1599 parser = XPCShellOptions()
1600 options, args = parser.parse_args()
1601
1602 if len(args) < 2 and options.manifest is None or \
1603 (len(args) < 1 and options.manifest is not None):
1604 print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
1605 or: %s --manifest=test.manifest <path to xpcshell>""" % (sys.argv[0],
1606 sys.argv[0])
1607 sys.exit(1)
1608
1609 xpcsh = XPCShellTests()
1610
1611 if options.interactive and not options.testPath:
1612 print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
1613 sys.exit(1)
1614
1615 if not xpcsh.runTests(args[0], testdirs=args[1:], **options.__dict__):
1616 sys.exit(1)
1617
1618 if __name__ == '__main__':
1619 main()

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial