|
1 #!/usr/bin/env python |
|
2 # This Source Code Form is subject to the terms of the Mozilla Public |
|
3 # License, v. 2.0. If a copy of the MPL was not distributed with this |
|
4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
|
5 |
|
6 |
|
7 # jit_test.py -- Python harness for JavaScript trace tests. |
|
8 |
|
9 from __future__ import print_function |
|
10 import os, posixpath, sys, tempfile, traceback, time |
|
11 import subprocess |
|
12 from subprocess import Popen, PIPE |
|
13 from threading import Thread |
|
14 import signal |
|
15 import StringIO |
|
16 |
|
17 try: |
|
18 from multiprocessing import Process, Manager, cpu_count |
|
19 HAVE_MULTIPROCESSING = True |
|
20 except ImportError: |
|
21 HAVE_MULTIPROCESSING = False |
|
22 |
|
23 from progressbar import ProgressBar, NullProgressBar |
|
24 from results import TestOutput |
|
25 |
|
26 TESTS_LIB_DIR = os.path.dirname(os.path.abspath(__file__)) |
|
27 JS_DIR = os.path.dirname(os.path.dirname(TESTS_LIB_DIR)) |
|
28 TOP_SRC_DIR = os.path.dirname(os.path.dirname(JS_DIR)) |
|
29 TEST_DIR = os.path.join(JS_DIR, 'jit-test', 'tests') |
|
30 LIB_DIR = os.path.join(JS_DIR, 'jit-test', 'lib') + os.path.sep |
|
31 JS_CACHE_DIR = os.path.join(JS_DIR, 'jit-test', '.js-cache') |
|
32 JS_TESTS_DIR = posixpath.join(JS_DIR, 'tests') |
|
33 |
|
34 # Backported from Python 3.1 posixpath.py |
|
35 def _relpath(path, start=None): |
|
36 """Return a relative version of a path""" |
|
37 |
|
38 if not path: |
|
39 raise ValueError("no path specified") |
|
40 |
|
41 if start is None: |
|
42 start = os.curdir |
|
43 |
|
44 start_list = os.path.abspath(start).split(os.sep) |
|
45 path_list = os.path.abspath(path).split(os.sep) |
|
46 |
|
47 # Work out how much of the filepath is shared by start and path. |
|
48 i = len(os.path.commonprefix([start_list, path_list])) |
|
49 |
|
50 rel_list = [os.pardir] * (len(start_list)-i) + path_list[i:] |
|
51 if not rel_list: |
|
52 return os.curdir |
|
53 return os.path.join(*rel_list) |
|
54 |
|
55 os.path.relpath = _relpath |
|
56 |
|
57 class Test: |
|
58 |
|
59 VALGRIND_CMD = [] |
|
60 paths = (d for d in os.environ['PATH'].split(os.pathsep)) |
|
61 valgrinds = (os.path.join(d, 'valgrind') for d in paths) |
|
62 if any(os.path.exists(p) for p in valgrinds): |
|
63 VALGRIND_CMD = [ |
|
64 'valgrind', '-q', '--smc-check=all-non-file', |
|
65 '--error-exitcode=1', '--gen-suppressions=all', |
|
66 '--show-possibly-lost=no', '--leak-check=full', |
|
67 ] |
|
68 if os.uname()[0] == 'Darwin': |
|
69 VALGRIND_CMD.append('--dsymutil=yes') |
|
70 |
|
71 del paths |
|
72 del valgrinds |
|
73 |
|
74 def __init__(self, path): |
|
75 # Absolute path of the test file. |
|
76 self.path = path |
|
77 |
|
78 # Path relative to the top mozilla/ directory. |
|
79 self.relpath_top = os.path.relpath(path, TOP_SRC_DIR) |
|
80 |
|
81 # Path relative to mozilla/js/src/jit-test/tests/. |
|
82 self.relpath_tests = os.path.relpath(path, TEST_DIR) |
|
83 |
|
84 self.jitflags = [] # jit flags to enable |
|
85 self.slow = False # True means the test is slow-running |
|
86 self.allow_oom = False # True means that OOM is not considered a failure |
|
87 self.valgrind = False # True means run under valgrind |
|
88 self.tz_pacific = False # True means force Pacific time for the test |
|
89 self.expect_error = '' # Errors to expect and consider passing |
|
90 self.expect_status = 0 # Exit status to expect from shell |
|
91 |
|
92 def copy(self): |
|
93 t = Test(self.path) |
|
94 t.jitflags = self.jitflags[:] |
|
95 t.slow = self.slow |
|
96 t.allow_oom = self.allow_oom |
|
97 t.valgrind = self.valgrind |
|
98 t.tz_pacific = self.tz_pacific |
|
99 t.expect_error = self.expect_error |
|
100 t.expect_status = self.expect_status |
|
101 return t |
|
102 |
|
103 COOKIE = '|jit-test|' |
|
104 CacheDir = JS_CACHE_DIR |
|
105 |
|
106 @classmethod |
|
107 def from_file(cls, path, options): |
|
108 test = cls(path) |
|
109 |
|
110 line = open(path).readline() |
|
111 i = line.find(cls.COOKIE) |
|
112 if i != -1: |
|
113 meta = line[i + len(cls.COOKIE):].strip('\n') |
|
114 parts = meta.split(';') |
|
115 for part in parts: |
|
116 part = part.strip() |
|
117 if not part: |
|
118 continue |
|
119 name, _, value = part.partition(':') |
|
120 if value: |
|
121 value = value.strip() |
|
122 if name == 'error': |
|
123 test.expect_error = value |
|
124 elif name == 'exitstatus': |
|
125 try: |
|
126 test.expect_status = int(value, 0); |
|
127 except ValueError: |
|
128 print("warning: couldn't parse exit status %s" % value) |
|
129 elif name == 'thread-count': |
|
130 try: |
|
131 test.jitflags.append('--thread-count=' + int(value, 0)); |
|
132 except ValueError: |
|
133 print("warning: couldn't parse thread-count %s" % value) |
|
134 else: |
|
135 print('warning: unrecognized |jit-test| attribute %s' % part) |
|
136 else: |
|
137 if name == 'slow': |
|
138 test.slow = True |
|
139 elif name == 'allow-oom': |
|
140 test.allow_oom = True |
|
141 elif name == 'valgrind': |
|
142 test.valgrind = options.valgrind |
|
143 elif name == 'tz-pacific': |
|
144 test.tz_pacific = True |
|
145 elif name == 'debug': |
|
146 test.jitflags.append('--debugjit') |
|
147 elif name == 'ion-eager': |
|
148 test.jitflags.append('--ion-eager') |
|
149 elif name == 'no-ion': |
|
150 test.jitflags.append('--no-ion') |
|
151 elif name == 'dump-bytecode': |
|
152 test.jitflags.append('--dump-bytecode') |
|
153 else: |
|
154 print('warning: unrecognized |jit-test| attribute %s' % part) |
|
155 |
|
156 if options.valgrind_all: |
|
157 test.valgrind = True |
|
158 |
|
159 return test |
|
160 |
|
161 def command(self, prefix, libdir, remote_prefix=None): |
|
162 path = self.path |
|
163 if remote_prefix: |
|
164 path = self.path.replace(TEST_DIR, remote_prefix) |
|
165 |
|
166 scriptdir_var = os.path.dirname(path); |
|
167 if not scriptdir_var.endswith('/'): |
|
168 scriptdir_var += '/' |
|
169 |
|
170 # Platforms where subprocess immediately invokes exec do not care |
|
171 # whether we use double or single quotes. On windows and when using |
|
172 # a remote device, however, we have to be careful to use the quote |
|
173 # style that is the opposite of what the exec wrapper uses. |
|
174 # This uses %r to get single quotes on windows and special cases |
|
175 # the remote device. |
|
176 fmt = 'const platform=%r; const libdir=%r; const scriptdir=%r' |
|
177 if remote_prefix: |
|
178 fmt = 'const platform="%s"; const libdir="%s"; const scriptdir="%s"' |
|
179 expr = fmt % (sys.platform, libdir, scriptdir_var) |
|
180 |
|
181 # We may have specified '-a' or '-d' twice: once via --jitflags, once |
|
182 # via the "|jit-test|" line. Remove dups because they are toggles. |
|
183 cmd = prefix + ['--js-cache', Test.CacheDir] |
|
184 cmd += list(set(self.jitflags)) + ['-e', expr, '-f', path] |
|
185 if self.valgrind: |
|
186 cmd = self.VALGRIND_CMD + cmd |
|
187 return cmd |
|
188 |
|
189 def find_tests(substring=None): |
|
190 ans = [] |
|
191 for dirpath, dirnames, filenames in os.walk(TEST_DIR): |
|
192 dirnames.sort() |
|
193 filenames.sort() |
|
194 if dirpath == '.': |
|
195 continue |
|
196 for filename in filenames: |
|
197 if not filename.endswith('.js'): |
|
198 continue |
|
199 if filename in ('shell.js', 'browser.js', 'jsref.js'): |
|
200 continue |
|
201 test = os.path.join(dirpath, filename) |
|
202 if substring is None or substring in os.path.relpath(test, TEST_DIR): |
|
203 ans.append(test) |
|
204 return ans |
|
205 |
|
206 def tmppath(token): |
|
207 fd, path = tempfile.mkstemp(prefix=token) |
|
208 os.close(fd) |
|
209 return path |
|
210 |
|
211 def read_and_unlink(path): |
|
212 f = open(path) |
|
213 d = f.read() |
|
214 f.close() |
|
215 os.unlink(path) |
|
216 return d |
|
217 |
|
218 def th_run_cmd(cmdline, options, l): |
|
219 # close_fds is not supported on Windows and will cause a ValueError. |
|
220 if sys.platform != 'win32': |
|
221 options["close_fds"] = True |
|
222 p = Popen(cmdline, stdin=PIPE, stdout=PIPE, stderr=PIPE, **options) |
|
223 |
|
224 l[0] = p |
|
225 out, err = p.communicate() |
|
226 l[1] = (out, err, p.returncode) |
|
227 |
|
228 def run_timeout_cmd(cmdline, options, timeout=60.0): |
|
229 l = [ None, None ] |
|
230 timed_out = False |
|
231 th = Thread(target=th_run_cmd, args=(cmdline, options, l)) |
|
232 |
|
233 # If our SIGINT handler is set to SIG_IGN (ignore) |
|
234 # then we are running as a child process for parallel |
|
235 # execution and we must ensure to kill our child |
|
236 # when we are signaled to exit. |
|
237 sigint_handler = signal.getsignal(signal.SIGINT) |
|
238 sigterm_handler = signal.getsignal(signal.SIGTERM) |
|
239 if (sigint_handler == signal.SIG_IGN): |
|
240 def handleChildSignal(sig, frame): |
|
241 try: |
|
242 if sys.platform != 'win32': |
|
243 os.kill(l[0].pid, signal.SIGKILL) |
|
244 else: |
|
245 import ctypes |
|
246 ctypes.windll.kernel32.TerminateProcess(int(l[0]._handle), -1) |
|
247 except OSError: |
|
248 pass |
|
249 if (sig == signal.SIGTERM): |
|
250 sys.exit(0) |
|
251 signal.signal(signal.SIGINT, handleChildSignal) |
|
252 signal.signal(signal.SIGTERM, handleChildSignal) |
|
253 |
|
254 th.start() |
|
255 th.join(timeout) |
|
256 while th.isAlive(): |
|
257 if l[0] is not None: |
|
258 try: |
|
259 # In Python 3, we could just do l[0].kill(). |
|
260 if sys.platform != 'win32': |
|
261 os.kill(l[0].pid, signal.SIGKILL) |
|
262 else: |
|
263 import ctypes |
|
264 ctypes.windll.kernel32.TerminateProcess(int(l[0]._handle), -1) |
|
265 time.sleep(.1) |
|
266 timed_out = True |
|
267 except OSError: |
|
268 # Expecting a "No such process" error |
|
269 pass |
|
270 th.join() |
|
271 |
|
272 # Restore old signal handlers |
|
273 if (sigint_handler == signal.SIG_IGN): |
|
274 signal.signal(signal.SIGINT, signal.SIG_IGN) |
|
275 signal.signal(signal.SIGTERM, sigterm_handler) |
|
276 |
|
277 (out, err, code) = l[1] |
|
278 |
|
279 return (out, err, code, timed_out) |
|
280 |
|
281 def run_cmd(cmdline, env, timeout): |
|
282 return run_timeout_cmd(cmdline, { 'env': env }, timeout) |
|
283 |
|
284 def run_cmd_avoid_stdio(cmdline, env, timeout): |
|
285 stdoutPath, stderrPath = tmppath('jsstdout'), tmppath('jsstderr') |
|
286 env['JS_STDOUT'] = stdoutPath |
|
287 env['JS_STDERR'] = stderrPath |
|
288 _, __, code = run_timeout_cmd(cmdline, { 'env': env }, timeout) |
|
289 return read_and_unlink(stdoutPath), read_and_unlink(stderrPath), code |
|
290 |
|
291 def run_test(test, prefix, options): |
|
292 cmd = test.command(prefix, LIB_DIR) |
|
293 if options.show_cmd: |
|
294 print(subprocess.list2cmdline(cmd)) |
|
295 |
|
296 if options.avoid_stdio: |
|
297 run = run_cmd_avoid_stdio |
|
298 else: |
|
299 run = run_cmd |
|
300 |
|
301 env = os.environ.copy() |
|
302 if test.tz_pacific: |
|
303 env['TZ'] = 'PST8PDT' |
|
304 |
|
305 # Ensure interpreter directory is in shared library path. |
|
306 pathvar = '' |
|
307 if sys.platform.startswith('linux'): |
|
308 pathvar = 'LD_LIBRARY_PATH' |
|
309 elif sys.platform.startswith('darwin'): |
|
310 pathvar = 'DYLD_LIBRARY_PATH' |
|
311 elif sys.platform.startswith('win'): |
|
312 pathvar = 'PATH' |
|
313 if pathvar: |
|
314 bin_dir = os.path.dirname(cmd[0]) |
|
315 if pathvar in env: |
|
316 env[pathvar] = '%s%s%s' % (bin_dir, os.pathsep, env[pathvar]) |
|
317 else: |
|
318 env[pathvar] = bin_dir |
|
319 |
|
320 out, err, code, timed_out = run(cmd, env, options.timeout) |
|
321 return TestOutput(test, cmd, out, err, code, None, timed_out) |
|
322 |
|
323 def run_test_remote(test, device, prefix, options): |
|
324 cmd = test.command(prefix, posixpath.join(options.remote_test_root, 'lib/'), posixpath.join(options.remote_test_root, 'tests')) |
|
325 if options.show_cmd: |
|
326 print(subprocess.list2cmdline(cmd)) |
|
327 |
|
328 env = {} |
|
329 if test.tz_pacific: |
|
330 env['TZ'] = 'PST8PDT' |
|
331 |
|
332 env['LD_LIBRARY_PATH'] = options.remote_test_root |
|
333 |
|
334 buf = StringIO.StringIO() |
|
335 returncode = device.shell(cmd, buf, env=env, cwd=options.remote_test_root, |
|
336 timeout=int(options.timeout)) |
|
337 |
|
338 out = buf.getvalue() |
|
339 # We can't distinguish between stdout and stderr so we pass |
|
340 # the same buffer to both. |
|
341 return TestOutput(test, cmd, out, out, returncode, None, False) |
|
342 |
|
343 def check_output(out, err, rc, timed_out, test): |
|
344 if timed_out: |
|
345 # The shell sometimes hangs on shutdown on Windows 7 and Windows |
|
346 # Server 2008. See bug 970063 comment 7 for a description of the |
|
347 # problem. Until bug 956899 is fixed, ignore timeouts on these |
|
348 # platforms (versions 6.0 and 6.1). |
|
349 if sys.platform == 'win32': |
|
350 ver = sys.getwindowsversion() |
|
351 if ver.major == 6 and ver.minor <= 1: |
|
352 return True |
|
353 return False |
|
354 |
|
355 if test.expect_error: |
|
356 # The shell exits with code 3 on uncaught exceptions. |
|
357 # Sometimes 0 is returned on Windows for unknown reasons. |
|
358 # See bug 899697. |
|
359 if sys.platform in ['win32', 'cygwin']: |
|
360 if rc != 3 and rc != 0: |
|
361 return False |
|
362 else: |
|
363 if rc != 3: |
|
364 return False |
|
365 |
|
366 return test.expect_error in err |
|
367 |
|
368 for line in out.split('\n'): |
|
369 if line.startswith('Trace stats check failed'): |
|
370 return False |
|
371 |
|
372 for line in err.split('\n'): |
|
373 if 'Assertion failed:' in line: |
|
374 return False |
|
375 |
|
376 if rc != test.expect_status: |
|
377 # Tests which expect a timeout check for exit code 6. |
|
378 # Sometimes 0 is returned on Windows for unknown reasons. |
|
379 # See bug 899697. |
|
380 if sys.platform in ['win32', 'cygwin'] and rc == 0: |
|
381 return True |
|
382 |
|
383 # Allow a non-zero exit code if we want to allow OOM, but only if we |
|
384 # actually got OOM. |
|
385 return test.allow_oom and 'out of memory' in err and 'Assertion failure' not in err |
|
386 |
|
387 return True |
|
388 |
|
389 def print_tinderbox(ok, res): |
|
390 # Output test failures in a TBPL parsable format, eg: |
|
391 # TEST-RESULT | filename.js | Failure description (code N, args "--foobar") |
|
392 # |
|
393 # Example: |
|
394 # TEST-PASS | foo/bar/baz.js | (code 0, args "--ion-eager") |
|
395 # TEST-UNEXPECTED-FAIL | foo/bar/baz.js | TypeError: or something (code -9, args "--no-ion") |
|
396 # INFO exit-status : 3 |
|
397 # INFO timed-out : False |
|
398 # INFO stdout > foo |
|
399 # INFO stdout > bar |
|
400 # INFO stdout > baz |
|
401 # INFO stderr 2> TypeError: or something |
|
402 # TEST-UNEXPECTED-FAIL | jit_test.py: Test execution interrupted by user |
|
403 result = "TEST-PASS" if ok else "TEST-UNEXPECTED-FAIL" |
|
404 message = "Success" if ok else res.describe_failure() |
|
405 jitflags = " ".join(res.test.jitflags) |
|
406 print("{} | {} | {} (code {}, args \"{}\")".format( |
|
407 result, res.test.relpath_top, message, res.rc, jitflags)) |
|
408 |
|
409 # For failed tests, print as much information as we have, to aid debugging. |
|
410 if ok: |
|
411 return |
|
412 print("INFO exit-status : {}".format(res.rc)) |
|
413 print("INFO timed-out : {}".format(res.timed_out)) |
|
414 for line in res.out.splitlines(): |
|
415 print("INFO stdout > " + line.strip()) |
|
416 for line in res.err.splitlines(): |
|
417 print("INFO stderr 2> " + line.strip()) |
|
418 |
|
419 def wrap_parallel_run_test(test, prefix, resultQueue, options): |
|
420 # Ignore SIGINT in the child |
|
421 signal.signal(signal.SIGINT, signal.SIG_IGN) |
|
422 result = run_test(test, prefix, options) |
|
423 resultQueue.put(result) |
|
424 return result |
|
425 |
|
426 def run_tests_parallel(tests, prefix, options): |
|
427 # This queue will contain the results of the various tests run. |
|
428 # We could make this queue a global variable instead of using |
|
429 # a manager to share, but this will not work on Windows. |
|
430 queue_manager = Manager() |
|
431 async_test_result_queue = queue_manager.Queue() |
|
432 |
|
433 # This queue will be used by the result process to indicate |
|
434 # that it has received a result and we can start a new process |
|
435 # on our end. The advantage is that we don't have to sleep and |
|
436 # check for worker completion ourselves regularly. |
|
437 notify_queue = queue_manager.Queue() |
|
438 |
|
439 # This queue will contain the return value of the function |
|
440 # processing the test results. |
|
441 total_tests = len(tests) * options.repeat |
|
442 result_process_return_queue = queue_manager.Queue() |
|
443 result_process = Process(target=process_test_results_parallel, |
|
444 args=(async_test_result_queue, result_process_return_queue, |
|
445 notify_queue, total_tests, options)) |
|
446 result_process.start() |
|
447 |
|
448 # Ensure that a SIGTERM is handled the same way as SIGINT |
|
449 # to terminate all child processes. |
|
450 sigint_handler = signal.getsignal(signal.SIGINT) |
|
451 signal.signal(signal.SIGTERM, sigint_handler) |
|
452 |
|
453 worker_processes = [] |
|
454 |
|
455 def remove_completed_workers(workers): |
|
456 new_workers = [] |
|
457 for worker in workers: |
|
458 if worker.is_alive(): |
|
459 new_workers.append(worker) |
|
460 else: |
|
461 worker.join() |
|
462 return new_workers |
|
463 |
|
464 try: |
|
465 testcnt = 0 |
|
466 # Initially start as many jobs as allowed to run parallel |
|
467 for i in range(min(options.max_jobs,total_tests)): |
|
468 notify_queue.put(True) |
|
469 |
|
470 # For every item in the notify queue, start one new worker. |
|
471 # Every completed worker adds a new item to this queue. |
|
472 while notify_queue.get(): |
|
473 if (testcnt < total_tests): |
|
474 # Start one new worker |
|
475 test = tests[testcnt % len(tests)] |
|
476 worker_process = Process(target=wrap_parallel_run_test, args=(test, prefix, async_test_result_queue, options)) |
|
477 worker_processes.append(worker_process) |
|
478 worker_process.start() |
|
479 testcnt += 1 |
|
480 |
|
481 # Collect completed workers |
|
482 worker_processes = remove_completed_workers(worker_processes) |
|
483 else: |
|
484 break |
|
485 |
|
486 # Wait for all processes to terminate |
|
487 while len(worker_processes) > 0: |
|
488 worker_processes = remove_completed_workers(worker_processes) |
|
489 |
|
490 # Signal completion to result processor, then wait for it to complete on its own |
|
491 async_test_result_queue.put(None) |
|
492 result_process.join() |
|
493 |
|
494 # Return what the result process has returned to us |
|
495 return result_process_return_queue.get() |
|
496 except (Exception, KeyboardInterrupt) as e: |
|
497 # Print the exception if it's not an interrupt, |
|
498 # might point to a bug or other faulty condition |
|
499 if not isinstance(e,KeyboardInterrupt): |
|
500 traceback.print_exc() |
|
501 |
|
502 for worker in worker_processes: |
|
503 try: |
|
504 worker.terminate() |
|
505 except: |
|
506 pass |
|
507 |
|
508 result_process.terminate() |
|
509 |
|
510 return False |
|
511 |
|
512 def get_parallel_results(async_test_result_queue, notify_queue): |
|
513 while True: |
|
514 async_test_result = async_test_result_queue.get() |
|
515 |
|
516 # Check if we are supposed to terminate |
|
517 if (async_test_result == None): |
|
518 return |
|
519 |
|
520 # Notify parent that we got a result |
|
521 notify_queue.put(True) |
|
522 |
|
523 yield async_test_result |
|
524 |
|
525 def process_test_results_parallel(async_test_result_queue, return_queue, notify_queue, num_tests, options): |
|
526 gen = get_parallel_results(async_test_result_queue, notify_queue) |
|
527 ok = process_test_results(gen, num_tests, options) |
|
528 return_queue.put(ok) |
|
529 |
|
530 def print_test_summary(num_tests, failures, complete, doing, options): |
|
531 if failures: |
|
532 if options.write_failures: |
|
533 try: |
|
534 out = open(options.write_failures, 'w') |
|
535 # Don't write duplicate entries when we are doing multiple failures per job. |
|
536 written = set() |
|
537 for res in failures: |
|
538 if res.test.path not in written: |
|
539 out.write(os.path.relpath(res.test.path, TEST_DIR) + '\n') |
|
540 if options.write_failure_output: |
|
541 out.write(res.out) |
|
542 out.write(res.err) |
|
543 out.write('Exit code: ' + str(res.rc) + "\n") |
|
544 written.add(res.test.path) |
|
545 out.close() |
|
546 except IOError: |
|
547 sys.stderr.write("Exception thrown trying to write failure file '%s'\n"% |
|
548 options.write_failures) |
|
549 traceback.print_exc() |
|
550 sys.stderr.write('---\n') |
|
551 |
|
552 def show_test(res): |
|
553 if options.show_failed: |
|
554 print(' ' + subprocess.list2cmdline(res.cmd)) |
|
555 else: |
|
556 print(' ' + ' '.join(res.test.jitflags + [res.test.path])) |
|
557 |
|
558 print('FAILURES:') |
|
559 for res in failures: |
|
560 if not res.timed_out: |
|
561 show_test(res) |
|
562 |
|
563 print('TIMEOUTS:') |
|
564 for res in failures: |
|
565 if res.timed_out: |
|
566 show_test(res) |
|
567 else: |
|
568 print('PASSED ALL' + ('' if complete else ' (partial run -- interrupted by user %s)' % doing)) |
|
569 |
|
570 if options.tinderbox: |
|
571 num_failures = len(failures) if failures else 0 |
|
572 print('Result summary:') |
|
573 print('Passed: %d' % (num_tests - num_failures)) |
|
574 print('Failed: %d' % num_failures) |
|
575 |
|
576 return not failures |
|
577 |
|
578 def process_test_results(results, num_tests, options): |
|
579 pb = NullProgressBar() |
|
580 if not options.hide_progress and not options.show_cmd and ProgressBar.conservative_isatty(): |
|
581 fmt = [ |
|
582 {'value': 'PASS', 'color': 'green'}, |
|
583 {'value': 'FAIL', 'color': 'red'}, |
|
584 {'value': 'TIMEOUT', 'color': 'blue'}, |
|
585 {'value': 'SKIP', 'color': 'brightgray'}, |
|
586 ] |
|
587 pb = ProgressBar(num_tests, fmt) |
|
588 |
|
589 failures = [] |
|
590 timeouts = 0 |
|
591 complete = False |
|
592 doing = 'before starting' |
|
593 try: |
|
594 for i, res in enumerate(results): |
|
595 if options.show_output: |
|
596 sys.stdout.write(res.out) |
|
597 sys.stdout.write(res.err) |
|
598 sys.stdout.write('Exit code: %s\n' % res.rc) |
|
599 if res.test.valgrind: |
|
600 sys.stdout.write(res.err) |
|
601 |
|
602 ok = check_output(res.out, res.err, res.rc, res.timed_out, res.test) |
|
603 doing = 'after %s' % res.test.relpath_tests |
|
604 if not ok: |
|
605 failures.append(res) |
|
606 if res.timed_out: |
|
607 pb.message("TIMEOUT - %s" % res.test.relpath_tests) |
|
608 timeouts += 1 |
|
609 else: |
|
610 pb.message("FAIL - %s" % res.test.relpath_tests) |
|
611 |
|
612 if options.tinderbox: |
|
613 print_tinderbox(ok, res) |
|
614 |
|
615 n = i + 1 |
|
616 pb.update(n, { |
|
617 'PASS': n - len(failures), |
|
618 'FAIL': len(failures), |
|
619 'TIMEOUT': timeouts, |
|
620 'SKIP': 0} |
|
621 ) |
|
622 complete = True |
|
623 except KeyboardInterrupt: |
|
624 print("TEST-UNEXPECTED-FAIL | jit_test.py" + |
|
625 " : Test execution interrupted by user") |
|
626 |
|
627 pb.finish(True) |
|
628 return print_test_summary(num_tests, failures, complete, doing, options) |
|
629 |
|
630 def get_serial_results(tests, prefix, options): |
|
631 for i in xrange(0, options.repeat): |
|
632 for test in tests: |
|
633 yield run_test(test, prefix, options) |
|
634 |
|
635 def run_tests(tests, prefix, options): |
|
636 gen = get_serial_results(tests, prefix, options) |
|
637 ok = process_test_results(gen, len(tests) * options.repeat, options) |
|
638 return ok |
|
639 |
|
640 def get_remote_results(tests, device, prefix, options): |
|
641 for i in xrange(0, options.repeat): |
|
642 for test in tests: |
|
643 yield run_test_remote(test, device, prefix, options) |
|
644 |
|
645 def push_libs(options, device): |
|
646 # This saves considerable time in pushing unnecessary libraries |
|
647 # to the device but needs to be updated if the dependencies change. |
|
648 required_libs = ['libnss3.so', 'libmozglue.so'] |
|
649 |
|
650 for file in os.listdir(options.local_lib): |
|
651 if file in required_libs: |
|
652 remote_file = posixpath.join(options.remote_test_root, file) |
|
653 device.pushFile(os.path.join(options.local_lib, file), remote_file) |
|
654 |
|
655 def push_progs(options, device, progs): |
|
656 for local_file in progs: |
|
657 remote_file = posixpath.join(options.remote_test_root, os.path.basename(local_file)) |
|
658 device.pushFile(local_file, remote_file) |
|
659 |
|
660 def run_tests_remote(tests, prefix, options): |
|
661 # Setup device with everything needed to run our tests. |
|
662 from mozdevice import devicemanager, devicemanagerADB, devicemanagerSUT |
|
663 |
|
664 if options.device_transport == 'adb': |
|
665 if options.device_ip: |
|
666 dm = devicemanagerADB.DeviceManagerADB(options.device_ip, options.device_port, deviceSerial=options.device_serial, packageName=None, deviceRoot=options.remote_test_root) |
|
667 else: |
|
668 dm = devicemanagerADB.DeviceManagerADB(deviceSerial=options.device_serial, packageName=None, deviceRoot=options.remote_test_root) |
|
669 else: |
|
670 dm = devicemanagerSUT.DeviceManagerSUT(options.device_ip, options.device_port, deviceRoot=options.remote_test_root) |
|
671 if options.device_ip == None: |
|
672 print('Error: you must provide a device IP to connect to via the --device option') |
|
673 sys.exit(1) |
|
674 |
|
675 # Update the test root to point to our test directory. |
|
676 jit_tests_dir = posixpath.join(options.remote_test_root, 'jit-tests') |
|
677 options.remote_test_root = posixpath.join(jit_tests_dir, 'jit-tests') |
|
678 |
|
679 # Push js shell and libraries. |
|
680 if dm.dirExists(jit_tests_dir): |
|
681 dm.removeDir(jit_tests_dir) |
|
682 dm.mkDirs(options.remote_test_root) |
|
683 push_libs(options, dm) |
|
684 push_progs(options, dm, [prefix[0]]) |
|
685 dm.chmodDir(options.remote_test_root) |
|
686 |
|
687 Test.CacheDir = posixpath.join(options.remote_test_root, '.js-cache') |
|
688 dm.mkDir(Test.CacheDir) |
|
689 |
|
690 dm.pushDir(JS_TESTS_DIR, posixpath.join(jit_tests_dir, 'tests'), timeout=600) |
|
691 |
|
692 dm.pushDir(os.path.dirname(TEST_DIR), options.remote_test_root, timeout=600) |
|
693 prefix[0] = os.path.join(options.remote_test_root, 'js') |
|
694 |
|
695 # Run all tests. |
|
696 gen = get_remote_results(tests, dm, prefix, options) |
|
697 ok = process_test_results(gen, len(tests) * options.repeat, options) |
|
698 return ok |
|
699 |
|
700 def parse_jitflags(options): |
|
701 jitflags = [ [ '-' + flag for flag in flags ] |
|
702 for flags in options.jitflags.split(',') ] |
|
703 for flags in jitflags: |
|
704 for flag in flags: |
|
705 if flag not in ('-m', '-a', '-p', '-d', '-n'): |
|
706 print('Invalid jit flag: "%s"' % flag) |
|
707 sys.exit(1) |
|
708 return jitflags |
|
709 |
|
710 def platform_might_be_android(): |
|
711 try: |
|
712 # The python package for SL4A provides an |android| module. |
|
713 # If that module is present, we're likely in SL4A-python on |
|
714 # device. False positives and negatives are possible, |
|
715 # however. |
|
716 import android |
|
717 return True |
|
718 except ImportError: |
|
719 return False |
|
720 |
|
721 def stdio_might_be_broken(): |
|
722 return platform_might_be_android() |
|
723 |
|
724 if __name__ == '__main__': |
|
725 print('Use ../jit-test/jit_test.py to run these tests.') |