The Tor Browser / file comparison

comparison: layout/tools/reftest/mach_commands.py

layout/tools/reftest/mach_commands.py

branch
TOR_BUG_9701
changeset 13
44a2da4a2ab2
equal deleted inserted replaced
-1:000000000000 0:98153a7dca3e
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5 from __future__ import unicode_literals
6
7 import mozpack.path
8 import os
9 import re
10 import sys
11 import warnings
12 import which
13
14 from mozbuild.base import (
15 MachCommandBase,
16 MachCommandConditions as conditions,
17 MozbuildObject,
18 )
19
20 from mach.decorators import (
21 CommandArgument,
22 CommandProvider,
23 Command,
24 )
25
26
27 DEBUGGER_HELP = 'Debugger binary to run test in. Program name or path.'
28
29 ADB_NOT_FOUND = '''
30 The %s command requires the adb binary to be on your path.
31
32 If you have a B2G build, this can be found in
33 '%s/out/host/<platform>/bin'.
34 '''.lstrip()
35
36 GAIA_PROFILE_NOT_FOUND = '''
37 The %s command requires a non-debug gaia profile. Either pass in --profile,
38 or set the GAIA_PROFILE environment variable.
39
40 If you do not have a non-debug gaia profile, you can build one:
41 $ git clone https://github.com/mozilla-b2g/gaia
42 $ cd gaia
43 $ make
44
45 The profile should be generated in a directory called 'profile'.
46 '''.lstrip()
47
48 GAIA_PROFILE_IS_DEBUG = '''
49 The %s command requires a non-debug gaia profile. The specified profile,
50 %s, is a debug profile.
51
52 If you do not have a non-debug gaia profile, you can build one:
53 $ git clone https://github.com/mozilla-b2g/gaia
54 $ cd gaia
55 $ make
56
57 The profile should be generated in a directory called 'profile'.
58 '''.lstrip()
59
60 MARIONETTE_DISABLED = '''
61 The %s command requires a marionette enabled build.
62
63 Add 'ENABLE_MARIONETTE=1' to your mozconfig file and re-build the application.
64 Your currently active mozconfig is %s.
65 '''.lstrip()
66
67 class ReftestRunner(MozbuildObject):
68 """Easily run reftests.
69
70 This currently contains just the basics for running reftests. We may want
71 to hook up result parsing, etc.
72 """
73 def __init__(self, *args, **kwargs):
74 MozbuildObject.__init__(self, *args, **kwargs)
75
76 # TODO Bug 794506 remove once mach integrates with virtualenv.
77 build_path = os.path.join(self.topobjdir, 'build')
78 if build_path not in sys.path:
79 sys.path.append(build_path)
80
81 self.tests_dir = os.path.join(self.topobjdir, '_tests')
82 self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
83
84 def _manifest_file(self, suite):
85 """Returns the manifest file used for a given test suite."""
86 files = {
87 'reftest': 'reftest.list',
88 'reftest-ipc': 'reftest.list',
89 'crashtest': 'crashtests.list',
90 'crashtest-ipc': 'crashtests.list',
91 }
92 assert suite in files
93 return files[suite]
94
95 def _find_manifest(self, suite, test_file):
96 assert test_file
97 path_arg = self._wrap_path_argument(test_file)
98 relpath = path_arg.relpath()
99
100 if os.path.isdir(path_arg.srcdir_path()):
101 return mozpack.path.join(relpath, self._manifest_file(suite))
102
103 if relpath.endswith('.list'):
104 return relpath
105
106 raise Exception('Running a single test is not currently supported')
107
108 def _make_shell_string(self, s):
109 return "'%s'" % re.sub("'", r"'\''", s)
110
111 def run_b2g_test(self, b2g_home=None, xre_path=None, test_file=None,
112 suite=None, **kwargs):
113 """Runs a b2g reftest.
114
115 test_file is a path to a test file. It can be a relative path from the
116 top source directory, an absolute filename, or a directory containing
117 test files.
118
119 suite is the type of reftest to run. It can be one of ('reftest',
120 'crashtest').
121 """
122 if suite not in ('reftest', 'crashtest'):
123 raise Exception('None or unrecognized reftest suite type.')
124
125 # Find the manifest file
126 if not test_file:
127 if suite == 'reftest':
128 test_file = mozpack.path.join('layout', 'reftests')
129 elif suite == 'crashtest':
130 test_file = mozpack.path.join('testing', 'crashtest')
131
132 if not os.path.exists(os.path.join(self.topsrcdir, test_file)):
133 test_file = mozpack.path.relpath(os.path.abspath(test_file),
134 self.topsrcdir)
135
136 manifest = self._find_manifest(suite, test_file)
137 if not os.path.exists(mozpack.path.join(self.topsrcdir, manifest)):
138 raise Exception('No manifest file was found at %s.' % manifest)
139
140 # Need to chdir to reftest_dir otherwise imports fail below.
141 os.chdir(self.reftest_dir)
142
143 # The imp module can spew warnings if the modules below have
144 # already been imported, ignore them.
145 with warnings.catch_warnings():
146 warnings.simplefilter('ignore')
147
148 import imp
149 path = os.path.join(self.reftest_dir, 'runreftestb2g.py')
150 with open(path, 'r') as fh:
151 imp.load_module('reftest', fh, path, ('.py', 'r', imp.PY_SOURCE))
152 import reftest
153
154 # Set up the reftest options.
155 parser = reftest.B2GOptions()
156 options, args = parser.parse_args([])
157
158 # Tests need to be served from a subdirectory of the server. Symlink
159 # topsrcdir here to get around this.
160 tests = os.path.join(self.reftest_dir, 'tests')
161 if not os.path.isdir(tests):
162 os.symlink(self.topsrcdir, tests)
163 args.insert(0, os.path.join('tests', manifest))
164
165 for k, v in kwargs.iteritems():
166 setattr(options, k, v)
167
168 if conditions.is_b2g_desktop(self):
169 if self.substs.get('ENABLE_MARIONETTE') != '1':
170 print(MARIONETTE_DISABLED % ('mochitest-b2g-desktop',
171 self.mozconfig['path']))
172 return 1
173
174 options.profile = options.profile or os.environ.get('GAIA_PROFILE')
175 if not options.profile:
176 print(GAIA_PROFILE_NOT_FOUND % 'reftest-b2g-desktop')
177 return 1
178
179 if os.path.isfile(os.path.join(options.profile, 'extensions', \
180 'httpd@gaiamobile.org')):
181 print(GAIA_PROFILE_IS_DEBUG % ('mochitest-b2g-desktop',
182 options.profile))
183 return 1
184
185 options.desktop = True
186 options.app = self.get_binary_path()
187 if not options.app.endswith('-bin'):
188 options.app = '%s-bin' % options.app
189 if not os.path.isfile(options.app):
190 options.app = options.app[:-len('-bin')]
191
192 return reftest.run_desktop_reftests(parser, options, args)
193
194
195 try:
196 which.which('adb')
197 except which.WhichError:
198 # TODO Find adb automatically if it isn't on the path
199 raise Exception(ADB_NOT_FOUND % ('%s-remote' % suite, b2g_home))
200
201 options.b2gPath = b2g_home
202 options.logcat_dir = self.reftest_dir
203 options.httpdPath = os.path.join(self.topsrcdir, 'netwerk', 'test', 'httpserver')
204 options.xrePath = xre_path
205 options.ignoreWindowSize = True
206 return reftest.run_remote_reftests(parser, options, args)
207
208 def run_desktop_test(self, test_file=None, filter=None, suite=None,
209 debugger=None, parallel=False, shuffle=False,
210 e10s=False, this_chunk=None, total_chunks=None):
211 """Runs a reftest.
212
213 test_file is a path to a test file. It can be a relative path from the
214 top source directory, an absolute filename, or a directory containing
215 test files.
216
217 filter is a regular expression (in JS syntax, as could be passed to the
218 RegExp constructor) to select which reftests to run from the manifest.
219
220 suite is the type of reftest to run. It can be one of ('reftest',
221 'crashtest').
222
223 debugger is the program name (in $PATH) or the full path of the
224 debugger to run.
225
226 parallel indicates whether tests should be run in parallel or not.
227
228 shuffle indicates whether to run tests in random order.
229 """
230
231 if suite not in ('reftest', 'reftest-ipc', 'crashtest', 'crashtest-ipc'):
232 raise Exception('None or unrecognized reftest suite type.')
233
234 env = {}
235 extra_args = []
236
237 if test_file:
238 path = self._find_manifest(suite, test_file)
239 if not os.path.exists(mozpack.path.join(self.topsrcdir, path)):
240 raise Exception('No manifest file was found at %s.' % path)
241 env[b'TEST_PATH'] = path
242 if filter:
243 extra_args.extend(['--filter', self._make_shell_string(filter)])
244
245 pass_thru = False
246
247 if debugger:
248 extra_args.append('--debugger=%s' % debugger)
249 pass_thru = True
250
251 if parallel:
252 extra_args.append('--run-tests-in-parallel')
253
254 if shuffle:
255 extra_args.append('--shuffle')
256
257 if e10s:
258 extra_args.append('--e10s')
259
260 if this_chunk:
261 extra_args.append('--this-chunk=%s' % this_chunk)
262
263 if total_chunks:
264 extra_args.append('--total-chunks=%s' % total_chunks)
265
266 if extra_args:
267 args = [os.environ.get(b'EXTRA_TEST_ARGS', '')]
268 args.extend(extra_args)
269 env[b'EXTRA_TEST_ARGS'] = ' '.join(args)
270
271 # TODO hook up harness via native Python
272 return self._run_make(directory='.', target=suite, append_env=env,
273 pass_thru=pass_thru, ensure_exit_code=False)
274
275
276 def ReftestCommand(func):
277 """Decorator that adds shared command arguments to reftest commands."""
278
279 debugger = CommandArgument('--debugger', metavar='DEBUGGER',
280 help=DEBUGGER_HELP)
281 func = debugger(func)
282
283 flter = CommandArgument('--filter', metavar='REGEX',
284 help='A JS regular expression to match test URLs against, to select '
285 'a subset of tests to run.')
286 func = flter(func)
287
288 path = CommandArgument('test_file', nargs='?', metavar='MANIFEST',
289 help='Reftest manifest file, or a directory in which to select '
290 'reftest.list. If omitted, the entire test suite is executed.')
291 func = path(func)
292
293 parallel = CommandArgument('--parallel', action='store_true',
294 help='Run tests in parallel.')
295 func = parallel(func)
296
297 shuffle = CommandArgument('--shuffle', action='store_true',
298 help='Run tests in random order.')
299 func = shuffle(func)
300
301 e10s = CommandArgument('--e10s', action='store_true',
302 help='Use content processes.')
303 func = e10s(func)
304
305 totalChunks = CommandArgument('--total-chunks',
306 help = 'How many chunks to split the tests up into.')
307 func = totalChunks(func)
308
309 thisChunk = CommandArgument('--this-chunk',
310 help = 'Which chunk to run between 1 and --total-chunks.')
311 func = thisChunk(func)
312
313 return func
314
315 def B2GCommand(func):
316 """Decorator that adds shared command arguments to b2g mochitest commands."""
317
318 busybox = CommandArgument('--busybox', default=None,
319 help='Path to busybox binary to install on device')
320 func = busybox(func)
321
322 logcatdir = CommandArgument('--logcat-dir', default=None,
323 help='directory to store logcat dump files')
324 func = logcatdir(func)
325
326 geckopath = CommandArgument('--gecko-path', default=None,
327 help='the path to a gecko distribution that should \
328 be installed on the emulator prior to test')
329 func = geckopath(func)
330
331 sdcard = CommandArgument('--sdcard', default="10MB",
332 help='Define size of sdcard: 1MB, 50MB...etc')
333 func = sdcard(func)
334
335 emulator_res = CommandArgument('--emulator-res', default='800x1000',
336 help='Emulator resolution of the format \'<width>x<height>\'')
337 func = emulator_res(func)
338
339 emulator = CommandArgument('--emulator', default='arm',
340 help='Architecture of emulator to use: x86 or arm')
341 func = emulator(func)
342
343 marionette = CommandArgument('--marionette', default=None,
344 help='host:port to use when connecting to Marionette')
345 func = marionette(func)
346
347 totalChunks = CommandArgument('--total-chunks', dest='totalChunks',
348 help = 'How many chunks to split the tests up into.')
349 func = totalChunks(func)
350
351 thisChunk = CommandArgument('--this-chunk', dest='thisChunk',
352 help = 'Which chunk to run between 1 and --total-chunks.')
353 func = thisChunk(func)
354
355 path = CommandArgument('test_file', default=None, nargs='?',
356 metavar='TEST',
357 help='Test to run. Can be specified as a single file, a ' \
358 'directory, or omitted. If omitted, the entire test suite is ' \
359 'executed.')
360 func = path(func)
361
362 return func
363
364
365 @CommandProvider
366 class MachCommands(MachCommandBase):
367 @Command('reftest', category='testing', description='Run reftests.')
368 @ReftestCommand
369 def run_reftest(self, test_file, **kwargs):
370 return self._run_reftest(test_file, suite='reftest', **kwargs)
371
372 @Command('reftest-ipc', category='testing',
373 description='Run IPC reftests.')
374 @ReftestCommand
375 def run_ipc(self, test_file, **kwargs):
376 return self._run_reftest(test_file, suite='reftest-ipc', **kwargs)
377
378 @Command('crashtest', category='testing',
379 description='Run crashtests.')
380 @ReftestCommand
381 def run_crashtest(self, test_file, **kwargs):
382 return self._run_reftest(test_file, suite='crashtest', **kwargs)
383
384 @Command('crashtest-ipc', category='testing',
385 description='Run IPC crashtests.')
386 @ReftestCommand
387 def run_crashtest_ipc(self, test_file, **kwargs):
388 return self._run_reftest(test_file, suite='crashtest-ipc', **kwargs)
389
390 def _run_reftest(self, test_file=None, suite=None, **kwargs):
391 reftest = self._spawn(ReftestRunner)
392 return reftest.run_desktop_test(test_file, suite=suite, **kwargs)
393
394
395 # TODO For now b2g commands will only work with the emulator,
396 # they should be modified to work with all devices.
397 def is_emulator(cls):
398 """Emulator needs to be configured."""
399 return cls.device_name.find('emulator') == 0
400
401
402 @CommandProvider
403 class B2GCommands(MachCommandBase):
404 def __init__(self, context):
405 MachCommandBase.__init__(self, context)
406
407 for attr in ('b2g_home', 'xre_path', 'device_name'):
408 setattr(self, attr, getattr(context, attr, None))
409
410 @Command('reftest-remote', category='testing',
411 description='Run a remote reftest.',
412 conditions=[conditions.is_b2g, is_emulator])
413 @B2GCommand
414 def run_reftest_remote(self, test_file, **kwargs):
415 return self._run_reftest(test_file, suite='reftest', **kwargs)
416
417 @Command('reftest-b2g-desktop', category='testing',
418 description='Run a b2g desktop reftest.',
419 conditions=[conditions.is_b2g_desktop])
420 @B2GCommand
421 def run_reftest_b2g_desktop(self, test_file, **kwargs):
422 return self._run_reftest(test_file, suite='reftest', **kwargs)
423
424 @Command('crashtest-remote', category='testing',
425 description='Run a remote crashtest.',
426 conditions=[conditions.is_b2g, is_emulator])
427 @B2GCommand
428 def run_crashtest_remote(self, test_file, **kwargs):
429 return self._run_reftest(test_file, suite='crashtest', **kwargs)
430
431 def _run_reftest(self, test_file=None, suite=None, **kwargs):
432 reftest = self._spawn(ReftestRunner)
433 return reftest.run_b2g_test(self.b2g_home, self.xre_path,
434 test_file, suite=suite, **kwargs)

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial