1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/js/src/tests/lib/manifest.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,378 @@
1.4 +# Library for JSTest manifests.
1.5 +#
1.6 +# This includes classes for representing and parsing JS manifests.
1.7 +
1.8 +import os, os.path, re, sys
1.9 +from subprocess import Popen, PIPE
1.10 +
1.11 +from tests import TestCase
1.12 +
1.13 +
1.14 +def split_path_into_dirs(path):
1.15 + dirs = [path]
1.16 +
1.17 + while True:
1.18 + path, tail = os.path.split(path)
1.19 + if not tail:
1.20 + break
1.21 + dirs.append(path)
1.22 + return dirs
1.23 +
1.24 +class XULInfo:
1.25 + def __init__(self, abi, os, isdebug):
1.26 + self.abi = abi
1.27 + self.os = os
1.28 + self.isdebug = isdebug
1.29 + self.browserIsRemote = False
1.30 +
1.31 + def as_js(self):
1.32 + """Return JS that when executed sets up variables so that JS expression
1.33 + predicates on XUL build info evaluate properly."""
1.34 +
1.35 + return ('var xulRuntime = { OS: "%s", XPCOMABI: "%s", shell: true };' +
1.36 + 'var isDebugBuild=%s; var Android=%s; var browserIsRemote=%s') % (
1.37 + self.os,
1.38 + self.abi,
1.39 + str(self.isdebug).lower(),
1.40 + str(self.os == "Android").lower(),
1.41 + str(self.browserIsRemote).lower())
1.42 +
1.43 + @classmethod
1.44 + def create(cls, jsdir):
1.45 + """Create a XULInfo based on the current platform's characteristics."""
1.46 +
1.47 + # Our strategy is to find the autoconf.mk generated for the build and
1.48 + # read the values from there.
1.49 +
1.50 + # Find config/autoconf.mk.
1.51 + dirs = split_path_into_dirs(os.getcwd()) + split_path_into_dirs(jsdir)
1.52 +
1.53 + path = None
1.54 + for dir in dirs:
1.55 + _path = os.path.join(dir, 'config/autoconf.mk')
1.56 + if os.path.isfile(_path):
1.57 + path = _path
1.58 + break
1.59 +
1.60 + if path == None:
1.61 + print ("Can't find config/autoconf.mk on a directory containing the JS shell"
1.62 + " (searched from %s)") % jsdir
1.63 + sys.exit(1)
1.64 +
1.65 + # Read the values.
1.66 + val_re = re.compile(r'(TARGET_XPCOM_ABI|OS_TARGET|MOZ_DEBUG)\s*=\s*(.*)')
1.67 + kw = { 'isdebug': False }
1.68 + for line in open(path):
1.69 + m = val_re.match(line)
1.70 + if m:
1.71 + key, val = m.groups()
1.72 + val = val.rstrip()
1.73 + if key == 'TARGET_XPCOM_ABI':
1.74 + kw['abi'] = val
1.75 + if key == 'OS_TARGET':
1.76 + kw['os'] = val
1.77 + if key == 'MOZ_DEBUG':
1.78 + kw['isdebug'] = (val == '1')
1.79 + return cls(**kw)
1.80 +
1.81 +class XULInfoTester:
1.82 + def __init__(self, xulinfo, js_bin):
1.83 + self.js_prolog = xulinfo.as_js()
1.84 + self.js_bin = js_bin
1.85 + # Maps JS expr to evaluation result.
1.86 + self.cache = {}
1.87 +
1.88 + def test(self, cond):
1.89 + """Test a XUL predicate condition against this local info."""
1.90 + ans = self.cache.get(cond, None)
1.91 + if ans is None:
1.92 + cmd = [ self.js_bin, '-e', self.js_prolog, '-e', 'print(!!(%s))'%cond ]
1.93 + p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
1.94 + out, err = p.communicate()
1.95 + if out in ('true\n', 'true\r\n'):
1.96 + ans = True
1.97 + elif out in ('false\n', 'false\r\n'):
1.98 + ans = False
1.99 + else:
1.100 + raise Exception(("Failed to test XUL condition %r;"
1.101 + + " output was %r, stderr was %r")
1.102 + % (cond, out, err))
1.103 + self.cache[cond] = ans
1.104 + return ans
1.105 +
1.106 +class NullXULInfoTester:
1.107 + """Can be used to parse manifests without a JS shell."""
1.108 + def test(self, cond):
1.109 + return False
1.110 +
1.111 +def _parse_one(testcase, xul_tester):
1.112 + pos = 0
1.113 + parts = testcase.terms.split()
1.114 + while pos < len(parts):
1.115 + if parts[pos] == 'fails':
1.116 + testcase.expect = False
1.117 + pos += 1
1.118 + elif parts[pos] == 'skip':
1.119 + testcase.expect = testcase.enable = False
1.120 + pos += 1
1.121 + elif parts[pos] == 'random':
1.122 + testcase.random = True
1.123 + pos += 1
1.124 + elif parts[pos].startswith('fails-if'):
1.125 + cond = parts[pos][len('fails-if('):-1]
1.126 + if xul_tester.test(cond):
1.127 + testcase.expect = False
1.128 + pos += 1
1.129 + elif parts[pos].startswith('asserts-if'):
1.130 + # This directive means we may flunk some number of
1.131 + # NS_ASSERTIONs in the browser. For the shell, ignore it.
1.132 + pos += 1
1.133 + elif parts[pos].startswith('skip-if'):
1.134 + cond = parts[pos][len('skip-if('):-1]
1.135 + if xul_tester.test(cond):
1.136 + testcase.expect = testcase.enable = False
1.137 + pos += 1
1.138 + elif parts[pos].startswith('random-if'):
1.139 + cond = parts[pos][len('random-if('):-1]
1.140 + if xul_tester.test(cond):
1.141 + testcase.random = True
1.142 + pos += 1
1.143 + elif parts[pos].startswith('require-or'):
1.144 + cond = parts[pos][len('require-or('):-1]
1.145 + (preconditions, fallback_action) = re.split(",", cond)
1.146 + for precondition in re.split("&&", preconditions):
1.147 + if precondition == 'debugMode':
1.148 + testcase.options.append('-d')
1.149 + elif precondition == 'true':
1.150 + pass
1.151 + else:
1.152 + if fallback_action == "skip":
1.153 + testcase.expect = testcase.enable = False
1.154 + elif fallback_action == "fail":
1.155 + testcase.expect = False
1.156 + elif fallback_action == "random":
1.157 + testcase.random = True
1.158 + else:
1.159 + raise Exception(("Invalid precondition '%s' or fallback " +
1.160 + " action '%s'") % (precondition, fallback_action))
1.161 + break
1.162 + pos += 1
1.163 + elif parts[pos] == 'slow':
1.164 + testcase.slow = True
1.165 + pos += 1
1.166 + elif parts[pos] == 'silentfail':
1.167 + # silentfails use tons of memory, and Darwin doesn't support ulimit.
1.168 + if xul_tester.test("xulRuntime.OS == 'Darwin'"):
1.169 + testcase.expect = testcase.enable = False
1.170 + pos += 1
1.171 + else:
1.172 + print 'warning: invalid manifest line element "%s"'%parts[pos]
1.173 + pos += 1
1.174 +
1.175 +def _build_manifest_script_entry(script_name, test):
1.176 + line = []
1.177 + if test.terms:
1.178 + line.append(test.terms)
1.179 + line.append("script")
1.180 + line.append(script_name)
1.181 + if test.comment:
1.182 + line.append("#")
1.183 + line.append(test.comment)
1.184 + return ' '.join(line)
1.185 +
1.186 +def _map_prefixes_left(test_list):
1.187 + """
1.188 + Splits tests into a dictionary keyed on the first component of the test
1.189 + path, aggregating tests with a common base path into a list.
1.190 + """
1.191 + byprefix = {}
1.192 + for t in test_list:
1.193 + left, sep, remainder = t.path.partition(os.sep)
1.194 + if left not in byprefix:
1.195 + byprefix[left] = []
1.196 + if remainder:
1.197 + t.path = remainder
1.198 + byprefix[left].append(t)
1.199 + return byprefix
1.200 +
1.201 +def _emit_manifest_at(location, relative, test_list, depth):
1.202 + """
1.203 + location - str: absolute path where we want to write the manifest
1.204 + relative - str: relative path from topmost manifest directory to current
1.205 + test_list - [str]: list of all test paths and directorys
1.206 + depth - int: number of dirs we are below the topmost manifest dir
1.207 + """
1.208 + manifests = _map_prefixes_left(test_list)
1.209 +
1.210 + filename = os.path.join(location, 'jstests.list')
1.211 + manifest = []
1.212 + numTestFiles = 0
1.213 + for k, test_list in manifests.iteritems():
1.214 + fullpath = os.path.join(location, k)
1.215 + if os.path.isdir(fullpath):
1.216 + manifest.append("include " + k + "/jstests.list")
1.217 + relpath = os.path.join(relative, k)
1.218 + _emit_manifest_at(fullpath, relpath, test_list, depth + 1)
1.219 + else:
1.220 + numTestFiles += 1
1.221 + assert len(test_list) == 1
1.222 + line = _build_manifest_script_entry(k, test_list[0])
1.223 + manifest.append(line)
1.224 +
1.225 + # Always present our manifest in sorted order.
1.226 + manifest.sort()
1.227 +
1.228 + # If we have tests, we have to set the url-prefix so reftest can find them.
1.229 + if numTestFiles > 0:
1.230 + manifest = (["url-prefix %sjsreftest.html?test=%s/" % ('../' * depth, relative)]
1.231 + + manifest)
1.232 +
1.233 + fp = open(filename, 'w')
1.234 + try:
1.235 + fp.write('\n'.join(manifest) + '\n')
1.236 + finally:
1.237 + fp.close()
1.238 +
1.239 +def make_manifests(location, test_list):
1.240 + _emit_manifest_at(location, '', test_list, 0)
1.241 +
1.242 +def _find_all_js_files(base, location):
1.243 + for root, dirs, files in os.walk(location):
1.244 + root = root[len(base) + 1:]
1.245 + for fn in files:
1.246 + if fn.endswith('.js'):
1.247 + yield root, fn
1.248 +
1.249 +TEST_HEADER_PATTERN_INLINE = re.compile(r'//\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?$')
1.250 +TEST_HEADER_PATTERN_MULTI = re.compile(r'/\*\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?\*/')
1.251 +
1.252 +def _parse_test_header(fullpath, testcase, xul_tester):
1.253 + """
1.254 + This looks a bit weird. The reason is that it needs to be efficient, since
1.255 + it has to be done on every test
1.256 + """
1.257 + fp = open(fullpath, 'r')
1.258 + try:
1.259 + buf = fp.read(512)
1.260 + finally:
1.261 + fp.close()
1.262 +
1.263 + # Bail early if we do not start with a single comment.
1.264 + if not buf.startswith("//"):
1.265 + return
1.266 +
1.267 + # Extract the token.
1.268 + buf, _, _ = buf.partition('\n')
1.269 + matches = TEST_HEADER_PATTERN_INLINE.match(buf)
1.270 +
1.271 + if not matches:
1.272 + matches = TEST_HEADER_PATTERN_MULTI.match(buf)
1.273 + if not matches:
1.274 + return
1.275 +
1.276 + testcase.tag = matches.group(1)
1.277 + testcase.terms = matches.group(2)
1.278 + testcase.comment = matches.group(4)
1.279 +
1.280 + _parse_one(testcase, xul_tester)
1.281 +
1.282 +def _parse_external_manifest(filename, relpath):
1.283 + """
1.284 + Reads an external manifest file for test suites whose individual test cases
1.285 + can't be decorated with reftest comments.
1.286 + filename - str: name of the manifest file
1.287 + relpath - str: relative path of the directory containing the manifest
1.288 + within the test suite
1.289 + """
1.290 + entries = []
1.291 +
1.292 + with open(filename, 'r') as fp:
1.293 + manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$')
1.294 + for line in fp:
1.295 + line, _, comment = line.partition('#')
1.296 + line = line.strip()
1.297 + if not line:
1.298 + continue
1.299 + matches = manifest_re.match(line)
1.300 + if not matches:
1.301 + print('warning: unrecognized line in jstests.list: {0}'.format(line))
1.302 + continue
1.303 +
1.304 + path = os.path.normpath(os.path.join(relpath, matches.group(3)))
1.305 + if matches.group(2) == 'include':
1.306 + # The manifest spec wants a reference to another manifest here,
1.307 + # but we need just the directory. We do need the trailing
1.308 + # separator so we don't accidentally match other paths of which
1.309 + # this one is a prefix.
1.310 + assert(path.endswith('jstests.list'))
1.311 + path = path[:-len('jstests.list')]
1.312 +
1.313 + entries.append({'path': path, 'terms': matches.group(1), 'comment': comment.strip()})
1.314 +
1.315 + # if one directory name is a prefix of another, we want the shorter one first
1.316 + entries.sort(key=lambda x: x["path"])
1.317 + return entries
1.318 +
1.319 +def _apply_external_manifests(filename, testcase, entries, xul_tester):
1.320 + for entry in entries:
1.321 + if filename.startswith(entry["path"]):
1.322 + # The reftest spec would require combining the terms (failure types)
1.323 + # that may already be defined in the test case with the terms
1.324 + # specified in entry; for example, a skip overrides a random, which
1.325 + # overrides a fails. Since we don't necessarily know yet in which
1.326 + # environment the test cases will be run, we'd also have to
1.327 + # consider skip-if, random-if, and fails-if with as-yet unresolved
1.328 + # conditions.
1.329 + # At this point, we use external manifests only for test cases
1.330 + # that can't have their own failure type comments, so we simply
1.331 + # use the terms for the most specific path.
1.332 + testcase.terms = entry["terms"]
1.333 + testcase.comment = entry["comment"]
1.334 + _parse_one(testcase, xul_tester)
1.335 +
1.336 +def load(location, xul_tester, reldir = ''):
1.337 + """
1.338 + Locates all tests by walking the filesystem starting at |location|.
1.339 + Uses xul_tester to evaluate any test conditions in the test header.
1.340 + Failure type and comment for a test case can come from
1.341 + - an external manifest entry for the test case,
1.342 + - an external manifest entry for a containing directory,
1.343 + - most commonly: the header of the test case itself.
1.344 + """
1.345 + # The list of tests that we are collecting.
1.346 + tests = []
1.347 +
1.348 + # Any file whose basename matches something in this set is ignored.
1.349 + EXCLUDED = set(('browser.js', 'shell.js', 'jsref.js', 'template.js',
1.350 + 'user.js', 'sta.js',
1.351 + 'test262-browser.js', 'test262-shell.js',
1.352 + 'test402-browser.js', 'test402-shell.js',
1.353 + 'testBuiltInObject.js', 'testIntl.js',
1.354 + 'js-test-driver-begin.js', 'js-test-driver-end.js'))
1.355 +
1.356 + manifestFile = os.path.join(location, 'jstests.list')
1.357 + externalManifestEntries = _parse_external_manifest(manifestFile, '')
1.358 +
1.359 + for root, basename in _find_all_js_files(location, location):
1.360 + # Skip js files in the root test directory.
1.361 + if not root:
1.362 + continue
1.363 +
1.364 + # Skip files that we know are not tests.
1.365 + if basename in EXCLUDED:
1.366 + continue
1.367 +
1.368 + # Get the full path and relative location of the file.
1.369 + filename = os.path.join(root, basename)
1.370 + fullpath = os.path.join(location, filename)
1.371 +
1.372 + # Skip empty files.
1.373 + statbuf = os.stat(fullpath)
1.374 + if statbuf.st_size == 0:
1.375 + continue
1.376 +
1.377 + testcase = TestCase(os.path.join(reldir, filename))
1.378 + _apply_external_manifests(filename, testcase, externalManifestEntries, xul_tester)
1.379 + _parse_test_header(fullpath, testcase, xul_tester)
1.380 + tests.append(testcase)
1.381 + return tests
