michael@0: # Library for JSTest manifests.
michael@0: #
michael@0: # This includes classes for representing and parsing JS manifests.
michael@0: 
michael@0: import os, os.path, re, sys
michael@0: from subprocess import Popen, PIPE
michael@0: 
michael@0: from tests import TestCase
michael@0: 
michael@0: 
michael@0: def split_path_into_dirs(path):
michael@0:     dirs = [path]
michael@0: 
michael@0:     while True:
michael@0:         path, tail = os.path.split(path)
michael@0:         if not tail:
michael@0:             break
michael@0:         dirs.append(path)
michael@0:     return dirs
michael@0: 
michael@0: class XULInfo:
michael@0:     def __init__(self, abi, os, isdebug):
michael@0:         self.abi = abi
michael@0:         self.os = os
michael@0:         self.isdebug = isdebug
michael@0:         self.browserIsRemote = False
michael@0: 
michael@0:     def as_js(self):
michael@0:         """Return JS that when executed sets up variables so that JS expression
michael@0:         predicates on XUL build info evaluate properly."""
michael@0: 
michael@0:         return ('var xulRuntime = { OS: "%s", XPCOMABI: "%s", shell: true };' +
michael@0:                 'var isDebugBuild=%s; var Android=%s; var browserIsRemote=%s') % (
michael@0:             self.os,
michael@0:             self.abi,
michael@0:             str(self.isdebug).lower(),
michael@0:             str(self.os == "Android").lower(),
michael@0:             str(self.browserIsRemote).lower())
michael@0: 
michael@0:     @classmethod
michael@0:     def create(cls, jsdir):
michael@0:         """Create a XULInfo based on the current platform's characteristics."""
michael@0: 
michael@0:         # Our strategy is to find the autoconf.mk generated for the build and
michael@0:         # read the values from there.
michael@0: 
michael@0:         # Find config/autoconf.mk.
michael@0:         dirs = split_path_into_dirs(os.getcwd()) + split_path_into_dirs(jsdir)
michael@0: 
michael@0:         path = None
michael@0:         for dir in dirs:
michael@0:             _path = os.path.join(dir, 'config/autoconf.mk')
michael@0:             if os.path.isfile(_path):
michael@0:                 path = _path
michael@0:                 break
michael@0: 
michael@0:         if path == None:
michael@0:             print ("Can't find config/autoconf.mk on a directory containing the JS shell"
michael@0:                    " (searched from %s)") % jsdir
michael@0:             sys.exit(1)
michael@0: 
michael@0:         # Read the values.
michael@0:         val_re = re.compile(r'(TARGET_XPCOM_ABI|OS_TARGET|MOZ_DEBUG)\s*=\s*(.*)')
michael@0:         kw = { 'isdebug': False }
michael@0:         for line in open(path):
michael@0:             m = val_re.match(line)
michael@0:             if m:
michael@0:                 key, val = m.groups()
michael@0:                 val = val.rstrip()
michael@0:                 if key == 'TARGET_XPCOM_ABI':
michael@0:                     kw['abi'] = val
michael@0:                 if key == 'OS_TARGET':
michael@0:                     kw['os'] = val
michael@0:                 if key == 'MOZ_DEBUG':
michael@0:                     kw['isdebug'] = (val == '1')
michael@0:         return cls(**kw)
michael@0: 
michael@0: class XULInfoTester:
michael@0:     def __init__(self, xulinfo, js_bin):
michael@0:         self.js_prolog = xulinfo.as_js()
michael@0:         self.js_bin = js_bin
michael@0:         # Maps JS expr to evaluation result.
michael@0:         self.cache = {}
michael@0: 
michael@0:     def test(self, cond):
michael@0:         """Test a XUL predicate condition against this local info."""
michael@0:         ans = self.cache.get(cond, None)
michael@0:         if ans is None:
michael@0:             cmd = [ self.js_bin, '-e', self.js_prolog, '-e', 'print(!!(%s))'%cond ]
michael@0:             p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
michael@0:             out, err = p.communicate()
michael@0:             if out in ('true\n', 'true\r\n'):
michael@0:                 ans = True
michael@0:             elif out in ('false\n', 'false\r\n'):
michael@0:                 ans = False
michael@0:             else:
michael@0:                 raise Exception(("Failed to test XUL condition %r;"
michael@0:                                  + " output was %r, stderr was %r")
michael@0:                                  % (cond, out, err))
michael@0:             self.cache[cond] = ans
michael@0:         return ans
michael@0: 
michael@0: class NullXULInfoTester:
michael@0:     """Can be used to parse manifests without a JS shell."""
michael@0:     def test(self, cond):
michael@0:         return False
michael@0: 
michael@0: def _parse_one(testcase, xul_tester):
michael@0:     pos = 0
michael@0:     parts = testcase.terms.split()
michael@0:     while pos < len(parts):
michael@0:         if parts[pos] == 'fails':
michael@0:             testcase.expect = False
michael@0:             pos += 1
michael@0:         elif parts[pos] == 'skip':
michael@0:             testcase.expect = testcase.enable = False
michael@0:             pos += 1
michael@0:         elif parts[pos] == 'random':
michael@0:             testcase.random = True
michael@0:             pos += 1
michael@0:         elif parts[pos].startswith('fails-if'):
michael@0:             cond = parts[pos][len('fails-if('):-1]
michael@0:             if xul_tester.test(cond):
michael@0:                 testcase.expect = False
michael@0:             pos += 1
michael@0:         elif parts[pos].startswith('asserts-if'):
michael@0:             # This directive means we may flunk some number of
michael@0:             # NS_ASSERTIONs in the browser. For the shell, ignore it.
michael@0:             pos += 1
michael@0:         elif parts[pos].startswith('skip-if'):
michael@0:             cond = parts[pos][len('skip-if('):-1]
michael@0:             if xul_tester.test(cond):
michael@0:                 testcase.expect = testcase.enable = False
michael@0:             pos += 1
michael@0:         elif parts[pos].startswith('random-if'):
michael@0:             cond = parts[pos][len('random-if('):-1]
michael@0:             if xul_tester.test(cond):
michael@0:                 testcase.random = True
michael@0:             pos += 1
michael@0:         elif parts[pos].startswith('require-or'):
michael@0:             cond = parts[pos][len('require-or('):-1]
michael@0:             (preconditions, fallback_action) = re.split(",", cond)
michael@0:             for precondition in re.split("&&", preconditions):
michael@0:                 if precondition == 'debugMode':
michael@0:                     testcase.options.append('-d')
michael@0:                 elif precondition == 'true':
michael@0:                     pass
michael@0:                 else:
michael@0:                     if fallback_action == "skip":
michael@0:                         testcase.expect = testcase.enable = False
michael@0:                     elif fallback_action == "fail":
michael@0:                         testcase.expect = False
michael@0:                     elif fallback_action == "random":
michael@0:                         testcase.random = True
michael@0:                     else:
michael@0:                         raise Exception(("Invalid precondition '%s' or fallback " +
michael@0:                                          " action '%s'") % (precondition, fallback_action))
michael@0:                     break
michael@0:             pos += 1
michael@0:         elif parts[pos] == 'slow':
michael@0:             testcase.slow = True
michael@0:             pos += 1
michael@0:         elif parts[pos] == 'silentfail':
michael@0:             # silentfails use tons of memory, and Darwin doesn't support ulimit.
michael@0:             if xul_tester.test("xulRuntime.OS == 'Darwin'"):
michael@0:                 testcase.expect = testcase.enable = False
michael@0:             pos += 1
michael@0:         else:
michael@0:             print 'warning: invalid manifest line element "%s"'%parts[pos]
michael@0:             pos += 1
michael@0: 
michael@0: def _build_manifest_script_entry(script_name, test):
michael@0:     line = []
michael@0:     if test.terms:
michael@0:         line.append(test.terms)
michael@0:     line.append("script")
michael@0:     line.append(script_name)
michael@0:     if test.comment:
michael@0:         line.append("#")
michael@0:         line.append(test.comment)
michael@0:     return ' '.join(line)
michael@0: 
michael@0: def _map_prefixes_left(test_list):
michael@0:     """
michael@0:     Splits tests into a dictionary keyed on the first component of the test
michael@0:     path, aggregating tests with a common base path into a list.
michael@0:     """
michael@0:     byprefix = {}
michael@0:     for t in test_list:
michael@0:         left, sep, remainder = t.path.partition(os.sep)
michael@0:         if left not in byprefix:
michael@0:             byprefix[left] = []
michael@0:         if remainder:
michael@0:             t.path = remainder
michael@0:         byprefix[left].append(t)
michael@0:     return byprefix
michael@0: 
michael@0: def _emit_manifest_at(location, relative, test_list, depth):
michael@0:     """
michael@0:     location  - str: absolute path where we want to write the manifest
michael@0:     relative  - str: relative path from topmost manifest directory to current
michael@0:     test_list - [str]: list of all test paths and directorys
michael@0:     depth     - int: number of dirs we are below the topmost manifest dir
michael@0:     """
michael@0:     manifests = _map_prefixes_left(test_list)
michael@0: 
michael@0:     filename = os.path.join(location, 'jstests.list')
michael@0:     manifest = []
michael@0:     numTestFiles = 0
michael@0:     for k, test_list in manifests.iteritems():
michael@0:         fullpath = os.path.join(location, k)
michael@0:         if os.path.isdir(fullpath):
michael@0:             manifest.append("include " + k + "/jstests.list")
michael@0:             relpath = os.path.join(relative, k)
michael@0:             _emit_manifest_at(fullpath, relpath, test_list, depth + 1)
michael@0:         else:
michael@0:             numTestFiles += 1
michael@0:             assert len(test_list) == 1
michael@0:             line = _build_manifest_script_entry(k, test_list[0])
michael@0:             manifest.append(line)
michael@0: 
michael@0:     # Always present our manifest in sorted order.
michael@0:     manifest.sort()
michael@0: 
michael@0:     # If we have tests, we have to set the url-prefix so reftest can find them.
michael@0:     if numTestFiles > 0:
michael@0:         manifest = (["url-prefix %sjsreftest.html?test=%s/" % ('../' * depth, relative)]
michael@0:                     + manifest)
michael@0: 
michael@0:     fp = open(filename, 'w')
michael@0:     try:
michael@0:         fp.write('\n'.join(manifest) + '\n')
michael@0:     finally:
michael@0:         fp.close()
michael@0: 
michael@0: def make_manifests(location, test_list):
michael@0:     _emit_manifest_at(location, '', test_list, 0)
michael@0: 
michael@0: def _find_all_js_files(base, location):
michael@0:     for root, dirs, files in os.walk(location):
michael@0:         root = root[len(base) + 1:]
michael@0:         for fn in files:
michael@0:             if fn.endswith('.js'):
michael@0:                 yield root, fn
michael@0: 
michael@0: TEST_HEADER_PATTERN_INLINE = re.compile(r'//\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?$')
michael@0: TEST_HEADER_PATTERN_MULTI  = re.compile(r'/\*\s*\|(.*?)\|\s*(.*?)\s*(--\s*(.*))?\*/')
michael@0: 
michael@0: def _parse_test_header(fullpath, testcase, xul_tester):
michael@0:     """
michael@0:     This looks a bit weird.  The reason is that it needs to be efficient, since
michael@0:     it has to be done on every test
michael@0:     """
michael@0:     fp = open(fullpath, 'r')
michael@0:     try:
michael@0:         buf = fp.read(512)
michael@0:     finally:
michael@0:         fp.close()
michael@0: 
michael@0:     # Bail early if we do not start with a single comment.
michael@0:     if not buf.startswith("//"):
michael@0:         return
michael@0: 
michael@0:     # Extract the token.
michael@0:     buf, _, _ = buf.partition('\n')
michael@0:     matches = TEST_HEADER_PATTERN_INLINE.match(buf)
michael@0: 
michael@0:     if not matches:
michael@0:         matches = TEST_HEADER_PATTERN_MULTI.match(buf)
michael@0:         if not matches:
michael@0:             return
michael@0: 
michael@0:     testcase.tag = matches.group(1)
michael@0:     testcase.terms = matches.group(2)
michael@0:     testcase.comment = matches.group(4)
michael@0: 
michael@0:     _parse_one(testcase, xul_tester)
michael@0: 
michael@0: def _parse_external_manifest(filename, relpath):
michael@0:     """
michael@0:     Reads an external manifest file for test suites whose individual test cases
michael@0:     can't be decorated with reftest comments.
michael@0:     filename - str: name of the manifest file
michael@0:     relpath - str: relative path of the directory containing the manifest
michael@0:                    within the test suite
michael@0:     """
michael@0:     entries = []
michael@0: 
michael@0:     with open(filename, 'r') as fp:
michael@0:         manifest_re = re.compile(r'^\s*(.*)\s+(include|script)\s+(\S+)$')
michael@0:         for line in fp:
michael@0:             line, _, comment = line.partition('#')
michael@0:             line = line.strip()
michael@0:             if not line:
michael@0:                 continue
michael@0:             matches = manifest_re.match(line)
michael@0:             if not matches:
michael@0:                 print('warning: unrecognized line in jstests.list: {0}'.format(line))
michael@0:                 continue
michael@0: 
michael@0:             path = os.path.normpath(os.path.join(relpath, matches.group(3)))
michael@0:             if matches.group(2) == 'include':
michael@0:                 # The manifest spec wants a reference to another manifest here,
michael@0:                 # but we need just the directory. We do need the trailing
michael@0:                 # separator so we don't accidentally match other paths of which
michael@0:                 # this one is a prefix.
michael@0:                 assert(path.endswith('jstests.list'))
michael@0:                 path = path[:-len('jstests.list')]
michael@0: 
michael@0:             entries.append({'path': path, 'terms': matches.group(1), 'comment': comment.strip()})
michael@0: 
michael@0:     # if one directory name is a prefix of another, we want the shorter one first
michael@0:     entries.sort(key=lambda x: x["path"])
michael@0:     return entries
michael@0: 
michael@0: def _apply_external_manifests(filename, testcase, entries, xul_tester):
michael@0:     for entry in entries:
michael@0:         if filename.startswith(entry["path"]):
michael@0:             # The reftest spec would require combining the terms (failure types)
michael@0:             # that may already be defined in the test case with the terms
michael@0:             # specified in entry; for example, a skip overrides a random, which
michael@0:             # overrides a fails. Since we don't necessarily know yet in which
michael@0:             # environment the test cases will be run, we'd also have to
michael@0:             # consider skip-if, random-if, and fails-if with as-yet unresolved
michael@0:             # conditions.
michael@0:             # At this point, we use external manifests only for test cases
michael@0:             # that can't have their own failure type comments, so we simply
michael@0:             # use the terms for the most specific path.
michael@0:             testcase.terms = entry["terms"]
michael@0:             testcase.comment = entry["comment"]
michael@0:             _parse_one(testcase, xul_tester)
michael@0: 
michael@0: def load(location, xul_tester, reldir = ''):
michael@0:     """
michael@0:     Locates all tests by walking the filesystem starting at |location|.
michael@0:     Uses xul_tester to evaluate any test conditions in the test header.
michael@0:     Failure type and comment for a test case can come from
michael@0:     - an external manifest entry for the test case,
michael@0:     - an external manifest entry for a containing directory,
michael@0:     - most commonly: the header of the test case itself.
michael@0:     """
michael@0:     # The list of tests that we are collecting.
michael@0:     tests = []
michael@0: 
michael@0:     # Any file whose basename matches something in this set is ignored.
michael@0:     EXCLUDED = set(('browser.js', 'shell.js', 'jsref.js', 'template.js',
michael@0:                     'user.js', 'sta.js',
michael@0:                     'test262-browser.js', 'test262-shell.js',
michael@0:                     'test402-browser.js', 'test402-shell.js',
michael@0:                     'testBuiltInObject.js', 'testIntl.js',
michael@0:                     'js-test-driver-begin.js', 'js-test-driver-end.js'))
michael@0: 
michael@0:     manifestFile = os.path.join(location, 'jstests.list')
michael@0:     externalManifestEntries = _parse_external_manifest(manifestFile, '')
michael@0: 
michael@0:     for root, basename in _find_all_js_files(location, location):
michael@0:         # Skip js files in the root test directory.
michael@0:         if not root:
michael@0:             continue
michael@0: 
michael@0:         # Skip files that we know are not tests.
michael@0:         if basename in EXCLUDED:
michael@0:             continue
michael@0: 
michael@0:         # Get the full path and relative location of the file.
michael@0:         filename = os.path.join(root, basename)
michael@0:         fullpath = os.path.join(location, filename)
michael@0: 
michael@0:         # Skip empty files.
michael@0:         statbuf = os.stat(fullpath)
michael@0:         if statbuf.st_size == 0:
michael@0:             continue
michael@0: 
michael@0:         testcase = TestCase(os.path.join(reldir, filename))
michael@0:         _apply_external_manifests(filename, testcase, externalManifestEntries, xul_tester)
michael@0:         _parse_test_header(fullpath, testcase, xul_tester)
michael@0:         tests.append(testcase)
michael@0:     return tests