michael@0: #!/usr/bin/python
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: import os
michael@0: from optparse import OptionParser
michael@0: from subprocess import Popen, PIPE
michael@0: import xml.dom.minidom
michael@0: import html5lib
michael@0: import shutil
michael@0: import sys
michael@0: import re
michael@0: 
michael@0: # FIXME:
michael@0: #   * Import more tests rather than just the very limited set currently
michael@0: #     chosen.
michael@0: #   * Read in a (checked-in) input file with a list of test assertions
michael@0: #     expected to fail.
michael@0: #   * Read in a (checked-in) input file with a list of reference choices
michael@0: #     for tests with multiple rel="match" references.  (But still go
michael@0: #     though all those references in case they, in turn, have references.)
michael@0: 
michael@0: # Eventually we should import all the tests that have references.  (At
michael@0: # least for a subset of secs.  And we probably want to organize the
michael@0: # directory structure by spec to avoid constant file moves when files
michael@0: # move in the W3C repository.  And we probably also want to import each
michael@0: # test only once, even if it covers more than one spec.)
michael@0: 
michael@0: # But for now, let's just import a few sets of tests.
michael@0: 
michael@0: gSubtrees = [
michael@0:     os.path.join("approved", "css3-namespace", "src"),
michael@0:     #os.path.join("approved", "css3-multicol", "src"),
michael@0:     os.path.join("contributors", "opera", "submitted", "css3-conditional"),
michael@0:     #os.path.join("contributors", "opera", "submitted", "multicol")
michael@0: ]
michael@0: 
michael@0: gPrefixedProperties = [
michael@0:     "column-count",
michael@0:     "column-fill",
michael@0:     "column-gap",
michael@0:     "column-rule",
michael@0:     "column-rule-color",
michael@0:     "column-rule-style",
michael@0:     "column-rule-width",
michael@0:     "columns",
michael@0:     "column-span",
michael@0:     "column-width"
michael@0: ]
michael@0: 
michael@0: gDefaultPreferences = {
michael@0:     "css3-conditional": "pref(layout.css.supports-rule.enabled,true)"
michael@0: }
michael@0: 
michael@0: gLog = None
michael@0: gFailList = {}
michael@0: gDestPath = None
michael@0: gSrcPath = None
michael@0: support_dirs_mapped = set()
michael@0: filemap = {}
michael@0: speclinkmap = {}
michael@0: propsaddedfor = []
michael@0: tests = []
michael@0: gOptions = None
michael@0: gArgs = None
michael@0: gTestfiles = []
michael@0: gTestFlags = {}
michael@0: 
michael@0: def log_output_of(subprocess):
michael@0:     global gLog
michael@0:     subprocess.wait()
michael@0:     if (subprocess.returncode != 0):
michael@0:         raise StandardError("error while running subprocess")
michael@0:     gLog.write(subprocess.stdout.readline().rstrip())
michael@0: 
michael@0: def write_log_header():
michael@0:     global gLog, gSrcPath
michael@0:     gLog.write("Importing revision: ")
michael@0:     log_output_of(Popen(["hg", "parent", "--template={node}"],
michael@0:                   stdout=PIPE, cwd=gSrcPath))
michael@0:     gLog.write("\nfrom repository: ")
michael@0:     log_output_of(Popen(["hg", "paths", "default"],
michael@0:                   stdout=PIPE, cwd=gSrcPath))
michael@0:     gLog.write("\n")
michael@0: 
michael@0: def remove_existing_dirs():
michael@0:     global gDestPath
michael@0:     # Remove existing directories that we're going to regenerate.  This
michael@0:     # is necessary so that we can give errors in cases where our import
michael@0:     # might copy two files to the same location, which we do by giving
michael@0:     # errors if a copy would overwrite a file.
michael@0:     for dirname in os.listdir(gDestPath):
michael@0:         fulldir = os.path.join(gDestPath, dirname)
michael@0:         if not os.path.isdir(fulldir):
michael@0:             continue
michael@0:         shutil.rmtree(fulldir)
michael@0: 
michael@0: def populate_test_files():
michael@0:     global gSubtrees, gTestfiles
michael@0:     for subtree in gSubtrees:
michael@0:         for dirpath, dirnames, filenames in os.walk(subtree, topdown=True):
michael@0:             if "support" in dirnames:
michael@0:                dirnames.remove("support")
michael@0:             if "reftest" in dirnames:
michael@0:                dirnames.remove("reftest")
michael@0:             for f in filenames:
michael@0:                 if f == "README" or \
michael@0:                    f.find("-ref.") != -1:
michael@0:                     continue
michael@0:                 gTestfiles.append(os.path.join(dirpath, f))
michael@0: 
michael@0:     gTestfiles.sort()
michael@0: 
michael@0: def copy_file(test, srcfile, destname, isSupportFile=False):
michael@0:     global gDestPath, gLog, gSrcPath
michael@0:     if not srcfile.startswith(gSrcPath):
michael@0:         raise StandardError("Filename " + srcfile + " does not start with " + gSrcPath)
michael@0:     logname = srcfile[len(gSrcPath):]
michael@0:     gLog.write("Importing " + logname + " to " + destname + "\n")
michael@0:     destfile = os.path.join(gDestPath, destname)
michael@0:     destdir = os.path.dirname(destfile)
michael@0:     if not os.path.exists(destdir):
michael@0:         os.makedirs(destdir)
michael@0:     if os.path.exists(destfile):
michael@0:         raise StandardError("file " + destfile + " already exists")
michael@0:     copy_and_prefix(test, srcfile, destfile, gPrefixedProperties, isSupportFile)
michael@0: 
michael@0: def copy_support_files(test, dirname, spec):
michael@0:     if dirname in support_dirs_mapped:
michael@0:         return
michael@0:     support_dirs_mapped.add(dirname)
michael@0:     support_dir = os.path.join(dirname, "support")
michael@0:     if not os.path.exists(support_dir):
michael@0:         return
michael@0:     for dirpath, dirnames, filenames in os.walk(support_dir):
michael@0:         for fn in filenames:
michael@0:             if fn == "LOCK":
michael@0:                 continue
michael@0:             full_fn = os.path.join(dirpath, fn)
michael@0:             copy_file(test, full_fn, os.path.join(spec, "support", full_fn[len(support_dir)+1:]), True)
michael@0: 
michael@0: def map_file(fn, spec):
michael@0:     if fn in filemap:
michael@0:         return filemap[fn]
michael@0:     destname = os.path.join(spec, os.path.basename(fn))
michael@0:     filemap[fn] = destname
michael@0:     load_flags_for(fn, spec)
michael@0:     copy_file(destname, fn, destname, False)
michael@0:     copy_support_files(destname, os.path.dirname(fn), spec)
michael@0:     return destname
michael@0: 
michael@0: def load_flags_for(fn, spec):
michael@0:     global gTestFlags
michael@0:     document = get_document_for(fn, spec)
michael@0:     destname = os.path.join(spec, os.path.basename(fn))
michael@0:     gTestFlags[destname] = []
michael@0: 
michael@0:     for meta in document.getElementsByTagName("meta"):
michael@0:         name = meta.getAttribute("name")
michael@0:         if name == "flags":
michael@0:             gTestFlags[destname] = meta.getAttribute("content").split()
michael@0: 
michael@0: def get_document_for(fn, spec):
michael@0:     document = None # an xml.dom.minidom document
michael@0:     if fn.endswith(".htm") or fn.endswith(".html"):
michael@0:         # An HTML file
michael@0:         f = open(fn, "r")
michael@0:         parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("dom"))
michael@0:         document = parser.parse(f)
michael@0:         f.close()
michael@0:     else:
michael@0:         # An XML file
michael@0:         document = xml.dom.minidom.parse(fn)
michael@0:     return document
michael@0: 
michael@0: def add_test_items(fn, spec):
michael@0:     document = get_document_for(fn, spec)
michael@0:     refs = []
michael@0:     notrefs = []
michael@0:     for link in document.getElementsByTagName("link"):
michael@0:         rel = link.getAttribute("rel")
michael@0:         if rel == "help" and spec == None:
michael@0:             specurl = link.getAttribute("href")
michael@0:             startidx = specurl.find("/TR/")
michael@0:             if startidx != -1:
michael@0:                 startidx = startidx + 4
michael@0:                 endidx = specurl.find("/", startidx)
michael@0:                 if endidx != -1:
michael@0:                     spec = str(specurl[startidx:endidx])
michael@0:         if rel == "match":
michael@0:             arr = refs
michael@0:         elif rel == "mismatch":
michael@0:             arr = notrefs
michael@0:         else:
michael@0:             continue
michael@0:         arr.append(os.path.join(os.path.dirname(fn), str(link.getAttribute("href"))))
michael@0:     if len(refs) > 1:
michael@0:         raise StandardError("Need to add code to specify which reference we want to match.")
michael@0:     if spec is None:
michael@0:         raise StandardError("Could not associate test with specification")
michael@0:     for ref in refs:
michael@0:         tests.append(["==", map_file(fn, spec), map_file(ref, spec)])
michael@0:     for notref in notrefs:
michael@0:         tests.append(["!=", map_file(fn, spec), map_file(notref, spec)])
michael@0:     # Add chained references too
michael@0:     for ref in refs:
michael@0:         add_test_items(ref, spec=spec)
michael@0:     for notref in notrefs:
michael@0:         add_test_items(notref, spec=spec)
michael@0: 
michael@0: def copy_and_prefix(test, aSourceFileName, aDestFileName, aProps, isSupportFile=False):
michael@0:     global gTestFlags
michael@0:     newFile = open(aDestFileName, 'w')
michael@0:     unPrefixedFile = open(aSourceFileName)
michael@0:     testName = aDestFileName[len(gDestPath)+1:]
michael@0:     ahemFontAdded = False
michael@0:     for line in unPrefixedFile:
michael@0:         replacementLine = line
michael@0:         searchRegex = "\s*<style\s*"
michael@0: 
michael@0:         if not isSupportFile and not ahemFontAdded and 'ahem' in gTestFlags[test] and re.search(searchRegex, line):
michael@0:             # First put our ahem font declation before the first <style>
michael@0:             # element
michael@0:             ahemFontDecl = "<style type=\"text/css\"><![CDATA[\n@font-face "\
michael@0:                            "{\n  font-family: Ahem;\n  src: url("\
michael@0:                            "\"../../../fonts/Ahem.ttf\");\n}\n]]></style>\n"
michael@0:             newFile.write(ahemFontDecl)
michael@0:             ahemFontAdded = True
michael@0: 
michael@0:         for rule in aProps:
michael@0:             replacementLine = replacementLine.replace(rule, "-moz-" + rule)
michael@0:         newFile.write(replacementLine)
michael@0: 
michael@0:     newFile.close()
michael@0:     unPrefixedFile.close()
michael@0: 
michael@0: def read_options():
michael@0:     global gArgs, gOptions
michael@0:     op = OptionParser()
michael@0:     op.usage = \
michael@0:     '''%prog <clone of hg repository>
michael@0:             Import reftests from a W3C hg repository clone. The location of
michael@0:             the local clone of the hg repository must be given on the command
michael@0:             line.'''
michael@0:     (gOptions, gArgs) = op.parse_args()
michael@0:     if len(gArgs) != 1:
michael@0:         op.error("Too few arguments specified.")
michael@0: 
michael@0: def setup_paths():
michael@0:     global gSubtrees, gDestPath, gSrcPath
michael@0:     # FIXME: generate gSrcPath with consistent trailing / regardless of input.
michael@0:     # (We currently expect the argument to have a trailing slash.)
michael@0:     gSrcPath = gArgs[0]
michael@0:     if not os.path.isdir(gSrcPath) or \
michael@0:     not os.path.isdir(os.path.join(gSrcPath, ".hg")):
michael@0:         raise StandardError("source path does not appear to be a mercurial clone")
michael@0: 
michael@0:     gDestPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "received")
michael@0:     newSubtrees = []
michael@0:     for relPath in gSubtrees:
michael@0:         newSubtrees[len(gSubtrees):] = [os.path.join(gSrcPath, relPath)]
michael@0:     gSubtrees = newSubtrees
michael@0: 
michael@0: def setup_log():
michael@0:     global gLog
michael@0:     # Since we're going to commit the tests, we should also commit
michael@0:     # information about where they came from.
michael@0:     gLog = open(os.path.join(gDestPath, "import.log"), "w")
michael@0: 
michael@0: def read_fail_list():
michael@0:     global gFailList
michael@0:     dirname = os.path.realpath(__file__).split(os.path.sep)
michael@0:     dirname = os.path.sep.join(dirname[:len(dirname)-1])
michael@0:     failListFile = open(os.path.join(dirname, "failures.list"), "r")
michael@0:     gFailList = [x for x in [x.lstrip().rstrip() for x in failListFile] if bool(x)
michael@0:                  and not x.startswith("#")]
michael@0:     failListFile.close()
michael@0: 
michael@0: def main():
michael@0:     global gDestPath, gLog, gTestfiles, gTestFlags, gFailList
michael@0:     read_options()
michael@0:     setup_paths()
michael@0:     read_fail_list()
michael@0:     setup_log()
michael@0:     write_log_header()
michael@0:     remove_existing_dirs()
michael@0:     populate_test_files()
michael@0: 
michael@0:     for t in gTestfiles:
michael@0:         add_test_items(t, spec=None)
michael@0: 
michael@0:     listfile = open(os.path.join(gDestPath, "reftest.list"), "w")
michael@0:     listfile.write("# THIS FILE IS AUTOGENERATED BY {0}\n# DO NOT EDIT!\n".format(os.path.basename(__file__)))
michael@0:     lastDefaultPreferences = None
michael@0:     for test in tests:
michael@0:         defaultPreferences = gDefaultPreferences.get(test[1].split("/")[0], None)
michael@0:         if defaultPreferences != lastDefaultPreferences:
michael@0:             if defaultPreferences is None:
michael@0:                 listfile.write("\ndefault-preferences\n\n")
michael@0:             else:
michael@0:                 listfile.write("\ndefault-preferences {0}\n\n".format(defaultPreferences))
michael@0:             lastDefaultPreferences = defaultPreferences
michael@0:         key = 0
michael@0:         while not test[key] in gTestFlags.keys() and key < len(test):
michael@0:             key = key + 1
michael@0:         testKey = test[key]
michael@0:         if 'ahem' in gTestFlags[testKey]:
michael@0:             test = ["HTTP(../../..)"] + test
michael@0:         if testKey in gFailList:
michael@0:             test = ["fails"] + test
michael@0:         listfile.write(" ".join(test) + "\n")
michael@0:     listfile.close()
michael@0: 
michael@0:     gLog.close()
michael@0: 
michael@0: if __name__ == '__main__':
michael@0:     main()