1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/layout/reftests/w3c-css/import-tests.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,317 @@
1.4 +#!/usr/bin/python
1.5 +# This Source Code Form is subject to the terms of the Mozilla Public
1.6 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.8 +import os
1.9 +from optparse import OptionParser
1.10 +from subprocess import Popen, PIPE
1.11 +import xml.dom.minidom
1.12 +import html5lib
1.13 +import shutil
1.14 +import sys
1.15 +import re
1.16 +
1.17 +# FIXME:
1.18 +# * Import more tests rather than just the very limited set currently
1.19 +# chosen.
1.20 +# * Read in a (checked-in) input file with a list of test assertions
1.21 +# expected to fail.
1.22 +# * Read in a (checked-in) input file with a list of reference choices
1.23 +# for tests with multiple rel="match" references. (But still go
1.24 +# though all those references in case they, in turn, have references.)
1.25 +
1.26 +# Eventually we should import all the tests that have references. (At
1.27 +# least for a subset of secs. And we probably want to organize the
1.28 +# directory structure by spec to avoid constant file moves when files
1.29 +# move in the W3C repository. And we probably also want to import each
1.30 +# test only once, even if it covers more than one spec.)
1.31 +
1.32 +# But for now, let's just import a few sets of tests.
1.33 +
1.34 +gSubtrees = [
1.35 + os.path.join("approved", "css3-namespace", "src"),
1.36 + #os.path.join("approved", "css3-multicol", "src"),
1.37 + os.path.join("contributors", "opera", "submitted", "css3-conditional"),
1.38 + #os.path.join("contributors", "opera", "submitted", "multicol")
1.39 +]
1.40 +
1.41 +gPrefixedProperties = [
1.42 + "column-count",
1.43 + "column-fill",
1.44 + "column-gap",
1.45 + "column-rule",
1.46 + "column-rule-color",
1.47 + "column-rule-style",
1.48 + "column-rule-width",
1.49 + "columns",
1.50 + "column-span",
1.51 + "column-width"
1.52 +]
1.53 +
1.54 +gDefaultPreferences = {
1.55 + "css3-conditional": "pref(layout.css.supports-rule.enabled,true)"
1.56 +}
1.57 +
1.58 +gLog = None
1.59 +gFailList = {}
1.60 +gDestPath = None
1.61 +gSrcPath = None
1.62 +support_dirs_mapped = set()
1.63 +filemap = {}
1.64 +speclinkmap = {}
1.65 +propsaddedfor = []
1.66 +tests = []
1.67 +gOptions = None
1.68 +gArgs = None
1.69 +gTestfiles = []
1.70 +gTestFlags = {}
1.71 +
1.72 +def log_output_of(subprocess):
1.73 + global gLog
1.74 + subprocess.wait()
1.75 + if (subprocess.returncode != 0):
1.76 + raise StandardError("error while running subprocess")
1.77 + gLog.write(subprocess.stdout.readline().rstrip())
1.78 +
1.79 +def write_log_header():
1.80 + global gLog, gSrcPath
1.81 + gLog.write("Importing revision: ")
1.82 + log_output_of(Popen(["hg", "parent", "--template={node}"],
1.83 + stdout=PIPE, cwd=gSrcPath))
1.84 + gLog.write("\nfrom repository: ")
1.85 + log_output_of(Popen(["hg", "paths", "default"],
1.86 + stdout=PIPE, cwd=gSrcPath))
1.87 + gLog.write("\n")
1.88 +
1.89 +def remove_existing_dirs():
1.90 + global gDestPath
1.91 + # Remove existing directories that we're going to regenerate. This
1.92 + # is necessary so that we can give errors in cases where our import
1.93 + # might copy two files to the same location, which we do by giving
1.94 + # errors if a copy would overwrite a file.
1.95 + for dirname in os.listdir(gDestPath):
1.96 + fulldir = os.path.join(gDestPath, dirname)
1.97 + if not os.path.isdir(fulldir):
1.98 + continue
1.99 + shutil.rmtree(fulldir)
1.100 +
1.101 +def populate_test_files():
1.102 + global gSubtrees, gTestfiles
1.103 + for subtree in gSubtrees:
1.104 + for dirpath, dirnames, filenames in os.walk(subtree, topdown=True):
1.105 + if "support" in dirnames:
1.106 + dirnames.remove("support")
1.107 + if "reftest" in dirnames:
1.108 + dirnames.remove("reftest")
1.109 + for f in filenames:
1.110 + if f == "README" or \
1.111 + f.find("-ref.") != -1:
1.112 + continue
1.113 + gTestfiles.append(os.path.join(dirpath, f))
1.114 +
1.115 + gTestfiles.sort()
1.116 +
1.117 +def copy_file(test, srcfile, destname, isSupportFile=False):
1.118 + global gDestPath, gLog, gSrcPath
1.119 + if not srcfile.startswith(gSrcPath):
1.120 + raise StandardError("Filename " + srcfile + " does not start with " + gSrcPath)
1.121 + logname = srcfile[len(gSrcPath):]
1.122 + gLog.write("Importing " + logname + " to " + destname + "\n")
1.123 + destfile = os.path.join(gDestPath, destname)
1.124 + destdir = os.path.dirname(destfile)
1.125 + if not os.path.exists(destdir):
1.126 + os.makedirs(destdir)
1.127 + if os.path.exists(destfile):
1.128 + raise StandardError("file " + destfile + " already exists")
1.129 + copy_and_prefix(test, srcfile, destfile, gPrefixedProperties, isSupportFile)
1.130 +
1.131 +def copy_support_files(test, dirname, spec):
1.132 + if dirname in support_dirs_mapped:
1.133 + return
1.134 + support_dirs_mapped.add(dirname)
1.135 + support_dir = os.path.join(dirname, "support")
1.136 + if not os.path.exists(support_dir):
1.137 + return
1.138 + for dirpath, dirnames, filenames in os.walk(support_dir):
1.139 + for fn in filenames:
1.140 + if fn == "LOCK":
1.141 + continue
1.142 + full_fn = os.path.join(dirpath, fn)
1.143 + copy_file(test, full_fn, os.path.join(spec, "support", full_fn[len(support_dir)+1:]), True)
1.144 +
1.145 +def map_file(fn, spec):
1.146 + if fn in filemap:
1.147 + return filemap[fn]
1.148 + destname = os.path.join(spec, os.path.basename(fn))
1.149 + filemap[fn] = destname
1.150 + load_flags_for(fn, spec)
1.151 + copy_file(destname, fn, destname, False)
1.152 + copy_support_files(destname, os.path.dirname(fn), spec)
1.153 + return destname
1.154 +
1.155 +def load_flags_for(fn, spec):
1.156 + global gTestFlags
1.157 + document = get_document_for(fn, spec)
1.158 + destname = os.path.join(spec, os.path.basename(fn))
1.159 + gTestFlags[destname] = []
1.160 +
1.161 + for meta in document.getElementsByTagName("meta"):
1.162 + name = meta.getAttribute("name")
1.163 + if name == "flags":
1.164 + gTestFlags[destname] = meta.getAttribute("content").split()
1.165 +
1.166 +def get_document_for(fn, spec):
1.167 + document = None # an xml.dom.minidom document
1.168 + if fn.endswith(".htm") or fn.endswith(".html"):
1.169 + # An HTML file
1.170 + f = open(fn, "r")
1.171 + parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("dom"))
1.172 + document = parser.parse(f)
1.173 + f.close()
1.174 + else:
1.175 + # An XML file
1.176 + document = xml.dom.minidom.parse(fn)
1.177 + return document
1.178 +
1.179 +def add_test_items(fn, spec):
1.180 + document = get_document_for(fn, spec)
1.181 + refs = []
1.182 + notrefs = []
1.183 + for link in document.getElementsByTagName("link"):
1.184 + rel = link.getAttribute("rel")
1.185 + if rel == "help" and spec == None:
1.186 + specurl = link.getAttribute("href")
1.187 + startidx = specurl.find("/TR/")
1.188 + if startidx != -1:
1.189 + startidx = startidx + 4
1.190 + endidx = specurl.find("/", startidx)
1.191 + if endidx != -1:
1.192 + spec = str(specurl[startidx:endidx])
1.193 + if rel == "match":
1.194 + arr = refs
1.195 + elif rel == "mismatch":
1.196 + arr = notrefs
1.197 + else:
1.198 + continue
1.199 + arr.append(os.path.join(os.path.dirname(fn), str(link.getAttribute("href"))))
1.200 + if len(refs) > 1:
1.201 + raise StandardError("Need to add code to specify which reference we want to match.")
1.202 + if spec is None:
1.203 + raise StandardError("Could not associate test with specification")
1.204 + for ref in refs:
1.205 + tests.append(["==", map_file(fn, spec), map_file(ref, spec)])
1.206 + for notref in notrefs:
1.207 + tests.append(["!=", map_file(fn, spec), map_file(notref, spec)])
1.208 + # Add chained references too
1.209 + for ref in refs:
1.210 + add_test_items(ref, spec=spec)
1.211 + for notref in notrefs:
1.212 + add_test_items(notref, spec=spec)
1.213 +
1.214 +def copy_and_prefix(test, aSourceFileName, aDestFileName, aProps, isSupportFile=False):
1.215 + global gTestFlags
1.216 + newFile = open(aDestFileName, 'w')
1.217 + unPrefixedFile = open(aSourceFileName)
1.218 + testName = aDestFileName[len(gDestPath)+1:]
1.219 + ahemFontAdded = False
1.220 + for line in unPrefixedFile:
1.221 + replacementLine = line
1.222 + searchRegex = "\s*<style\s*"
1.223 +
1.224 + if not isSupportFile and not ahemFontAdded and 'ahem' in gTestFlags[test] and re.search(searchRegex, line):
1.225 + # First put our ahem font declation before the first <style>
1.226 + # element
1.227 + ahemFontDecl = "<style type=\"text/css\"><![CDATA[\n@font-face "\
1.228 + "{\n font-family: Ahem;\n src: url("\
1.229 + "\"../../../fonts/Ahem.ttf\");\n}\n]]></style>\n"
1.230 + newFile.write(ahemFontDecl)
1.231 + ahemFontAdded = True
1.232 +
1.233 + for rule in aProps:
1.234 + replacementLine = replacementLine.replace(rule, "-moz-" + rule)
1.235 + newFile.write(replacementLine)
1.236 +
1.237 + newFile.close()
1.238 + unPrefixedFile.close()
1.239 +
1.240 +def read_options():
1.241 + global gArgs, gOptions
1.242 + op = OptionParser()
1.243 + op.usage = \
1.244 + '''%prog <clone of hg repository>
1.245 + Import reftests from a W3C hg repository clone. The location of
1.246 + the local clone of the hg repository must be given on the command
1.247 + line.'''
1.248 + (gOptions, gArgs) = op.parse_args()
1.249 + if len(gArgs) != 1:
1.250 + op.error("Too few arguments specified.")
1.251 +
1.252 +def setup_paths():
1.253 + global gSubtrees, gDestPath, gSrcPath
1.254 + # FIXME: generate gSrcPath with consistent trailing / regardless of input.
1.255 + # (We currently expect the argument to have a trailing slash.)
1.256 + gSrcPath = gArgs[0]
1.257 + if not os.path.isdir(gSrcPath) or \
1.258 + not os.path.isdir(os.path.join(gSrcPath, ".hg")):
1.259 + raise StandardError("source path does not appear to be a mercurial clone")
1.260 +
1.261 + gDestPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "received")
1.262 + newSubtrees = []
1.263 + for relPath in gSubtrees:
1.264 + newSubtrees[len(gSubtrees):] = [os.path.join(gSrcPath, relPath)]
1.265 + gSubtrees = newSubtrees
1.266 +
1.267 +def setup_log():
1.268 + global gLog
1.269 + # Since we're going to commit the tests, we should also commit
1.270 + # information about where they came from.
1.271 + gLog = open(os.path.join(gDestPath, "import.log"), "w")
1.272 +
1.273 +def read_fail_list():
1.274 + global gFailList
1.275 + dirname = os.path.realpath(__file__).split(os.path.sep)
1.276 + dirname = os.path.sep.join(dirname[:len(dirname)-1])
1.277 + failListFile = open(os.path.join(dirname, "failures.list"), "r")
1.278 + gFailList = [x for x in [x.lstrip().rstrip() for x in failListFile] if bool(x)
1.279 + and not x.startswith("#")]
1.280 + failListFile.close()
1.281 +
1.282 +def main():
1.283 + global gDestPath, gLog, gTestfiles, gTestFlags, gFailList
1.284 + read_options()
1.285 + setup_paths()
1.286 + read_fail_list()
1.287 + setup_log()
1.288 + write_log_header()
1.289 + remove_existing_dirs()
1.290 + populate_test_files()
1.291 +
1.292 + for t in gTestfiles:
1.293 + add_test_items(t, spec=None)
1.294 +
1.295 + listfile = open(os.path.join(gDestPath, "reftest.list"), "w")
1.296 + listfile.write("# THIS FILE IS AUTOGENERATED BY {0}\n# DO NOT EDIT!\n".format(os.path.basename(__file__)))
1.297 + lastDefaultPreferences = None
1.298 + for test in tests:
1.299 + defaultPreferences = gDefaultPreferences.get(test[1].split("/")[0], None)
1.300 + if defaultPreferences != lastDefaultPreferences:
1.301 + if defaultPreferences is None:
1.302 + listfile.write("\ndefault-preferences\n\n")
1.303 + else:
1.304 + listfile.write("\ndefault-preferences {0}\n\n".format(defaultPreferences))
1.305 + lastDefaultPreferences = defaultPreferences
1.306 + key = 0
1.307 + while not test[key] in gTestFlags.keys() and key < len(test):
1.308 + key = key + 1
1.309 + testKey = test[key]
1.310 + if 'ahem' in gTestFlags[testKey]:
1.311 + test = ["HTTP(../../..)"] + test
1.312 + if testKey in gFailList:
1.313 + test = ["fails"] + test
1.314 + listfile.write(" ".join(test) + "\n")
1.315 + listfile.close()
1.316 +
1.317 + gLog.close()
1.318 +
1.319 +if __name__ == '__main__':
1.320 + main()
