The Tor Browser: layout/reftests/w3c-css/import-tests.py@97036ab72558

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

     1 #!/usr/bin/python

     2 # This Source Code Form is subject to the terms of the Mozilla Public

     3 # License, v. 2.0. If a copy of the MPL was not distributed with this

     4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

     5 import os

     6 from optparse import OptionParser

     7 from subprocess import Popen, PIPE

     8 import xml.dom.minidom

     9 import html5lib

    10 import shutil

    11 import sys

    12 import re

    14 # FIXME:

    15 #   * Import more tests rather than just the very limited set currently

    16 #     chosen.

    17 #   * Read in a (checked-in) input file with a list of test assertions

    18 #     expected to fail.

    19 #   * Read in a (checked-in) input file with a list of reference choices

    20 #     for tests with multiple rel="match" references.  (But still go

    21 #     though all those references in case they, in turn, have references.)

    23 # Eventually we should import all the tests that have references.  (At

    24 # least for a subset of secs.  And we probably want to organize the

    25 # directory structure by spec to avoid constant file moves when files

    26 # move in the W3C repository.  And we probably also want to import each

    27 # test only once, even if it covers more than one spec.)

    29 # But for now, let's just import a few sets of tests.

    31 gSubtrees = [

    32     os.path.join("approved", "css3-namespace", "src"),

    33     #os.path.join("approved", "css3-multicol", "src"),

    34     os.path.join("contributors", "opera", "submitted", "css3-conditional"),

    35     #os.path.join("contributors", "opera", "submitted", "multicol")

    36 ]

    38 gPrefixedProperties = [

    39     "column-count",

    40     "column-fill",

    41     "column-gap",

    42     "column-rule",

    43     "column-rule-color",

    44     "column-rule-style",

    45     "column-rule-width",

    46     "columns",

    47     "column-span",

    48     "column-width"

    49 ]

    51 gDefaultPreferences = {

    52     "css3-conditional": "pref(layout.css.supports-rule.enabled,true)"

    53 }

    55 gLog = None

    56 gFailList = {}

    57 gDestPath = None

    58 gSrcPath = None

    59 support_dirs_mapped = set()

    60 filemap = {}

    61 speclinkmap = {}

    62 propsaddedfor = []

    63 tests = []

    64 gOptions = None

    65 gArgs = None

    66 gTestfiles = []

    67 gTestFlags = {}

    69 def log_output_of(subprocess):

    70     global gLog

    71     subprocess.wait()

    72     if (subprocess.returncode != 0):

    73         raise StandardError("error while running subprocess")

    74     gLog.write(subprocess.stdout.readline().rstrip())

    76 def write_log_header():

    77     global gLog, gSrcPath

    78     gLog.write("Importing revision: ")

    79     log_output_of(Popen(["hg", "parent", "--template={node}"],

    80                   stdout=PIPE, cwd=gSrcPath))

    81     gLog.write("\nfrom repository: ")

    82     log_output_of(Popen(["hg", "paths", "default"],

    83                   stdout=PIPE, cwd=gSrcPath))

    84     gLog.write("\n")

    86 def remove_existing_dirs():

    87     global gDestPath

    88     # Remove existing directories that we're going to regenerate.  This

    89     # is necessary so that we can give errors in cases where our import

    90     # might copy two files to the same location, which we do by giving

    91     # errors if a copy would overwrite a file.

    92     for dirname in os.listdir(gDestPath):

    93         fulldir = os.path.join(gDestPath, dirname)

    94         if not os.path.isdir(fulldir):

    95             continue

    96         shutil.rmtree(fulldir)

    98 def populate_test_files():

    99     global gSubtrees, gTestfiles

   100     for subtree in gSubtrees:

   101         for dirpath, dirnames, filenames in os.walk(subtree, topdown=True):

   102             if "support" in dirnames:

   103                dirnames.remove("support")

   104             if "reftest" in dirnames:

   105                dirnames.remove("reftest")

   106             for f in filenames:

   107                 if f == "README" or \

   108                    f.find("-ref.") != -1:

   109                     continue

   110                 gTestfiles.append(os.path.join(dirpath, f))

   112     gTestfiles.sort()

   114 def copy_file(test, srcfile, destname, isSupportFile=False):

   115     global gDestPath, gLog, gSrcPath

   116     if not srcfile.startswith(gSrcPath):

   117         raise StandardError("Filename " + srcfile + " does not start with " + gSrcPath)

   118     logname = srcfile[len(gSrcPath):]

   119     gLog.write("Importing " + logname + " to " + destname + "\n")

   120     destfile = os.path.join(gDestPath, destname)

   121     destdir = os.path.dirname(destfile)

   122     if not os.path.exists(destdir):

   123         os.makedirs(destdir)

   124     if os.path.exists(destfile):

   125         raise StandardError("file " + destfile + " already exists")

   126     copy_and_prefix(test, srcfile, destfile, gPrefixedProperties, isSupportFile)

   128 def copy_support_files(test, dirname, spec):

   129     if dirname in support_dirs_mapped:

   130         return

   131     support_dirs_mapped.add(dirname)

   132     support_dir = os.path.join(dirname, "support")

   133     if not os.path.exists(support_dir):

   134         return

   135     for dirpath, dirnames, filenames in os.walk(support_dir):

   136         for fn in filenames:

   137             if fn == "LOCK":

   138                 continue

   139             full_fn = os.path.join(dirpath, fn)

   140             copy_file(test, full_fn, os.path.join(spec, "support", full_fn[len(support_dir)+1:]), True)

   142 def map_file(fn, spec):

   143     if fn in filemap:

   144         return filemap[fn]

   145     destname = os.path.join(spec, os.path.basename(fn))

   146     filemap[fn] = destname

   147     load_flags_for(fn, spec)

   148     copy_file(destname, fn, destname, False)

   149     copy_support_files(destname, os.path.dirname(fn), spec)

   150     return destname

   152 def load_flags_for(fn, spec):

   153     global gTestFlags

   154     document = get_document_for(fn, spec)

   155     destname = os.path.join(spec, os.path.basename(fn))

   156     gTestFlags[destname] = []

   158     for meta in document.getElementsByTagName("meta"):

   159         name = meta.getAttribute("name")

   160         if name == "flags":

   161             gTestFlags[destname] = meta.getAttribute("content").split()

   163 def get_document_for(fn, spec):

   164     document = None # an xml.dom.minidom document

   165     if fn.endswith(".htm") or fn.endswith(".html"):

   166         # An HTML file

   167         f = open(fn, "r")

   168         parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("dom"))

   169         document = parser.parse(f)

   170         f.close()

   171     else:

   172         # An XML file

   173         document = xml.dom.minidom.parse(fn)

   174     return document

   176 def add_test_items(fn, spec):

   177     document = get_document_for(fn, spec)

   178     refs = []

   179     notrefs = []

   180     for link in document.getElementsByTagName("link"):

   181         rel = link.getAttribute("rel")

   182         if rel == "help" and spec == None:

   183             specurl = link.getAttribute("href")

   184             startidx = specurl.find("/TR/")

   185             if startidx != -1:

   186                 startidx = startidx + 4

   187                 endidx = specurl.find("/", startidx)

   188                 if endidx != -1:

   189                     spec = str(specurl[startidx:endidx])

   190         if rel == "match":

   191             arr = refs

   192         elif rel == "mismatch":

   193             arr = notrefs

   194         else:

   195             continue

   196         arr.append(os.path.join(os.path.dirname(fn), str(link.getAttribute("href"))))

   197     if len(refs) > 1:

   198         raise StandardError("Need to add code to specify which reference we want to match.")

   199     if spec is None:

   200         raise StandardError("Could not associate test with specification")

   201     for ref in refs:

   202         tests.append(["==", map_file(fn, spec), map_file(ref, spec)])

   203     for notref in notrefs:

   204         tests.append(["!=", map_file(fn, spec), map_file(notref, spec)])

   205     # Add chained references too

   206     for ref in refs:

   207         add_test_items(ref, spec=spec)

   208     for notref in notrefs:

   209         add_test_items(notref, spec=spec)

   211 def copy_and_prefix(test, aSourceFileName, aDestFileName, aProps, isSupportFile=False):

   212     global gTestFlags

   213     newFile = open(aDestFileName, 'w')

   214     unPrefixedFile = open(aSourceFileName)

   215     testName = aDestFileName[len(gDestPath)+1:]

   216     ahemFontAdded = False

   217     for line in unPrefixedFile:

   218         replacementLine = line

   219         searchRegex = "\s*<style\s*"

   221         if not isSupportFile and not ahemFontAdded and 'ahem' in gTestFlags[test] and re.search(searchRegex, line):

   222             # First put our ahem font declation before the first <style>

   223             # element

   224             ahemFontDecl = "<style type=\"text/css\"><![CDATA[\n@font-face "\

   225                            "{\n  font-family: Ahem;\n  src: url("\

   226                            "\"../../../fonts/Ahem.ttf\");\n}\n]]></style>\n"

   227             newFile.write(ahemFontDecl)

   228             ahemFontAdded = True

   230         for rule in aProps:

   231             replacementLine = replacementLine.replace(rule, "-moz-" + rule)

   232         newFile.write(replacementLine)

   234     newFile.close()

   235     unPrefixedFile.close()

   237 def read_options():

   238     global gArgs, gOptions

   239     op = OptionParser()

   240     op.usage = \

   241     '''%prog <clone of hg repository>

   242             Import reftests from a W3C hg repository clone. The location of

   243             the local clone of the hg repository must be given on the command

   244             line.'''

   245     (gOptions, gArgs) = op.parse_args()

   246     if len(gArgs) != 1:

   247         op.error("Too few arguments specified.")

   249 def setup_paths():

   250     global gSubtrees, gDestPath, gSrcPath

   251     # FIXME: generate gSrcPath with consistent trailing / regardless of input.

   252     # (We currently expect the argument to have a trailing slash.)

   253     gSrcPath = gArgs[0]

   254     if not os.path.isdir(gSrcPath) or \

   255     not os.path.isdir(os.path.join(gSrcPath, ".hg")):

   256         raise StandardError("source path does not appear to be a mercurial clone")

   258     gDestPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "received")

   259     newSubtrees = []

   260     for relPath in gSubtrees:

   261         newSubtrees[len(gSubtrees):] = [os.path.join(gSrcPath, relPath)]

   262     gSubtrees = newSubtrees

   264 def setup_log():

   265     global gLog

   266     # Since we're going to commit the tests, we should also commit

   267     # information about where they came from.

   268     gLog = open(os.path.join(gDestPath, "import.log"), "w")

   270 def read_fail_list():

   271     global gFailList

   272     dirname = os.path.realpath(__file__).split(os.path.sep)

   273     dirname = os.path.sep.join(dirname[:len(dirname)-1])

   274     failListFile = open(os.path.join(dirname, "failures.list"), "r")

   275     gFailList = [x for x in [x.lstrip().rstrip() for x in failListFile] if bool(x)

   276                  and not x.startswith("#")]

   277     failListFile.close()

   279 def main():

   280     global gDestPath, gLog, gTestfiles, gTestFlags, gFailList

   281     read_options()

   282     setup_paths()

   283     read_fail_list()

   284     setup_log()

   285     write_log_header()

   286     remove_existing_dirs()

   287     populate_test_files()

   289     for t in gTestfiles:

   290         add_test_items(t, spec=None)

   292     listfile = open(os.path.join(gDestPath, "reftest.list"), "w")

   293     listfile.write("# THIS FILE IS AUTOGENERATED BY {0}\n# DO NOT EDIT!\n".format(os.path.basename(__file__)))

   294     lastDefaultPreferences = None

   295     for test in tests:

   296         defaultPreferences = gDefaultPreferences.get(test[1].split("/")[0], None)

   297         if defaultPreferences != lastDefaultPreferences:

   298             if defaultPreferences is None:

   299                 listfile.write("\ndefault-preferences\n\n")

   300             else:

   301                 listfile.write("\ndefault-preferences {0}\n\n".format(defaultPreferences))

   302             lastDefaultPreferences = defaultPreferences

   303         key = 0

   304         while not test[key] in gTestFlags.keys() and key < len(test):

   305             key = key + 1

   306         testKey = test[key]

   307         if 'ahem' in gTestFlags[testKey]:

   308             test = ["HTTP(../../..)"] + test

   309         if testKey in gFailList:

   310             test = ["fails"] + test

   311         listfile.write(" ".join(test) + "\n")

   312     listfile.close()

   314     gLog.close()

   316 if __name__ == '__main__':

   317     main()

The Tor Browser / file revision

layout/reftests/w3c-css/import-tests.py@97036ab72558

layout/reftests/w3c-css/import-tests.py