The Tor Browser: testing/mozbase/manifestdestiny/manifestparser/manifestparser.py@6474c204b198 (annotated)

testing/mozbase/manifestdestiny/manifestparser/manifestparser.py@6474c204b198 (annotated)

testing/mozbase/manifestdestiny/manifestparser/manifestparser.py

Wed, 31 Dec 2014 06:09:35 +0100

author: Michael Schloh von Bennewitz <michael@schloh.com>
date: Wed, 31 Dec 2014 06:09:35 +0100
changeset 0: 6474c204b198
permissions: -rwxr-xr-x

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

 #!/usr/bin/env python
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Mozilla universal manifest parser
 """
 __all__ = ['read_ini', # .ini reader
            'ManifestParser', 'TestManifest', 'convert', # manifest handling
            'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
 import fnmatch
 import os
 import re
 import shutil
 import sys
 from optparse import OptionParser
 from StringIO import StringIO
 relpath = os.path.relpath
 string = (basestring,)
 # expr.py
 # from:
 # http://k0s.org/mozilla/hg/expressionparser
 # http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
 # Implements a top-down parser/evaluator for simple boolean expressions.
 # ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
 #
 # Rough grammar:
 # expr := literal
 #       | '(' expr ')'
 #       | expr '&&' expr
 #       | expr '||' expr
 #       | expr '==' expr
 #       | expr '!=' expr
 # literal := BOOL
 #          | INT
 #          | STRING
 #          | IDENT
 # BOOL   := true|false
 # INT    := [0-9]+
 # STRING := "[^"]*"
 # IDENT  := [A-Za-z_]\w*
 # Identifiers take their values from a mapping dictionary passed as the second
 # argument.
 # Glossary (see above URL for details):
 # - nud: null denotation
 # - led: left detonation
 # - lbp: left binding power
 # - rbp: right binding power
 class ident_token(object):
     def __init__(self, value):
         self.value = value
     def nud(self, parser):
         # identifiers take their value from the value mappings passed
         # to the parser
         return parser.value(self.value)
 class literal_token(object):
     def __init__(self, value):
         self.value = value
     def nud(self, parser):
         return self.value
 class eq_op_token(object):
     "=="
     def led(self, parser, left):
         return left == parser.expression(self.lbp)
 class neq_op_token(object):
     "!="
     def led(self, parser, left):
         return left != parser.expression(self.lbp)
 class not_op_token(object):
     "!"
     def nud(self, parser):
         return not parser.expression(100)
 class and_op_token(object):
     "&&"
     def led(self, parser, left):
         right = parser.expression(self.lbp)
         return left and right
 class or_op_token(object):
     "||"
     def led(self, parser, left):
         right = parser.expression(self.lbp)
         return left or right
 class lparen_token(object):
     "("
     def nud(self, parser):
         expr = parser.expression()
         parser.advance(rparen_token)
         return expr
 class rparen_token(object):
     ")"
 class end_token(object):
     """always ends parsing"""
 ### derived literal tokens
 class bool_token(literal_token):
     def __init__(self, value):
         value = {'true':True, 'false':False}[value]
         literal_token.__init__(self, value)
 class int_token(literal_token):
     def __init__(self, value):
         literal_token.__init__(self, int(value))
 class string_token(literal_token):
     def __init__(self, value):
         literal_token.__init__(self, value[1:-1])
 precedence = [(end_token, rparen_token),
               (or_op_token,),
               (and_op_token,),
               (eq_op_token, neq_op_token),
               (lparen_token,),
               ]
 for index, rank in enumerate(precedence):
     for token in rank:
         token.lbp = index # lbp = lowest left binding power
 class ParseError(Exception):
     """error parsing conditional expression"""
 class ExpressionParser(object):
     """
     A parser for a simple expression language.
     The expression language can be described as follows::
         EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION
         OP ::= '==' | '!=' | '&&' | '||'
         LITERAL ::= BOOL | INT | IDENT | STRING
         BOOL ::= 'true' | 'false'
         INT ::= [0-9]+
         IDENT ::= [a-zA-Z_]\w*
         STRING ::= '"' [^\"] '"' | ''' [^\'] '''
     At its core, expressions consist of booleans, integers, identifiers and.
     strings. Booleans are one of *true* or *false*. Integers are a series
     of digits. Identifiers are a series of English letters and underscores.
     Strings are a pair of matching quote characters (single or double) with
     zero or more characters inside.
     Expressions can be combined with operators: the equals (==) and not
     equals (!=) operators compare two expressions and produce a boolean. The
     and (&&) and or (||) operators take two expressions and produce the logical
     AND or OR value of them, respectively. An expression can also be prefixed
     with the not (!) operator, which produces its logical negation.
     Finally, any expression may be contained within parentheses for grouping.
     Identifiers take their values from the mapping provided.
     """
     def __init__(self, text, valuemapping, strict=False):
         """
         Initialize the parser
         :param text: The expression to parse as a string.
         :param valuemapping: A dict mapping identifier names to values.
         :param strict: If true, referencing an identifier that was not
                        provided in :valuemapping: will raise an error.
         """
         self.text = text
         self.valuemapping = valuemapping
         self.strict = strict
     def _tokenize(self):
         """
         Lex the input text into tokens and yield them in sequence.
         """
         # scanner callbacks
         def bool_(scanner, t): return bool_token(t)
         def identifier(scanner, t): return ident_token(t)
         def integer(scanner, t): return int_token(t)
         def eq(scanner, t): return eq_op_token()
         def neq(scanner, t): return neq_op_token()
         def or_(scanner, t): return or_op_token()
         def and_(scanner, t): return and_op_token()
         def lparen(scanner, t): return lparen_token()
         def rparen(scanner, t): return rparen_token()
         def string_(scanner, t): return string_token(t)
         def not_(scanner, t): return not_op_token()
         scanner = re.Scanner([
             # Note: keep these in sync with the class docstring above.
             (r"true|false", bool_),
             (r"[a-zA-Z_]\w*", identifier),
             (r"[0-9]+", integer),
             (r'("[^"]*")|(\'[^\']*\')', string_),
             (r"==", eq),
             (r"!=", neq),
             (r"\|\|", or_),
             (r"!", not_),
             (r"&&", and_),
             (r"\(", lparen),
             (r"\)", rparen),
             (r"\s+", None), # skip whitespace
             ])
         tokens, remainder = scanner.scan(self.text)
         for t in tokens:
             yield t
         yield end_token()
     def value(self, ident):
         """
         Look up the value of |ident| in the value mapping passed in the
         constructor.
         """
         if self.strict:
             return self.valuemapping[ident]
         else:
             return self.valuemapping.get(ident, None)
     def advance(self, expected):
         """
         Assert that the next token is an instance of |expected|, and advance
         to the next token.
         """
         if not isinstance(self.token, expected):
             raise Exception, "Unexpected token!"
         self.token = self.iter.next()
     def expression(self, rbp=0):
         """
         Parse and return the value of an expression until a token with
         right binding power greater than rbp is encountered.
         """
         t = self.token
         self.token = self.iter.next()
         left = t.nud(self)
         while rbp < self.token.lbp:
             t = self.token
             self.token = self.iter.next()
             left = t.led(self, left)
         return left
     def parse(self):
         """
         Parse and return the value of the expression in the text
         passed to the constructor. Raises a ParseError if the expression
         could not be parsed.
         """
         try:
             self.iter = self._tokenize()
             self.token = self.iter.next()
             return self.expression()
         except:
             raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
     __call__ = parse
 def parse(text, **values):
     """
     Parse and evaluate a boolean expression.
     :param text: The expression to parse, as a string.
     :param values: A dict containing a name to value mapping for identifiers
                    referenced in *text*.
     :rtype: the final value of the expression.
     :raises: :py:exc::ParseError: will be raised if parsing fails.
     """
     return ExpressionParser(text, values).parse()
 ### path normalization
 def normalize_path(path):
     """normalize a relative path"""
     if sys.platform.startswith('win'):
         return path.replace('/', os.path.sep)
     return path
 def denormalize_path(path):
     """denormalize a relative path"""
     if sys.platform.startswith('win'):
         return path.replace(os.path.sep, '/')
     return path
 ### .ini reader
 def read_ini(fp, variables=None, default='DEFAULT',
              comments=';#', separators=('=', ':'),
              strict=True):
     """
     read an .ini file and return a list of [(section, values)]
     - fp : file pointer or path to read
     - variables : default set of variables
     - default : name of the section for the default section
     - comments : characters that if they start a line denote a comment
     - separators : strings that denote key, value separation in order
     - strict : whether to be strict about parsing
     """
     # variables
     variables = variables or {}
     sections = []
     key = value = None
     section_names = set()
     if isinstance(fp, basestring):
         fp = file(fp)
     # read the lines
     for (linenum, line) in enumerate(fp.readlines(), start=1):
         stripped = line.strip()
         # ignore blank lines
         if not stripped:
             # reset key and value to avoid continuation lines
             key = value = None
             continue
         # ignore comment lines
         if stripped[0] in comments:
             continue
         # check for a new section
         if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
             section = stripped[1:-1].strip()
             key = value = None
             # deal with DEFAULT section
             if section.lower() == default.lower():
                 if strict:
                     assert default not in section_names
                 section_names.add(default)
                 current_section = variables
                 continue
             if strict:
                 # make sure this section doesn't already exist
                 assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names)
             section_names.add(section)
             current_section = {}
             sections.append((section, current_section))
             continue
         # if there aren't any sections yet, something bad happen
         if not section_names:
             raise Exception('No sections found')
         # (key, value) pair
         for separator in separators:
             if separator in stripped:
                 key, value = stripped.split(separator, 1)
                 key = key.strip()
                 value = value.strip()
                 if strict:
                     # make sure this key isn't already in the section or empty
                     assert key
                     if current_section is not variables:
                         assert key not in current_section
                 current_section[key] = value
                 break
         else:
             # continuation line ?
             if line[0].isspace() and key:
                 value = '%s%s%s' % (value, os.linesep, stripped)
                 current_section[key] = value
             else:
                 # something bad happened!
                 if hasattr(fp, 'name'):
                     filename = fp.name
                 else:
                     filename = 'unknown'
                 raise Exception("Error parsing manifest file '%s', line %s" %
                                 (filename, linenum))
     # interpret the variables
     def interpret_variables(global_dict, local_dict):
         variables = global_dict.copy()
         if 'skip-if' in local_dict and 'skip-if' in variables:
             local_dict['skip-if'] = "(%s) || (%s)" % (variables['skip-if'].split('#')[0], local_dict['skip-if'].split('#')[0])
         variables.update(local_dict)
         return variables
     sections = [(i, interpret_variables(variables, j)) for i, j in sections]
     return sections
 ### objects for parsing manifests
 class ManifestParser(object):
     """read .ini manifests"""
     def __init__(self, manifests=(), defaults=None, strict=True):
         self._defaults = defaults or {}
         self.tests = []
         self.manifest_defaults = {}
         self.strict = strict
         self.rootdir = None
         self.relativeRoot = None
         if manifests:
             self.read(*manifests)
     def getRelativeRoot(self, root):
         return root
     ### methods for reading manifests
     def _read(self, root, filename, defaults):
         # get directory of this file if not file-like object
         if isinstance(filename, string):
             filename = os.path.abspath(filename)
             fp = open(filename)
             here = os.path.dirname(filename)
         else:
             fp = filename
             filename = here = None
         defaults['here'] = here
         # Rootdir is needed for relative path calculation. Precompute it for
         # the microoptimization used below.
         if self.rootdir is None:
             rootdir = ""
         else:
             assert os.path.isabs(self.rootdir)
             rootdir = self.rootdir + os.path.sep
         # read the configuration
         sections = read_ini(fp=fp, variables=defaults, strict=self.strict)
         self.manifest_defaults[filename] = defaults
         # get the tests
         for section, data in sections:
             subsuite = ''
             if 'subsuite' in data:
                 subsuite = data['subsuite']
             # a file to include
             # TODO: keep track of included file structure:
             # self.manifests = {'manifest.ini': 'relative/path.ini'}
             if section.startswith('include:'):
                 include_file = section.split('include:', 1)[-1]
                 include_file = normalize_path(include_file)
                 if not os.path.isabs(include_file):
                     include_file = os.path.join(self.getRelativeRoot(here), include_file)
                 if not os.path.exists(include_file):
                     message = "Included file '%s' does not exist" % include_file
                     if self.strict:
                         raise IOError(message)
                     else:
                         sys.stderr.write("%s\n" % message)
                         continue
                 include_defaults = data.copy()
                 self._read(root, include_file, include_defaults)
                 continue
             # otherwise an item
             test = data
             test['name'] = section
             # Will be None if the manifest being read is a file-like object.
             test['manifest'] = filename
             # determine the path
             path = test.get('path', section)
             _relpath = path
             if '://' not in path: # don't futz with URLs
                 path = normalize_path(path)
                 if here and not os.path.isabs(path):
                     path = os.path.normpath(os.path.join(here, path))
                 # Microoptimization, because relpath is quite expensive.
                 # We know that rootdir is an absolute path or empty. If path
                 # starts with rootdir, then path is also absolute and the tail
                 # of the path is the relative path (possibly non-normalized,
                 # when here is unknown).
                 # For this to work rootdir needs to be terminated with a path
                 # separator, so that references to sibling directories with
                 # a common prefix don't get misscomputed (e.g. /root and
                 # /rootbeer/file).
                 # When the rootdir is unknown, the relpath needs to be left
                 # unchanged. We use an empty string as rootdir in that case,
                 # which leaves relpath unchanged after slicing.
                 if path.startswith(rootdir):
                     _relpath = path[len(rootdir):]
                 else:
                     _relpath = relpath(path, rootdir)
             test['subsuite'] = subsuite
             test['path'] = path
             test['relpath'] = _relpath
             # append the item
             self.tests.append(test)
     def read(self, *filenames, **defaults):
         """
         read and add manifests from file paths or file-like objects
         filenames -- file paths or file-like objects to read as manifests
         defaults -- default variables
         """
         # ensure all files exist
         missing = [filename for filename in filenames
                    if isinstance(filename, string) and not os.path.exists(filename) ]
         if missing:
             raise IOError('Missing files: %s' % ', '.join(missing))
         # default variables
         _defaults = defaults.copy() or self._defaults.copy()
         _defaults.setdefault('here', None)
         # process each file
         for filename in filenames:
             # set the per file defaults
             defaults = _defaults.copy()
             here = None
             if isinstance(filename, string):
                 here = os.path.dirname(os.path.abspath(filename))
                 defaults['here'] = here # directory of master .ini file
             if self.rootdir is None:
                 # set the root directory
                 # == the directory of the first manifest given
                 self.rootdir = here
             self._read(here, filename, defaults)
     ### methods for querying manifests
     def query(self, *checks, **kw):
         """
         general query function for tests
         - checks : callable conditions to test if the test fulfills the query
         """
         tests = kw.get('tests', None)
         if tests is None:
             tests = self.tests
         retval = []
         for test in tests:
             for check in checks:
                 if not check(test):
                     break
             else:
                 retval.append(test)
         return retval
     def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs):
         # TODO: pass a dict instead of kwargs since you might hav
         # e.g. 'inverse' as a key in the dict
         # TODO: tags should just be part of kwargs with None values
         # (None == any is kinda weird, but probably still better)
         # fix up tags
         if tags:
             tags = set(tags)
         else:
             tags = set()
         # make some check functions
         if inverse:
             has_tags = lambda test: not tags.intersection(test.keys())
             def dict_query(test):
                 for key, value in kwargs.items():
                     if test.get(key) == value:
                         return False
                 return True
         else:
             has_tags = lambda test: tags.issubset(test.keys())
             def dict_query(test):
                 for key, value in kwargs.items():
                     if test.get(key) != value:
                         return False
                 return True
         # query the tests
         tests = self.query(has_tags, dict_query, tests=tests)
         # if a key is given, return only a list of that key
         # useful for keys like 'name' or 'path'
         if _key:
             return [test[_key] for test in tests]
         # return the tests
         return tests
     def manifests(self, tests=None):
         """
         return manifests in order in which they appear in the tests
         """
         if tests is None:
             # Make sure to return all the manifests, even ones without tests.
             return self.manifest_defaults.keys()
         manifests = []
         for test in tests:
             manifest = test.get('manifest')
             if not manifest:
                 continue
             if manifest not in manifests:
                 manifests.append(manifest)
         return manifests
     def paths(self):
         return [i['path'] for i in self.tests]
     ### methods for auditing
     def missing(self, tests=None):
         """return list of tests that do not exist on the filesystem"""
         if tests is None:
             tests = self.tests
         return [test for test in tests
                 if not os.path.exists(test['path'])]
     def verifyDirectory(self, directories, pattern=None, extensions=None):
         """
         checks what is on the filesystem vs what is in a manifest
         returns a 2-tuple of sets:
         (missing_from_filesystem, missing_from_manifest)
         """
         files = set([])
         if isinstance(directories, basestring):
             directories = [directories]
         # get files in directories
         for directory in directories:
             for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
                 # only add files that match a pattern
                 if pattern:
                     filenames = fnmatch.filter(filenames, pattern)
                 # only add files that have one of the extensions
                 if extensions:
                     filenames = [filename for filename in filenames
                                  if os.path.splitext(filename)[-1] in extensions]
                 files.update([os.path.join(dirpath, filename) for filename in filenames])
         paths = set(self.paths())
         missing_from_filesystem = paths.difference(files)
         missing_from_manifest = files.difference(paths)
         return (missing_from_filesystem, missing_from_manifest)
     ### methods for output
     def write(self, fp=sys.stdout, rootdir=None,
               global_tags=None, global_kwargs=None,
               local_tags=None, local_kwargs=None):
         """
         write a manifest given a query
         global and local options will be munged to do the query
         globals will be written to the top of the file
         locals (if given) will be written per test
         """
         # open file if `fp` given as string
         close = False
         if isinstance(fp, string):
             fp = file(fp, 'w')
             close = True
         # root directory
         if rootdir is None:
             rootdir = self.rootdir
         # sanitize input
         global_tags = global_tags or set()
         local_tags = local_tags or set()
         global_kwargs = global_kwargs or {}
         local_kwargs = local_kwargs or {}
         # create the query
         tags = set([])
         tags.update(global_tags)
         tags.update(local_tags)
         kwargs = {}
         kwargs.update(global_kwargs)
         kwargs.update(local_kwargs)
         # get matching tests
         tests = self.get(tags=tags, **kwargs)
         # print the .ini manifest
         if global_tags or global_kwargs:
             print >> fp, '[DEFAULT]'
             for tag in global_tags:
                 print >> fp, '%s =' % tag
             for key, value in global_kwargs.items():
                 print >> fp, '%s = %s' % (key, value)
             print >> fp
         for test in tests:
             test = test.copy() # don't overwrite
             path = test['name']
             if not os.path.isabs(path):
                 path = test['path']
                 if self.rootdir:
                     path = relpath(test['path'], self.rootdir)
                 path = denormalize_path(path)
             print >> fp, '[%s]' % path
             # reserved keywords:
             reserved = ['path', 'name', 'here', 'manifest', 'relpath']
             for key in sorted(test.keys()):
                 if key in reserved:
                     continue
                 if key in global_kwargs:
                     continue
                 if key in global_tags and not test[key]:
                     continue
                 print >> fp, '%s = %s' % (key, test[key])
             print >> fp
         if close:
             # close the created file
             fp.close()
     def __str__(self):
         fp = StringIO()
         self.write(fp=fp)
         value = fp.getvalue()
         return value
     def copy(self, directory, rootdir=None, *tags, **kwargs):
         """
         copy the manifests and associated tests
         - directory : directory to copy to
         - rootdir : root directory to copy to (if not given from manifests)
         - tags : keywords the tests must have
         - kwargs : key, values the tests must match
         """
         # XXX note that copy does *not* filter the tests out of the
         # resulting manifest; it just stupidly copies them over.
         # ideally, it would reread the manifests and filter out the
         # tests that don't match *tags and **kwargs
         # destination
         if not os.path.exists(directory):
             os.path.makedirs(directory)
         else:
             # sanity check
             assert os.path.isdir(directory)
         # tests to copy
         tests = self.get(tags=tags, **kwargs)
         if not tests:
             return # nothing to do!
         # root directory
         if rootdir is None:
             rootdir = self.rootdir
         # copy the manifests + tests
         manifests = [relpath(manifest, rootdir) for manifest in self.manifests()]
         for manifest in manifests:
             destination = os.path.join(directory, manifest)
             dirname = os.path.dirname(destination)
             if not os.path.exists(dirname):
                 os.makedirs(dirname)
             else:
                 # sanity check
                 assert os.path.isdir(dirname)
             shutil.copy(os.path.join(rootdir, manifest), destination)
         for test in tests:
             if os.path.isabs(test['name']):
                 continue
             source = test['path']
             if not os.path.exists(source):
                 print >> sys.stderr, "Missing test: '%s' does not exist!" % source
                 continue
                 # TODO: should err on strict
             destination = os.path.join(directory, relpath(test['path'], rootdir))
             shutil.copy(source, destination)
             # TODO: ensure that all of the tests are below the from_dir
     def update(self, from_dir, rootdir=None, *tags, **kwargs):
         """
         update the tests as listed in a manifest from a directory
         - from_dir : directory where the tests live
         - rootdir : root directory to copy to (if not given from manifests)
         - tags : keys the tests must have
         - kwargs : key, values the tests must match
         """
         # get the tests
         tests = self.get(tags=tags, **kwargs)
         # get the root directory
         if not rootdir:
             rootdir = self.rootdir
         # copy them!
         for test in tests:
             if not os.path.isabs(test['name']):
                 _relpath = relpath(test['path'], rootdir)
                 source = os.path.join(from_dir, _relpath)
                 if not os.path.exists(source):
                     # TODO err on strict
                     print >> sys.stderr, "Missing test: '%s'; skipping" % test['name']
                     continue
                 destination = os.path.join(rootdir, _relpath)
                 shutil.copy(source, destination)
     ### directory importers
     @classmethod
     def _walk_directories(cls, directories, function, pattern=None, ignore=()):
         """
         internal function to import directories
         """
         class FilteredDirectoryContents(object):
             """class to filter directory contents"""
             sort = sorted
             def __init__(self, pattern=pattern, ignore=ignore, cache=None):
                 if pattern is None:
                     pattern = set()
                 if isinstance(pattern, basestring):
                     pattern = [pattern]
                 self.patterns = pattern
                 self.ignore = set(ignore)
                 # cache of (dirnames, filenames) keyed on directory real path
                 # assumes volume is frozen throughout scope
                 self._cache = cache or {}
             def __call__(self, directory):
                 """returns 2-tuple: dirnames, filenames"""
                 directory = os.path.realpath(directory)
                 if directory not in self._cache:
                     dirnames, filenames = self.contents(directory)
                     # filter out directories without progeny
                     # XXX recursive: should keep track of seen directories
                     dirnames = [ dirname for dirname in dirnames
                                  if not self.empty(os.path.join(directory, dirname)) ]
                     self._cache[directory] = (tuple(dirnames), filenames)
                 # return cached values
                 return self._cache[directory]
             def empty(self, directory):
                 """
                 returns if a directory and its descendents are empty
                 """
                 return self(directory) == ((), ())
             def contents(self, directory, sort=None):
                 """
                 return directory contents as (dirnames, filenames)
                 with `ignore` and `pattern` applied
                 """
                 if sort is None:
                     sort = self.sort
                 # split directories and files
                 dirnames = []
                 filenames = []
                 for item in os.listdir(directory):
                     path = os.path.join(directory, item)
                     if os.path.isdir(path):
                         dirnames.append(item)
                     else:
                         # XXX not sure what to do if neither a file or directory
                         # (if anything)
                         assert os.path.isfile(path)
                         filenames.append(item)
                 # filter contents;
                 # this could be done in situ re the above for loop
                 # but it is really disparate in intent
                 # and could conceivably go to a separate method
                 dirnames = [dirname for dirname in dirnames
                             if dirname not in self.ignore]
                 filenames = set(filenames)
                 # we use set functionality to filter filenames
                 if self.patterns:
                     matches = set()
                     matches.update(*[fnmatch.filter(filenames, pattern)
                                      for pattern in self.patterns])
                     filenames = matches
                 if sort is not None:
                     # sort dirnames, filenames
                     dirnames = sort(dirnames)
                     filenames = sort(filenames)
                 return (tuple(dirnames), tuple(filenames))
         # make a filtered directory object
         directory_contents = FilteredDirectoryContents(pattern=pattern, ignore=ignore)
         # walk the directories, generating manifests
         for index, directory in enumerate(directories):
             for dirpath, dirnames, filenames in os.walk(directory):
                 # get the directory contents from the caching object
                 _dirnames, filenames = directory_contents(dirpath)
                 # filter out directory names
                 dirnames[:] = _dirnames
                 # call callback function
                 function(directory, dirpath, dirnames, filenames)
     @classmethod
     def populate_directory_manifests(cls, directories, filename, pattern=None, ignore=(), overwrite=False):
         """
         walks directories and writes manifests of name `filename` in-place; returns `cls` instance populated
         with the given manifests
         filename -- filename of manifests to write
         pattern -- shell pattern (glob) or patterns of filenames to match
         ignore -- directory names to ignore
         overwrite -- whether to overwrite existing files of given name
         """
         manifest_dict = {}
         seen = [] # top-level directories seen
         if os.path.basename(filename) != filename:
             raise IOError("filename should not include directory name")
         # no need to hit directories more than once
         _directories = directories
         directories = []
         for directory in _directories:
             if directory not in directories:
                 directories.append(directory)
         def callback(directory, dirpath, dirnames, filenames):
             """write a manifest for each directory"""
             manifest_path = os.path.join(dirpath, filename)
             if (dirnames or filenames) and not (os.path.exists(manifest_path) and overwrite):
                 with file(manifest_path, 'w') as manifest:
                     for dirname in dirnames:
                         print >> manifest, '[include:%s]' % os.path.join(dirname, filename)
                     for _filename in filenames:
                         print >> manifest, '[%s]' % _filename
                 # add to list of manifests
                 manifest_dict.setdefault(directory, manifest_path)
         # walk the directories to gather files
         cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore)
         # get manifests
         manifests = [manifest_dict[directory] for directory in _directories]
         # create a `cls` instance with the manifests
         return cls(manifests=manifests)
     @classmethod
     def from_directories(cls, directories, pattern=None, ignore=(), write=None, relative_to=None):
         """
         convert directories to a simple manifest; returns ManifestParser instance
         pattern -- shell pattern (glob) or patterns of filenames to match
         ignore -- directory names to ignore
         write -- filename or file-like object of manifests to write;
                  if `None` then a StringIO instance will be created
         relative_to -- write paths relative to this path;
                        if false then the paths are absolute
         """
         # determine output
         opened_manifest_file = None # name of opened manifest file
         absolute = not relative_to # whether to output absolute path names as names
         if isinstance(write, string):
             opened_manifest_file = write
             write = file(write, 'w')
         if write is None:
             write = StringIO()
         # walk the directories, generating manifests
         def callback(directory, dirpath, dirnames, filenames):
             # absolute paths
             filenames = [os.path.join(dirpath, filename)
                          for filename in filenames]
             # ensure new manifest isn't added
             filenames = [filename for filename in filenames
                          if filename != opened_manifest_file]
             # normalize paths
             if not absolute and relative_to:
                 filenames = [relpath(filename, relative_to)
                              for filename in filenames]
             # write to manifest
             print >> write, '\n'.join(['[%s]' % denormalize_path(filename)
                                                for filename in filenames])
         cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore)
         if opened_manifest_file:
             # close file
             write.close()
             manifests = [opened_manifest_file]
         else:
             # manifests/write is a file-like object;
             # rewind buffer
             write.flush()
             write.seek(0)
             manifests = [write]
         # make a ManifestParser instance
         return cls(manifests=manifests)
 convert = ManifestParser.from_directories
 class TestManifest(ManifestParser):
     """
     apply logic to manifests;  this is your integration layer :)
     specific harnesses may subclass from this if they need more logic
     """
     def filter(self, values, tests):
         """
         filter on a specific list tag, e.g.:
         run-if = os == win linux
         skip-if = os == mac
         """
         # tags:
         run_tag = 'run-if'
         skip_tag = 'skip-if'
         fail_tag = 'fail-if'
         # loop over test
         for test in tests:
             reason = None # reason to disable
             # tagged-values to run
             if run_tag in test:
                 condition = test[run_tag]
                 if not parse(condition, **values):
                     reason = '%s: %s' % (run_tag, condition)
             # tagged-values to skip
             if skip_tag in test:
                 condition = test[skip_tag]
                 if parse(condition, **values):
                     reason = '%s: %s' % (skip_tag, condition)
             # mark test as disabled if there's a reason
             if reason:
                 test.setdefault('disabled', reason)
             # mark test as a fail if so indicated
             if fail_tag in test:
                 condition = test[fail_tag]
                 if parse(condition, **values):
                     test['expected'] = 'fail'
     def active_tests(self, exists=True, disabled=True, options=None, **values):
         """
         - exists : return only existing tests
         - disabled : whether to return disabled tests
         - tags : keys and values to filter on (e.g. `os = linux mac`)
         """
         tests = [i.copy() for i in self.tests] # shallow copy
         # Filter on current subsuite
         if options:
             if  options.subsuite:
                 tests = [test for test in tests if options.subsuite == test['subsuite']]
             else:
                 tests = [test for test in tests if not test['subsuite']]
         # mark all tests as passing unless indicated otherwise
         for test in tests:
             test['expected'] = test.get('expected', 'pass')
         # ignore tests that do not exist
         if exists:
             tests = [test for test in tests if os.path.exists(test['path'])]
         # filter by tags
         self.filter(values, tests)
         # ignore disabled tests if specified
         if not disabled:
             tests = [test for test in tests
                      if not 'disabled' in test]
         # return active tests
         return tests
     def test_paths(self):
         return [test['path'] for test in self.active_tests()]
 ### command line attributes
 class ParserError(Exception):
   """error for exceptions while parsing the command line"""
 def parse_args(_args):
     """
     parse and return:
     --keys=value (or --key value)
     -tags
     args
     """
     # return values
     _dict = {}
     tags = []
     args = []
     # parse the arguments
     key = None
     for arg in _args:
         if arg.startswith('---'):
             raise ParserError("arguments should start with '-' or '--' only")
         elif arg.startswith('--'):
             if key:
                 raise ParserError("Key %s still open" % key)
             key = arg[2:]
             if '=' in key:
                 key, value = key.split('=', 1)
                 _dict[key] = value
                 key = None
                 continue
         elif arg.startswith('-'):
             if key:
                 raise ParserError("Key %s still open" % key)
             tags.append(arg[1:])
             continue
         else:
             if key:
                 _dict[key] = arg
                 continue
             args.append(arg)
     # return values
     return (_dict, tags, args)
 ### classes for subcommands
 class CLICommand(object):
     usage = '%prog [options] command'
     def __init__(self, parser):
       self._parser = parser # master parser
     def parser(self):
       return OptionParser(usage=self.usage, description=self.__doc__,
                           add_help_option=False)
 class Copy(CLICommand):
     usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
     def __call__(self, options, args):
       # parse the arguments
       try:
         kwargs, tags, args = parse_args(args)
       except ParserError, e:
         self._parser.error(e.message)
       # make sure we have some manifests, otherwise it will
       # be quite boring
       if not len(args) == 2:
         HelpCLI(self._parser)(options, ['copy'])
         return
       # read the manifests
       # TODO: should probably ensure these exist here
       manifests = ManifestParser()
       manifests.read(args[0])
       # print the resultant query
       manifests.copy(args[1], None, *tags, **kwargs)
 class CreateCLI(CLICommand):
     """
     create a manifest from a list of directories
     """
     usage = '%prog [options] create directory <directory> <...>'
     def parser(self):
         parser = CLICommand.parser(self)
         parser.add_option('-p', '--pattern', dest='pattern',
                           help="glob pattern for files")
         parser.add_option('-i', '--ignore', dest='ignore',
                           default=[], action='append',
                           help='directories to ignore')
         parser.add_option('-w', '--in-place', dest='in_place',
                           help='Write .ini files in place; filename to write to')
         return parser
     def __call__(self, _options, args):
         parser = self.parser()
         options, args = parser.parse_args(args)
         # need some directories
         if not len(args):
             parser.print_usage()
             return
         # add the directories to the manifest
         for arg in args:
             assert os.path.exists(arg)
             assert os.path.isdir(arg)
             manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
                                write=options.in_place)
         if manifest:
             print manifest
 class WriteCLI(CLICommand):
     """
     write a manifest based on a query
     """
     usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
     def __call__(self, options, args):
         # parse the arguments
         try:
             kwargs, tags, args = parse_args(args)
         except ParserError, e:
             self._parser.error(e.message)
         # make sure we have some manifests, otherwise it will
         # be quite boring
         if not args:
             HelpCLI(self._parser)(options, ['write'])
             return
         # read the manifests
         # TODO: should probably ensure these exist here
         manifests = ManifestParser()
         manifests.read(*args)
         # print the resultant query
         manifests.write(global_tags=tags, global_kwargs=kwargs)
 class HelpCLI(CLICommand):
     """
     get help on a command
     """
     usage = '%prog [options] help [command]'
     def __call__(self, options, args):
         if len(args) == 1 and args[0] in commands:
             commands[args[0]](self._parser).parser().print_help()
         else:
             self._parser.print_help()
             print '\nCommands:'
             for command in sorted(commands):
                 print '  %s : %s' % (command, commands[command].__doc__.strip())
 class UpdateCLI(CLICommand):
     """
     update the tests as listed in a manifest from a directory
     """
     usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
     def __call__(self, options, args):
         # parse the arguments
         try:
             kwargs, tags, args = parse_args(args)
         except ParserError, e:
             self._parser.error(e.message)
         # make sure we have some manifests, otherwise it will
         # be quite boring
         if not len(args) == 2:
             HelpCLI(self._parser)(options, ['update'])
             return
         # read the manifests
         # TODO: should probably ensure these exist here
         manifests = ManifestParser()
         manifests.read(args[0])
         # print the resultant query
         manifests.update(args[1], None, *tags, **kwargs)
 # command -> class mapping
 commands = { 'create': CreateCLI,
              'help': HelpCLI,
              'update': UpdateCLI,
              'write': WriteCLI }
 def main(args=sys.argv[1:]):
     """console_script entry point"""
     # set up an option parser
     usage = '%prog [options] [command] ...'
     description = "%s. Use `help` to display commands" % __doc__.strip()
     parser = OptionParser(usage=usage, description=description)
     parser.add_option('-s', '--strict', dest='strict',
                       action='store_true', default=False,
                       help='adhere strictly to errors')
     parser.disable_interspersed_args()
     options, args = parser.parse_args(args)
     if not args:
         HelpCLI(parser)(options, args)
         parser.exit()
     # get the command
     command = args[0]
     if command not in commands:
         parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
     handler = commands[command](parser)
     handler(options, args[1:])
 if __name__ == '__main__':
     main()

The Tor Browser / annotate

testing/mozbase/manifestdestiny/manifestparser/manifestparser.py@6474c204b198 (annotated)

testing/mozbase/manifestdestiny/manifestparser/manifestparser.py