testing/mozbase/manifestdestiny/manifestparser/manifestparser.py@6474c204b198
testing/mozbase/manifestdestiny/manifestparser/manifestparser.py
Wed, 31 Dec 2014 06:09:35 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:09:35 +0100
- changeset 0
- 6474c204b198
- permissions
- -rwxr-xr-x
Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.
1 #!/usr/bin/env python
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 # You can obtain one at http://mozilla.org/MPL/2.0/.
7 """
8 Mozilla universal manifest parser
9 """
11 __all__ = ['read_ini', # .ini reader
12 'ManifestParser', 'TestManifest', 'convert', # manifest handling
13 'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
15 import fnmatch
16 import os
17 import re
18 import shutil
19 import sys
21 from optparse import OptionParser
22 from StringIO import StringIO
24 relpath = os.path.relpath
25 string = (basestring,)
28 # expr.py
29 # from:
30 # http://k0s.org/mozilla/hg/expressionparser
31 # http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
33 # Implements a top-down parser/evaluator for simple boolean expressions.
34 # ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
35 #
36 # Rough grammar:
37 # expr := literal
38 # | '(' expr ')'
39 # | expr '&&' expr
40 # | expr '||' expr
41 # | expr '==' expr
42 # | expr '!=' expr
43 # literal := BOOL
44 # | INT
45 # | STRING
46 # | IDENT
47 # BOOL := true|false
48 # INT := [0-9]+
49 # STRING := "[^"]*"
50 # IDENT := [A-Za-z_]\w*
52 # Identifiers take their values from a mapping dictionary passed as the second
53 # argument.
55 # Glossary (see above URL for details):
56 # - nud: null denotation
57 # - led: left detonation
58 # - lbp: left binding power
59 # - rbp: right binding power
61 class ident_token(object):
62 def __init__(self, value):
63 self.value = value
64 def nud(self, parser):
65 # identifiers take their value from the value mappings passed
66 # to the parser
67 return parser.value(self.value)
69 class literal_token(object):
70 def __init__(self, value):
71 self.value = value
72 def nud(self, parser):
73 return self.value
75 class eq_op_token(object):
76 "=="
77 def led(self, parser, left):
78 return left == parser.expression(self.lbp)
80 class neq_op_token(object):
81 "!="
82 def led(self, parser, left):
83 return left != parser.expression(self.lbp)
85 class not_op_token(object):
86 "!"
87 def nud(self, parser):
88 return not parser.expression(100)
90 class and_op_token(object):
91 "&&"
92 def led(self, parser, left):
93 right = parser.expression(self.lbp)
94 return left and right
96 class or_op_token(object):
97 "||"
98 def led(self, parser, left):
99 right = parser.expression(self.lbp)
100 return left or right
102 class lparen_token(object):
103 "("
104 def nud(self, parser):
105 expr = parser.expression()
106 parser.advance(rparen_token)
107 return expr
109 class rparen_token(object):
110 ")"
112 class end_token(object):
113 """always ends parsing"""
115 ### derived literal tokens
117 class bool_token(literal_token):
118 def __init__(self, value):
119 value = {'true':True, 'false':False}[value]
120 literal_token.__init__(self, value)
122 class int_token(literal_token):
123 def __init__(self, value):
124 literal_token.__init__(self, int(value))
126 class string_token(literal_token):
127 def __init__(self, value):
128 literal_token.__init__(self, value[1:-1])
130 precedence = [(end_token, rparen_token),
131 (or_op_token,),
132 (and_op_token,),
133 (eq_op_token, neq_op_token),
134 (lparen_token,),
135 ]
136 for index, rank in enumerate(precedence):
137 for token in rank:
138 token.lbp = index # lbp = lowest left binding power
140 class ParseError(Exception):
141 """error parsing conditional expression"""
143 class ExpressionParser(object):
144 """
145 A parser for a simple expression language.
147 The expression language can be described as follows::
149 EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION
150 OP ::= '==' | '!=' | '&&' | '||'
151 LITERAL ::= BOOL | INT | IDENT | STRING
152 BOOL ::= 'true' | 'false'
153 INT ::= [0-9]+
154 IDENT ::= [a-zA-Z_]\w*
155 STRING ::= '"' [^\"] '"' | ''' [^\'] '''
157 At its core, expressions consist of booleans, integers, identifiers and.
158 strings. Booleans are one of *true* or *false*. Integers are a series
159 of digits. Identifiers are a series of English letters and underscores.
160 Strings are a pair of matching quote characters (single or double) with
161 zero or more characters inside.
163 Expressions can be combined with operators: the equals (==) and not
164 equals (!=) operators compare two expressions and produce a boolean. The
165 and (&&) and or (||) operators take two expressions and produce the logical
166 AND or OR value of them, respectively. An expression can also be prefixed
167 with the not (!) operator, which produces its logical negation.
169 Finally, any expression may be contained within parentheses for grouping.
171 Identifiers take their values from the mapping provided.
172 """
173 def __init__(self, text, valuemapping, strict=False):
174 """
175 Initialize the parser
176 :param text: The expression to parse as a string.
177 :param valuemapping: A dict mapping identifier names to values.
178 :param strict: If true, referencing an identifier that was not
179 provided in :valuemapping: will raise an error.
180 """
181 self.text = text
182 self.valuemapping = valuemapping
183 self.strict = strict
185 def _tokenize(self):
186 """
187 Lex the input text into tokens and yield them in sequence.
188 """
189 # scanner callbacks
190 def bool_(scanner, t): return bool_token(t)
191 def identifier(scanner, t): return ident_token(t)
192 def integer(scanner, t): return int_token(t)
193 def eq(scanner, t): return eq_op_token()
194 def neq(scanner, t): return neq_op_token()
195 def or_(scanner, t): return or_op_token()
196 def and_(scanner, t): return and_op_token()
197 def lparen(scanner, t): return lparen_token()
198 def rparen(scanner, t): return rparen_token()
199 def string_(scanner, t): return string_token(t)
200 def not_(scanner, t): return not_op_token()
202 scanner = re.Scanner([
203 # Note: keep these in sync with the class docstring above.
204 (r"true|false", bool_),
205 (r"[a-zA-Z_]\w*", identifier),
206 (r"[0-9]+", integer),
207 (r'("[^"]*")|(\'[^\']*\')', string_),
208 (r"==", eq),
209 (r"!=", neq),
210 (r"\|\|", or_),
211 (r"!", not_),
212 (r"&&", and_),
213 (r"\(", lparen),
214 (r"\)", rparen),
215 (r"\s+", None), # skip whitespace
216 ])
217 tokens, remainder = scanner.scan(self.text)
218 for t in tokens:
219 yield t
220 yield end_token()
222 def value(self, ident):
223 """
224 Look up the value of |ident| in the value mapping passed in the
225 constructor.
226 """
227 if self.strict:
228 return self.valuemapping[ident]
229 else:
230 return self.valuemapping.get(ident, None)
232 def advance(self, expected):
233 """
234 Assert that the next token is an instance of |expected|, and advance
235 to the next token.
236 """
237 if not isinstance(self.token, expected):
238 raise Exception, "Unexpected token!"
239 self.token = self.iter.next()
241 def expression(self, rbp=0):
242 """
243 Parse and return the value of an expression until a token with
244 right binding power greater than rbp is encountered.
245 """
246 t = self.token
247 self.token = self.iter.next()
248 left = t.nud(self)
249 while rbp < self.token.lbp:
250 t = self.token
251 self.token = self.iter.next()
252 left = t.led(self, left)
253 return left
255 def parse(self):
256 """
257 Parse and return the value of the expression in the text
258 passed to the constructor. Raises a ParseError if the expression
259 could not be parsed.
260 """
261 try:
262 self.iter = self._tokenize()
263 self.token = self.iter.next()
264 return self.expression()
265 except:
266 raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
268 __call__ = parse
270 def parse(text, **values):
271 """
272 Parse and evaluate a boolean expression.
273 :param text: The expression to parse, as a string.
274 :param values: A dict containing a name to value mapping for identifiers
275 referenced in *text*.
276 :rtype: the final value of the expression.
277 :raises: :py:exc::ParseError: will be raised if parsing fails.
278 """
279 return ExpressionParser(text, values).parse()
282 ### path normalization
284 def normalize_path(path):
285 """normalize a relative path"""
286 if sys.platform.startswith('win'):
287 return path.replace('/', os.path.sep)
288 return path
290 def denormalize_path(path):
291 """denormalize a relative path"""
292 if sys.platform.startswith('win'):
293 return path.replace(os.path.sep, '/')
294 return path
297 ### .ini reader
299 def read_ini(fp, variables=None, default='DEFAULT',
300 comments=';#', separators=('=', ':'),
301 strict=True):
302 """
303 read an .ini file and return a list of [(section, values)]
304 - fp : file pointer or path to read
305 - variables : default set of variables
306 - default : name of the section for the default section
307 - comments : characters that if they start a line denote a comment
308 - separators : strings that denote key, value separation in order
309 - strict : whether to be strict about parsing
310 """
312 # variables
313 variables = variables or {}
314 sections = []
315 key = value = None
316 section_names = set()
317 if isinstance(fp, basestring):
318 fp = file(fp)
320 # read the lines
321 for (linenum, line) in enumerate(fp.readlines(), start=1):
323 stripped = line.strip()
325 # ignore blank lines
326 if not stripped:
327 # reset key and value to avoid continuation lines
328 key = value = None
329 continue
331 # ignore comment lines
332 if stripped[0] in comments:
333 continue
335 # check for a new section
336 if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
337 section = stripped[1:-1].strip()
338 key = value = None
340 # deal with DEFAULT section
341 if section.lower() == default.lower():
342 if strict:
343 assert default not in section_names
344 section_names.add(default)
345 current_section = variables
346 continue
348 if strict:
349 # make sure this section doesn't already exist
350 assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names)
352 section_names.add(section)
353 current_section = {}
354 sections.append((section, current_section))
355 continue
357 # if there aren't any sections yet, something bad happen
358 if not section_names:
359 raise Exception('No sections found')
361 # (key, value) pair
362 for separator in separators:
363 if separator in stripped:
364 key, value = stripped.split(separator, 1)
365 key = key.strip()
366 value = value.strip()
368 if strict:
369 # make sure this key isn't already in the section or empty
370 assert key
371 if current_section is not variables:
372 assert key not in current_section
374 current_section[key] = value
375 break
376 else:
377 # continuation line ?
378 if line[0].isspace() and key:
379 value = '%s%s%s' % (value, os.linesep, stripped)
380 current_section[key] = value
381 else:
382 # something bad happened!
383 if hasattr(fp, 'name'):
384 filename = fp.name
385 else:
386 filename = 'unknown'
387 raise Exception("Error parsing manifest file '%s', line %s" %
388 (filename, linenum))
390 # interpret the variables
391 def interpret_variables(global_dict, local_dict):
392 variables = global_dict.copy()
393 if 'skip-if' in local_dict and 'skip-if' in variables:
394 local_dict['skip-if'] = "(%s) || (%s)" % (variables['skip-if'].split('#')[0], local_dict['skip-if'].split('#')[0])
395 variables.update(local_dict)
397 return variables
399 sections = [(i, interpret_variables(variables, j)) for i, j in sections]
400 return sections
403 ### objects for parsing manifests
405 class ManifestParser(object):
406 """read .ini manifests"""
408 def __init__(self, manifests=(), defaults=None, strict=True):
409 self._defaults = defaults or {}
410 self.tests = []
411 self.manifest_defaults = {}
412 self.strict = strict
413 self.rootdir = None
414 self.relativeRoot = None
415 if manifests:
416 self.read(*manifests)
418 def getRelativeRoot(self, root):
419 return root
421 ### methods for reading manifests
423 def _read(self, root, filename, defaults):
425 # get directory of this file if not file-like object
426 if isinstance(filename, string):
427 filename = os.path.abspath(filename)
428 fp = open(filename)
429 here = os.path.dirname(filename)
430 else:
431 fp = filename
432 filename = here = None
433 defaults['here'] = here
435 # Rootdir is needed for relative path calculation. Precompute it for
436 # the microoptimization used below.
437 if self.rootdir is None:
438 rootdir = ""
439 else:
440 assert os.path.isabs(self.rootdir)
441 rootdir = self.rootdir + os.path.sep
443 # read the configuration
444 sections = read_ini(fp=fp, variables=defaults, strict=self.strict)
445 self.manifest_defaults[filename] = defaults
447 # get the tests
448 for section, data in sections:
449 subsuite = ''
450 if 'subsuite' in data:
451 subsuite = data['subsuite']
453 # a file to include
454 # TODO: keep track of included file structure:
455 # self.manifests = {'manifest.ini': 'relative/path.ini'}
456 if section.startswith('include:'):
457 include_file = section.split('include:', 1)[-1]
458 include_file = normalize_path(include_file)
459 if not os.path.isabs(include_file):
460 include_file = os.path.join(self.getRelativeRoot(here), include_file)
461 if not os.path.exists(include_file):
462 message = "Included file '%s' does not exist" % include_file
463 if self.strict:
464 raise IOError(message)
465 else:
466 sys.stderr.write("%s\n" % message)
467 continue
468 include_defaults = data.copy()
469 self._read(root, include_file, include_defaults)
470 continue
472 # otherwise an item
473 test = data
474 test['name'] = section
476 # Will be None if the manifest being read is a file-like object.
477 test['manifest'] = filename
479 # determine the path
480 path = test.get('path', section)
481 _relpath = path
482 if '://' not in path: # don't futz with URLs
483 path = normalize_path(path)
484 if here and not os.path.isabs(path):
485 path = os.path.normpath(os.path.join(here, path))
487 # Microoptimization, because relpath is quite expensive.
488 # We know that rootdir is an absolute path or empty. If path
489 # starts with rootdir, then path is also absolute and the tail
490 # of the path is the relative path (possibly non-normalized,
491 # when here is unknown).
492 # For this to work rootdir needs to be terminated with a path
493 # separator, so that references to sibling directories with
494 # a common prefix don't get misscomputed (e.g. /root and
495 # /rootbeer/file).
496 # When the rootdir is unknown, the relpath needs to be left
497 # unchanged. We use an empty string as rootdir in that case,
498 # which leaves relpath unchanged after slicing.
499 if path.startswith(rootdir):
500 _relpath = path[len(rootdir):]
501 else:
502 _relpath = relpath(path, rootdir)
504 test['subsuite'] = subsuite
505 test['path'] = path
506 test['relpath'] = _relpath
508 # append the item
509 self.tests.append(test)
511 def read(self, *filenames, **defaults):
512 """
513 read and add manifests from file paths or file-like objects
515 filenames -- file paths or file-like objects to read as manifests
516 defaults -- default variables
517 """
519 # ensure all files exist
520 missing = [filename for filename in filenames
521 if isinstance(filename, string) and not os.path.exists(filename) ]
522 if missing:
523 raise IOError('Missing files: %s' % ', '.join(missing))
525 # default variables
526 _defaults = defaults.copy() or self._defaults.copy()
527 _defaults.setdefault('here', None)
529 # process each file
530 for filename in filenames:
531 # set the per file defaults
532 defaults = _defaults.copy()
533 here = None
534 if isinstance(filename, string):
535 here = os.path.dirname(os.path.abspath(filename))
536 defaults['here'] = here # directory of master .ini file
538 if self.rootdir is None:
539 # set the root directory
540 # == the directory of the first manifest given
541 self.rootdir = here
543 self._read(here, filename, defaults)
546 ### methods for querying manifests
548 def query(self, *checks, **kw):
549 """
550 general query function for tests
551 - checks : callable conditions to test if the test fulfills the query
552 """
553 tests = kw.get('tests', None)
554 if tests is None:
555 tests = self.tests
556 retval = []
557 for test in tests:
558 for check in checks:
559 if not check(test):
560 break
561 else:
562 retval.append(test)
563 return retval
565 def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs):
566 # TODO: pass a dict instead of kwargs since you might hav
567 # e.g. 'inverse' as a key in the dict
569 # TODO: tags should just be part of kwargs with None values
570 # (None == any is kinda weird, but probably still better)
572 # fix up tags
573 if tags:
574 tags = set(tags)
575 else:
576 tags = set()
578 # make some check functions
579 if inverse:
580 has_tags = lambda test: not tags.intersection(test.keys())
581 def dict_query(test):
582 for key, value in kwargs.items():
583 if test.get(key) == value:
584 return False
585 return True
586 else:
587 has_tags = lambda test: tags.issubset(test.keys())
588 def dict_query(test):
589 for key, value in kwargs.items():
590 if test.get(key) != value:
591 return False
592 return True
594 # query the tests
595 tests = self.query(has_tags, dict_query, tests=tests)
597 # if a key is given, return only a list of that key
598 # useful for keys like 'name' or 'path'
599 if _key:
600 return [test[_key] for test in tests]
602 # return the tests
603 return tests
605 def manifests(self, tests=None):
606 """
607 return manifests in order in which they appear in the tests
608 """
609 if tests is None:
610 # Make sure to return all the manifests, even ones without tests.
611 return self.manifest_defaults.keys()
613 manifests = []
614 for test in tests:
615 manifest = test.get('manifest')
616 if not manifest:
617 continue
618 if manifest not in manifests:
619 manifests.append(manifest)
620 return manifests
622 def paths(self):
623 return [i['path'] for i in self.tests]
626 ### methods for auditing
628 def missing(self, tests=None):
629 """return list of tests that do not exist on the filesystem"""
630 if tests is None:
631 tests = self.tests
632 return [test for test in tests
633 if not os.path.exists(test['path'])]
635 def verifyDirectory(self, directories, pattern=None, extensions=None):
636 """
637 checks what is on the filesystem vs what is in a manifest
638 returns a 2-tuple of sets:
639 (missing_from_filesystem, missing_from_manifest)
640 """
642 files = set([])
643 if isinstance(directories, basestring):
644 directories = [directories]
646 # get files in directories
647 for directory in directories:
648 for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
650 # only add files that match a pattern
651 if pattern:
652 filenames = fnmatch.filter(filenames, pattern)
654 # only add files that have one of the extensions
655 if extensions:
656 filenames = [filename for filename in filenames
657 if os.path.splitext(filename)[-1] in extensions]
659 files.update([os.path.join(dirpath, filename) for filename in filenames])
661 paths = set(self.paths())
662 missing_from_filesystem = paths.difference(files)
663 missing_from_manifest = files.difference(paths)
664 return (missing_from_filesystem, missing_from_manifest)
667 ### methods for output
669 def write(self, fp=sys.stdout, rootdir=None,
670 global_tags=None, global_kwargs=None,
671 local_tags=None, local_kwargs=None):
672 """
673 write a manifest given a query
674 global and local options will be munged to do the query
675 globals will be written to the top of the file
676 locals (if given) will be written per test
677 """
679 # open file if `fp` given as string
680 close = False
681 if isinstance(fp, string):
682 fp = file(fp, 'w')
683 close = True
685 # root directory
686 if rootdir is None:
687 rootdir = self.rootdir
689 # sanitize input
690 global_tags = global_tags or set()
691 local_tags = local_tags or set()
692 global_kwargs = global_kwargs or {}
693 local_kwargs = local_kwargs or {}
695 # create the query
696 tags = set([])
697 tags.update(global_tags)
698 tags.update(local_tags)
699 kwargs = {}
700 kwargs.update(global_kwargs)
701 kwargs.update(local_kwargs)
703 # get matching tests
704 tests = self.get(tags=tags, **kwargs)
706 # print the .ini manifest
707 if global_tags or global_kwargs:
708 print >> fp, '[DEFAULT]'
709 for tag in global_tags:
710 print >> fp, '%s =' % tag
711 for key, value in global_kwargs.items():
712 print >> fp, '%s = %s' % (key, value)
713 print >> fp
715 for test in tests:
716 test = test.copy() # don't overwrite
718 path = test['name']
719 if not os.path.isabs(path):
720 path = test['path']
721 if self.rootdir:
722 path = relpath(test['path'], self.rootdir)
723 path = denormalize_path(path)
724 print >> fp, '[%s]' % path
726 # reserved keywords:
727 reserved = ['path', 'name', 'here', 'manifest', 'relpath']
728 for key in sorted(test.keys()):
729 if key in reserved:
730 continue
731 if key in global_kwargs:
732 continue
733 if key in global_tags and not test[key]:
734 continue
735 print >> fp, '%s = %s' % (key, test[key])
736 print >> fp
738 if close:
739 # close the created file
740 fp.close()
742 def __str__(self):
743 fp = StringIO()
744 self.write(fp=fp)
745 value = fp.getvalue()
746 return value
748 def copy(self, directory, rootdir=None, *tags, **kwargs):
749 """
750 copy the manifests and associated tests
751 - directory : directory to copy to
752 - rootdir : root directory to copy to (if not given from manifests)
753 - tags : keywords the tests must have
754 - kwargs : key, values the tests must match
755 """
756 # XXX note that copy does *not* filter the tests out of the
757 # resulting manifest; it just stupidly copies them over.
758 # ideally, it would reread the manifests and filter out the
759 # tests that don't match *tags and **kwargs
761 # destination
762 if not os.path.exists(directory):
763 os.path.makedirs(directory)
764 else:
765 # sanity check
766 assert os.path.isdir(directory)
768 # tests to copy
769 tests = self.get(tags=tags, **kwargs)
770 if not tests:
771 return # nothing to do!
773 # root directory
774 if rootdir is None:
775 rootdir = self.rootdir
777 # copy the manifests + tests
778 manifests = [relpath(manifest, rootdir) for manifest in self.manifests()]
779 for manifest in manifests:
780 destination = os.path.join(directory, manifest)
781 dirname = os.path.dirname(destination)
782 if not os.path.exists(dirname):
783 os.makedirs(dirname)
784 else:
785 # sanity check
786 assert os.path.isdir(dirname)
787 shutil.copy(os.path.join(rootdir, manifest), destination)
788 for test in tests:
789 if os.path.isabs(test['name']):
790 continue
791 source = test['path']
792 if not os.path.exists(source):
793 print >> sys.stderr, "Missing test: '%s' does not exist!" % source
794 continue
795 # TODO: should err on strict
796 destination = os.path.join(directory, relpath(test['path'], rootdir))
797 shutil.copy(source, destination)
798 # TODO: ensure that all of the tests are below the from_dir
800 def update(self, from_dir, rootdir=None, *tags, **kwargs):
801 """
802 update the tests as listed in a manifest from a directory
803 - from_dir : directory where the tests live
804 - rootdir : root directory to copy to (if not given from manifests)
805 - tags : keys the tests must have
806 - kwargs : key, values the tests must match
807 """
809 # get the tests
810 tests = self.get(tags=tags, **kwargs)
812 # get the root directory
813 if not rootdir:
814 rootdir = self.rootdir
816 # copy them!
817 for test in tests:
818 if not os.path.isabs(test['name']):
819 _relpath = relpath(test['path'], rootdir)
820 source = os.path.join(from_dir, _relpath)
821 if not os.path.exists(source):
822 # TODO err on strict
823 print >> sys.stderr, "Missing test: '%s'; skipping" % test['name']
824 continue
825 destination = os.path.join(rootdir, _relpath)
826 shutil.copy(source, destination)
828 ### directory importers
830 @classmethod
831 def _walk_directories(cls, directories, function, pattern=None, ignore=()):
832 """
833 internal function to import directories
834 """
836 class FilteredDirectoryContents(object):
837 """class to filter directory contents"""
839 sort = sorted
841 def __init__(self, pattern=pattern, ignore=ignore, cache=None):
842 if pattern is None:
843 pattern = set()
844 if isinstance(pattern, basestring):
845 pattern = [pattern]
846 self.patterns = pattern
847 self.ignore = set(ignore)
849 # cache of (dirnames, filenames) keyed on directory real path
850 # assumes volume is frozen throughout scope
851 self._cache = cache or {}
853 def __call__(self, directory):
854 """returns 2-tuple: dirnames, filenames"""
855 directory = os.path.realpath(directory)
856 if directory not in self._cache:
857 dirnames, filenames = self.contents(directory)
859 # filter out directories without progeny
860 # XXX recursive: should keep track of seen directories
861 dirnames = [ dirname for dirname in dirnames
862 if not self.empty(os.path.join(directory, dirname)) ]
864 self._cache[directory] = (tuple(dirnames), filenames)
866 # return cached values
867 return self._cache[directory]
869 def empty(self, directory):
870 """
871 returns if a directory and its descendents are empty
872 """
873 return self(directory) == ((), ())
875 def contents(self, directory, sort=None):
876 """
877 return directory contents as (dirnames, filenames)
878 with `ignore` and `pattern` applied
879 """
881 if sort is None:
882 sort = self.sort
884 # split directories and files
885 dirnames = []
886 filenames = []
887 for item in os.listdir(directory):
888 path = os.path.join(directory, item)
889 if os.path.isdir(path):
890 dirnames.append(item)
891 else:
892 # XXX not sure what to do if neither a file or directory
893 # (if anything)
894 assert os.path.isfile(path)
895 filenames.append(item)
897 # filter contents;
898 # this could be done in situ re the above for loop
899 # but it is really disparate in intent
900 # and could conceivably go to a separate method
901 dirnames = [dirname for dirname in dirnames
902 if dirname not in self.ignore]
903 filenames = set(filenames)
904 # we use set functionality to filter filenames
905 if self.patterns:
906 matches = set()
907 matches.update(*[fnmatch.filter(filenames, pattern)
908 for pattern in self.patterns])
909 filenames = matches
911 if sort is not None:
912 # sort dirnames, filenames
913 dirnames = sort(dirnames)
914 filenames = sort(filenames)
916 return (tuple(dirnames), tuple(filenames))
918 # make a filtered directory object
919 directory_contents = FilteredDirectoryContents(pattern=pattern, ignore=ignore)
921 # walk the directories, generating manifests
922 for index, directory in enumerate(directories):
924 for dirpath, dirnames, filenames in os.walk(directory):
926 # get the directory contents from the caching object
927 _dirnames, filenames = directory_contents(dirpath)
928 # filter out directory names
929 dirnames[:] = _dirnames
931 # call callback function
932 function(directory, dirpath, dirnames, filenames)
934 @classmethod
935 def populate_directory_manifests(cls, directories, filename, pattern=None, ignore=(), overwrite=False):
936 """
937 walks directories and writes manifests of name `filename` in-place; returns `cls` instance populated
938 with the given manifests
940 filename -- filename of manifests to write
941 pattern -- shell pattern (glob) or patterns of filenames to match
942 ignore -- directory names to ignore
943 overwrite -- whether to overwrite existing files of given name
944 """
946 manifest_dict = {}
947 seen = [] # top-level directories seen
949 if os.path.basename(filename) != filename:
950 raise IOError("filename should not include directory name")
952 # no need to hit directories more than once
953 _directories = directories
954 directories = []
955 for directory in _directories:
956 if directory not in directories:
957 directories.append(directory)
959 def callback(directory, dirpath, dirnames, filenames):
960 """write a manifest for each directory"""
962 manifest_path = os.path.join(dirpath, filename)
963 if (dirnames or filenames) and not (os.path.exists(manifest_path) and overwrite):
964 with file(manifest_path, 'w') as manifest:
965 for dirname in dirnames:
966 print >> manifest, '[include:%s]' % os.path.join(dirname, filename)
967 for _filename in filenames:
968 print >> manifest, '[%s]' % _filename
970 # add to list of manifests
971 manifest_dict.setdefault(directory, manifest_path)
973 # walk the directories to gather files
974 cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore)
975 # get manifests
976 manifests = [manifest_dict[directory] for directory in _directories]
978 # create a `cls` instance with the manifests
979 return cls(manifests=manifests)
981 @classmethod
982 def from_directories(cls, directories, pattern=None, ignore=(), write=None, relative_to=None):
983 """
984 convert directories to a simple manifest; returns ManifestParser instance
986 pattern -- shell pattern (glob) or patterns of filenames to match
987 ignore -- directory names to ignore
988 write -- filename or file-like object of manifests to write;
989 if `None` then a StringIO instance will be created
990 relative_to -- write paths relative to this path;
991 if false then the paths are absolute
992 """
995 # determine output
996 opened_manifest_file = None # name of opened manifest file
997 absolute = not relative_to # whether to output absolute path names as names
998 if isinstance(write, string):
999 opened_manifest_file = write
1000 write = file(write, 'w')
1001 if write is None:
1002 write = StringIO()
1004 # walk the directories, generating manifests
1005 def callback(directory, dirpath, dirnames, filenames):
1007 # absolute paths
1008 filenames = [os.path.join(dirpath, filename)
1009 for filename in filenames]
1010 # ensure new manifest isn't added
1011 filenames = [filename for filename in filenames
1012 if filename != opened_manifest_file]
1013 # normalize paths
1014 if not absolute and relative_to:
1015 filenames = [relpath(filename, relative_to)
1016 for filename in filenames]
1018 # write to manifest
1019 print >> write, '\n'.join(['[%s]' % denormalize_path(filename)
1020 for filename in filenames])
1023 cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore)
1025 if opened_manifest_file:
1026 # close file
1027 write.close()
1028 manifests = [opened_manifest_file]
1029 else:
1030 # manifests/write is a file-like object;
1031 # rewind buffer
1032 write.flush()
1033 write.seek(0)
1034 manifests = [write]
1037 # make a ManifestParser instance
1038 return cls(manifests=manifests)
1040 convert = ManifestParser.from_directories
1043 class TestManifest(ManifestParser):
1044 """
1045 apply logic to manifests; this is your integration layer :)
1046 specific harnesses may subclass from this if they need more logic
1047 """
1049 def filter(self, values, tests):
1050 """
1051 filter on a specific list tag, e.g.:
1052 run-if = os == win linux
1053 skip-if = os == mac
1054 """
1056 # tags:
1057 run_tag = 'run-if'
1058 skip_tag = 'skip-if'
1059 fail_tag = 'fail-if'
1061 # loop over test
1062 for test in tests:
1063 reason = None # reason to disable
1065 # tagged-values to run
1066 if run_tag in test:
1067 condition = test[run_tag]
1068 if not parse(condition, **values):
1069 reason = '%s: %s' % (run_tag, condition)
1071 # tagged-values to skip
1072 if skip_tag in test:
1073 condition = test[skip_tag]
1074 if parse(condition, **values):
1075 reason = '%s: %s' % (skip_tag, condition)
1077 # mark test as disabled if there's a reason
1078 if reason:
1079 test.setdefault('disabled', reason)
1081 # mark test as a fail if so indicated
1082 if fail_tag in test:
1083 condition = test[fail_tag]
1084 if parse(condition, **values):
1085 test['expected'] = 'fail'
1087 def active_tests(self, exists=True, disabled=True, options=None, **values):
1088 """
1089 - exists : return only existing tests
1090 - disabled : whether to return disabled tests
1091 - tags : keys and values to filter on (e.g. `os = linux mac`)
1092 """
1093 tests = [i.copy() for i in self.tests] # shallow copy
1095 # Filter on current subsuite
1096 if options:
1097 if options.subsuite:
1098 tests = [test for test in tests if options.subsuite == test['subsuite']]
1099 else:
1100 tests = [test for test in tests if not test['subsuite']]
1102 # mark all tests as passing unless indicated otherwise
1103 for test in tests:
1104 test['expected'] = test.get('expected', 'pass')
1106 # ignore tests that do not exist
1107 if exists:
1108 tests = [test for test in tests if os.path.exists(test['path'])]
1110 # filter by tags
1111 self.filter(values, tests)
1113 # ignore disabled tests if specified
1114 if not disabled:
1115 tests = [test for test in tests
1116 if not 'disabled' in test]
1118 # return active tests
1119 return tests
1121 def test_paths(self):
1122 return [test['path'] for test in self.active_tests()]
1125 ### command line attributes
1127 class ParserError(Exception):
1128 """error for exceptions while parsing the command line"""
1130 def parse_args(_args):
1131 """
1132 parse and return:
1133 --keys=value (or --key value)
1134 -tags
1135 args
1136 """
1138 # return values
1139 _dict = {}
1140 tags = []
1141 args = []
1143 # parse the arguments
1144 key = None
1145 for arg in _args:
1146 if arg.startswith('---'):
1147 raise ParserError("arguments should start with '-' or '--' only")
1148 elif arg.startswith('--'):
1149 if key:
1150 raise ParserError("Key %s still open" % key)
1151 key = arg[2:]
1152 if '=' in key:
1153 key, value = key.split('=', 1)
1154 _dict[key] = value
1155 key = None
1156 continue
1157 elif arg.startswith('-'):
1158 if key:
1159 raise ParserError("Key %s still open" % key)
1160 tags.append(arg[1:])
1161 continue
1162 else:
1163 if key:
1164 _dict[key] = arg
1165 continue
1166 args.append(arg)
1168 # return values
1169 return (_dict, tags, args)
1172 ### classes for subcommands
1174 class CLICommand(object):
1175 usage = '%prog [options] command'
1176 def __init__(self, parser):
1177 self._parser = parser # master parser
1178 def parser(self):
1179 return OptionParser(usage=self.usage, description=self.__doc__,
1180 add_help_option=False)
1182 class Copy(CLICommand):
1183 usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
1184 def __call__(self, options, args):
1185 # parse the arguments
1186 try:
1187 kwargs, tags, args = parse_args(args)
1188 except ParserError, e:
1189 self._parser.error(e.message)
1191 # make sure we have some manifests, otherwise it will
1192 # be quite boring
1193 if not len(args) == 2:
1194 HelpCLI(self._parser)(options, ['copy'])
1195 return
1197 # read the manifests
1198 # TODO: should probably ensure these exist here
1199 manifests = ManifestParser()
1200 manifests.read(args[0])
1202 # print the resultant query
1203 manifests.copy(args[1], None, *tags, **kwargs)
1206 class CreateCLI(CLICommand):
1207 """
1208 create a manifest from a list of directories
1209 """
1210 usage = '%prog [options] create directory <directory> <...>'
1212 def parser(self):
1213 parser = CLICommand.parser(self)
1214 parser.add_option('-p', '--pattern', dest='pattern',
1215 help="glob pattern for files")
1216 parser.add_option('-i', '--ignore', dest='ignore',
1217 default=[], action='append',
1218 help='directories to ignore')
1219 parser.add_option('-w', '--in-place', dest='in_place',
1220 help='Write .ini files in place; filename to write to')
1221 return parser
1223 def __call__(self, _options, args):
1224 parser = self.parser()
1225 options, args = parser.parse_args(args)
1227 # need some directories
1228 if not len(args):
1229 parser.print_usage()
1230 return
1232 # add the directories to the manifest
1233 for arg in args:
1234 assert os.path.exists(arg)
1235 assert os.path.isdir(arg)
1236 manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
1237 write=options.in_place)
1238 if manifest:
1239 print manifest
1242 class WriteCLI(CLICommand):
1243 """
1244 write a manifest based on a query
1245 """
1246 usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
1247 def __call__(self, options, args):
1249 # parse the arguments
1250 try:
1251 kwargs, tags, args = parse_args(args)
1252 except ParserError, e:
1253 self._parser.error(e.message)
1255 # make sure we have some manifests, otherwise it will
1256 # be quite boring
1257 if not args:
1258 HelpCLI(self._parser)(options, ['write'])
1259 return
1261 # read the manifests
1262 # TODO: should probably ensure these exist here
1263 manifests = ManifestParser()
1264 manifests.read(*args)
1266 # print the resultant query
1267 manifests.write(global_tags=tags, global_kwargs=kwargs)
1270 class HelpCLI(CLICommand):
1271 """
1272 get help on a command
1273 """
1274 usage = '%prog [options] help [command]'
1276 def __call__(self, options, args):
1277 if len(args) == 1 and args[0] in commands:
1278 commands[args[0]](self._parser).parser().print_help()
1279 else:
1280 self._parser.print_help()
1281 print '\nCommands:'
1282 for command in sorted(commands):
1283 print ' %s : %s' % (command, commands[command].__doc__.strip())
1285 class UpdateCLI(CLICommand):
1286 """
1287 update the tests as listed in a manifest from a directory
1288 """
1289 usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
1291 def __call__(self, options, args):
1292 # parse the arguments
1293 try:
1294 kwargs, tags, args = parse_args(args)
1295 except ParserError, e:
1296 self._parser.error(e.message)
1298 # make sure we have some manifests, otherwise it will
1299 # be quite boring
1300 if not len(args) == 2:
1301 HelpCLI(self._parser)(options, ['update'])
1302 return
1304 # read the manifests
1305 # TODO: should probably ensure these exist here
1306 manifests = ManifestParser()
1307 manifests.read(args[0])
1309 # print the resultant query
1310 manifests.update(args[1], None, *tags, **kwargs)
1313 # command -> class mapping
1314 commands = { 'create': CreateCLI,
1315 'help': HelpCLI,
1316 'update': UpdateCLI,
1317 'write': WriteCLI }
1319 def main(args=sys.argv[1:]):
1320 """console_script entry point"""
1322 # set up an option parser
1323 usage = '%prog [options] [command] ...'
1324 description = "%s. Use `help` to display commands" % __doc__.strip()
1325 parser = OptionParser(usage=usage, description=description)
1326 parser.add_option('-s', '--strict', dest='strict',
1327 action='store_true', default=False,
1328 help='adhere strictly to errors')
1329 parser.disable_interspersed_args()
1331 options, args = parser.parse_args(args)
1333 if not args:
1334 HelpCLI(parser)(options, args)
1335 parser.exit()
1337 # get the command
1338 command = args[0]
1339 if command not in commands:
1340 parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
1342 handler = commands[command](parser)
1343 handler(options, args[1:])
1345 if __name__ == '__main__':
1346 main()
