The Tor Browser / file comparison

comparison: testing/mozbase/manifestdestiny/manifestparser/manifestparser.py

testing/mozbase/manifestdestiny/manifestparser/manifestparser.py

changeset 0
6474c204b198
equal deleted inserted replaced
-1:000000000000 0:3133f160ea4f
1 #!/usr/bin/env python
2
3 # This Source Code Form is subject to the terms of the Mozilla Public
4 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
5 # You can obtain one at http://mozilla.org/MPL/2.0/.
6
7 """
8 Mozilla universal manifest parser
9 """
10
11 __all__ = ['read_ini', # .ini reader
12 'ManifestParser', 'TestManifest', 'convert', # manifest handling
13 'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
14
15 import fnmatch
16 import os
17 import re
18 import shutil
19 import sys
20
21 from optparse import OptionParser
22 from StringIO import StringIO
23
24 relpath = os.path.relpath
25 string = (basestring,)
26
27
28 # expr.py
29 # from:
30 # http://k0s.org/mozilla/hg/expressionparser
31 # http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
32
33 # Implements a top-down parser/evaluator for simple boolean expressions.
34 # ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
35 #
36 # Rough grammar:
37 # expr := literal
38 # | '(' expr ')'
39 # | expr '&&' expr
40 # | expr '||' expr
41 # | expr '==' expr
42 # | expr '!=' expr
43 # literal := BOOL
44 # | INT
45 # | STRING
46 # | IDENT
47 # BOOL := true|false
48 # INT := [0-9]+
49 # STRING := "[^"]*"
50 # IDENT := [A-Za-z_]\w*
51
52 # Identifiers take their values from a mapping dictionary passed as the second
53 # argument.
54
55 # Glossary (see above URL for details):
56 # - nud: null denotation
57 # - led: left detonation
58 # - lbp: left binding power
59 # - rbp: right binding power
60
61 class ident_token(object):
62 def __init__(self, value):
63 self.value = value
64 def nud(self, parser):
65 # identifiers take their value from the value mappings passed
66 # to the parser
67 return parser.value(self.value)
68
69 class literal_token(object):
70 def __init__(self, value):
71 self.value = value
72 def nud(self, parser):
73 return self.value
74
75 class eq_op_token(object):
76 "=="
77 def led(self, parser, left):
78 return left == parser.expression(self.lbp)
79
80 class neq_op_token(object):
81 "!="
82 def led(self, parser, left):
83 return left != parser.expression(self.lbp)
84
85 class not_op_token(object):
86 "!"
87 def nud(self, parser):
88 return not parser.expression(100)
89
90 class and_op_token(object):
91 "&&"
92 def led(self, parser, left):
93 right = parser.expression(self.lbp)
94 return left and right
95
96 class or_op_token(object):
97 "||"
98 def led(self, parser, left):
99 right = parser.expression(self.lbp)
100 return left or right
101
102 class lparen_token(object):
103 "("
104 def nud(self, parser):
105 expr = parser.expression()
106 parser.advance(rparen_token)
107 return expr
108
109 class rparen_token(object):
110 ")"
111
112 class end_token(object):
113 """always ends parsing"""
114
115 ### derived literal tokens
116
117 class bool_token(literal_token):
118 def __init__(self, value):
119 value = {'true':True, 'false':False}[value]
120 literal_token.__init__(self, value)
121
122 class int_token(literal_token):
123 def __init__(self, value):
124 literal_token.__init__(self, int(value))
125
126 class string_token(literal_token):
127 def __init__(self, value):
128 literal_token.__init__(self, value[1:-1])
129
130 precedence = [(end_token, rparen_token),
131 (or_op_token,),
132 (and_op_token,),
133 (eq_op_token, neq_op_token),
134 (lparen_token,),
135 ]
136 for index, rank in enumerate(precedence):
137 for token in rank:
138 token.lbp = index # lbp = lowest left binding power
139
140 class ParseError(Exception):
141 """error parsing conditional expression"""
142
143 class ExpressionParser(object):
144 """
145 A parser for a simple expression language.
146
147 The expression language can be described as follows::
148
149 EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION
150 OP ::= '==' | '!=' | '&&' | '||'
151 LITERAL ::= BOOL | INT | IDENT | STRING
152 BOOL ::= 'true' | 'false'
153 INT ::= [0-9]+
154 IDENT ::= [a-zA-Z_]\w*
155 STRING ::= '"' [^\"] '"' | ''' [^\'] '''
156
157 At its core, expressions consist of booleans, integers, identifiers and.
158 strings. Booleans are one of *true* or *false*. Integers are a series
159 of digits. Identifiers are a series of English letters and underscores.
160 Strings are a pair of matching quote characters (single or double) with
161 zero or more characters inside.
162
163 Expressions can be combined with operators: the equals (==) and not
164 equals (!=) operators compare two expressions and produce a boolean. The
165 and (&&) and or (||) operators take two expressions and produce the logical
166 AND or OR value of them, respectively. An expression can also be prefixed
167 with the not (!) operator, which produces its logical negation.
168
169 Finally, any expression may be contained within parentheses for grouping.
170
171 Identifiers take their values from the mapping provided.
172 """
173 def __init__(self, text, valuemapping, strict=False):
174 """
175 Initialize the parser
176 :param text: The expression to parse as a string.
177 :param valuemapping: A dict mapping identifier names to values.
178 :param strict: If true, referencing an identifier that was not
179 provided in :valuemapping: will raise an error.
180 """
181 self.text = text
182 self.valuemapping = valuemapping
183 self.strict = strict
184
185 def _tokenize(self):
186 """
187 Lex the input text into tokens and yield them in sequence.
188 """
189 # scanner callbacks
190 def bool_(scanner, t): return bool_token(t)
191 def identifier(scanner, t): return ident_token(t)
192 def integer(scanner, t): return int_token(t)
193 def eq(scanner, t): return eq_op_token()
194 def neq(scanner, t): return neq_op_token()
195 def or_(scanner, t): return or_op_token()
196 def and_(scanner, t): return and_op_token()
197 def lparen(scanner, t): return lparen_token()
198 def rparen(scanner, t): return rparen_token()
199 def string_(scanner, t): return string_token(t)
200 def not_(scanner, t): return not_op_token()
201
202 scanner = re.Scanner([
203 # Note: keep these in sync with the class docstring above.
204 (r"true|false", bool_),
205 (r"[a-zA-Z_]\w*", identifier),
206 (r"[0-9]+", integer),
207 (r'("[^"]*")|(\'[^\']*\')', string_),
208 (r"==", eq),
209 (r"!=", neq),
210 (r"\|\|", or_),
211 (r"!", not_),
212 (r"&&", and_),
213 (r"\(", lparen),
214 (r"\)", rparen),
215 (r"\s+", None), # skip whitespace
216 ])
217 tokens, remainder = scanner.scan(self.text)
218 for t in tokens:
219 yield t
220 yield end_token()
221
222 def value(self, ident):
223 """
224 Look up the value of |ident| in the value mapping passed in the
225 constructor.
226 """
227 if self.strict:
228 return self.valuemapping[ident]
229 else:
230 return self.valuemapping.get(ident, None)
231
232 def advance(self, expected):
233 """
234 Assert that the next token is an instance of |expected|, and advance
235 to the next token.
236 """
237 if not isinstance(self.token, expected):
238 raise Exception, "Unexpected token!"
239 self.token = self.iter.next()
240
241 def expression(self, rbp=0):
242 """
243 Parse and return the value of an expression until a token with
244 right binding power greater than rbp is encountered.
245 """
246 t = self.token
247 self.token = self.iter.next()
248 left = t.nud(self)
249 while rbp < self.token.lbp:
250 t = self.token
251 self.token = self.iter.next()
252 left = t.led(self, left)
253 return left
254
255 def parse(self):
256 """
257 Parse and return the value of the expression in the text
258 passed to the constructor. Raises a ParseError if the expression
259 could not be parsed.
260 """
261 try:
262 self.iter = self._tokenize()
263 self.token = self.iter.next()
264 return self.expression()
265 except:
266 raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
267
268 __call__ = parse
269
270 def parse(text, **values):
271 """
272 Parse and evaluate a boolean expression.
273 :param text: The expression to parse, as a string.
274 :param values: A dict containing a name to value mapping for identifiers
275 referenced in *text*.
276 :rtype: the final value of the expression.
277 :raises: :py:exc::ParseError: will be raised if parsing fails.
278 """
279 return ExpressionParser(text, values).parse()
280
281
282 ### path normalization
283
284 def normalize_path(path):
285 """normalize a relative path"""
286 if sys.platform.startswith('win'):
287 return path.replace('/', os.path.sep)
288 return path
289
290 def denormalize_path(path):
291 """denormalize a relative path"""
292 if sys.platform.startswith('win'):
293 return path.replace(os.path.sep, '/')
294 return path
295
296
297 ### .ini reader
298
299 def read_ini(fp, variables=None, default='DEFAULT',
300 comments=';#', separators=('=', ':'),
301 strict=True):
302 """
303 read an .ini file and return a list of [(section, values)]
304 - fp : file pointer or path to read
305 - variables : default set of variables
306 - default : name of the section for the default section
307 - comments : characters that if they start a line denote a comment
308 - separators : strings that denote key, value separation in order
309 - strict : whether to be strict about parsing
310 """
311
312 # variables
313 variables = variables or {}
314 sections = []
315 key = value = None
316 section_names = set()
317 if isinstance(fp, basestring):
318 fp = file(fp)
319
320 # read the lines
321 for (linenum, line) in enumerate(fp.readlines(), start=1):
322
323 stripped = line.strip()
324
325 # ignore blank lines
326 if not stripped:
327 # reset key and value to avoid continuation lines
328 key = value = None
329 continue
330
331 # ignore comment lines
332 if stripped[0] in comments:
333 continue
334
335 # check for a new section
336 if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
337 section = stripped[1:-1].strip()
338 key = value = None
339
340 # deal with DEFAULT section
341 if section.lower() == default.lower():
342 if strict:
343 assert default not in section_names
344 section_names.add(default)
345 current_section = variables
346 continue
347
348 if strict:
349 # make sure this section doesn't already exist
350 assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names)
351
352 section_names.add(section)
353 current_section = {}
354 sections.append((section, current_section))
355 continue
356
357 # if there aren't any sections yet, something bad happen
358 if not section_names:
359 raise Exception('No sections found')
360
361 # (key, value) pair
362 for separator in separators:
363 if separator in stripped:
364 key, value = stripped.split(separator, 1)
365 key = key.strip()
366 value = value.strip()
367
368 if strict:
369 # make sure this key isn't already in the section or empty
370 assert key
371 if current_section is not variables:
372 assert key not in current_section
373
374 current_section[key] = value
375 break
376 else:
377 # continuation line ?
378 if line[0].isspace() and key:
379 value = '%s%s%s' % (value, os.linesep, stripped)
380 current_section[key] = value
381 else:
382 # something bad happened!
383 if hasattr(fp, 'name'):
384 filename = fp.name
385 else:
386 filename = 'unknown'
387 raise Exception("Error parsing manifest file '%s', line %s" %
388 (filename, linenum))
389
390 # interpret the variables
391 def interpret_variables(global_dict, local_dict):
392 variables = global_dict.copy()
393 if 'skip-if' in local_dict and 'skip-if' in variables:
394 local_dict['skip-if'] = "(%s) || (%s)" % (variables['skip-if'].split('#')[0], local_dict['skip-if'].split('#')[0])
395 variables.update(local_dict)
396
397 return variables
398
399 sections = [(i, interpret_variables(variables, j)) for i, j in sections]
400 return sections
401
402
403 ### objects for parsing manifests
404
405 class ManifestParser(object):
406 """read .ini manifests"""
407
408 def __init__(self, manifests=(), defaults=None, strict=True):
409 self._defaults = defaults or {}
410 self.tests = []
411 self.manifest_defaults = {}
412 self.strict = strict
413 self.rootdir = None
414 self.relativeRoot = None
415 if manifests:
416 self.read(*manifests)
417
418 def getRelativeRoot(self, root):
419 return root
420
421 ### methods for reading manifests
422
423 def _read(self, root, filename, defaults):
424
425 # get directory of this file if not file-like object
426 if isinstance(filename, string):
427 filename = os.path.abspath(filename)
428 fp = open(filename)
429 here = os.path.dirname(filename)
430 else:
431 fp = filename
432 filename = here = None
433 defaults['here'] = here
434
435 # Rootdir is needed for relative path calculation. Precompute it for
436 # the microoptimization used below.
437 if self.rootdir is None:
438 rootdir = ""
439 else:
440 assert os.path.isabs(self.rootdir)
441 rootdir = self.rootdir + os.path.sep
442
443 # read the configuration
444 sections = read_ini(fp=fp, variables=defaults, strict=self.strict)
445 self.manifest_defaults[filename] = defaults
446
447 # get the tests
448 for section, data in sections:
449 subsuite = ''
450 if 'subsuite' in data:
451 subsuite = data['subsuite']
452
453 # a file to include
454 # TODO: keep track of included file structure:
455 # self.manifests = {'manifest.ini': 'relative/path.ini'}
456 if section.startswith('include:'):
457 include_file = section.split('include:', 1)[-1]
458 include_file = normalize_path(include_file)
459 if not os.path.isabs(include_file):
460 include_file = os.path.join(self.getRelativeRoot(here), include_file)
461 if not os.path.exists(include_file):
462 message = "Included file '%s' does not exist" % include_file
463 if self.strict:
464 raise IOError(message)
465 else:
466 sys.stderr.write("%s\n" % message)
467 continue
468 include_defaults = data.copy()
469 self._read(root, include_file, include_defaults)
470 continue
471
472 # otherwise an item
473 test = data
474 test['name'] = section
475
476 # Will be None if the manifest being read is a file-like object.
477 test['manifest'] = filename
478
479 # determine the path
480 path = test.get('path', section)
481 _relpath = path
482 if '://' not in path: # don't futz with URLs
483 path = normalize_path(path)
484 if here and not os.path.isabs(path):
485 path = os.path.normpath(os.path.join(here, path))
486
487 # Microoptimization, because relpath is quite expensive.
488 # We know that rootdir is an absolute path or empty. If path
489 # starts with rootdir, then path is also absolute and the tail
490 # of the path is the relative path (possibly non-normalized,
491 # when here is unknown).
492 # For this to work rootdir needs to be terminated with a path
493 # separator, so that references to sibling directories with
494 # a common prefix don't get misscomputed (e.g. /root and
495 # /rootbeer/file).
496 # When the rootdir is unknown, the relpath needs to be left
497 # unchanged. We use an empty string as rootdir in that case,
498 # which leaves relpath unchanged after slicing.
499 if path.startswith(rootdir):
500 _relpath = path[len(rootdir):]
501 else:
502 _relpath = relpath(path, rootdir)
503
504 test['subsuite'] = subsuite
505 test['path'] = path
506 test['relpath'] = _relpath
507
508 # append the item
509 self.tests.append(test)
510
511 def read(self, *filenames, **defaults):
512 """
513 read and add manifests from file paths or file-like objects
514
515 filenames -- file paths or file-like objects to read as manifests
516 defaults -- default variables
517 """
518
519 # ensure all files exist
520 missing = [filename for filename in filenames
521 if isinstance(filename, string) and not os.path.exists(filename) ]
522 if missing:
523 raise IOError('Missing files: %s' % ', '.join(missing))
524
525 # default variables
526 _defaults = defaults.copy() or self._defaults.copy()
527 _defaults.setdefault('here', None)
528
529 # process each file
530 for filename in filenames:
531 # set the per file defaults
532 defaults = _defaults.copy()
533 here = None
534 if isinstance(filename, string):
535 here = os.path.dirname(os.path.abspath(filename))
536 defaults['here'] = here # directory of master .ini file
537
538 if self.rootdir is None:
539 # set the root directory
540 # == the directory of the first manifest given
541 self.rootdir = here
542
543 self._read(here, filename, defaults)
544
545
546 ### methods for querying manifests
547
548 def query(self, *checks, **kw):
549 """
550 general query function for tests
551 - checks : callable conditions to test if the test fulfills the query
552 """
553 tests = kw.get('tests', None)
554 if tests is None:
555 tests = self.tests
556 retval = []
557 for test in tests:
558 for check in checks:
559 if not check(test):
560 break
561 else:
562 retval.append(test)
563 return retval
564
565 def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs):
566 # TODO: pass a dict instead of kwargs since you might hav
567 # e.g. 'inverse' as a key in the dict
568
569 # TODO: tags should just be part of kwargs with None values
570 # (None == any is kinda weird, but probably still better)
571
572 # fix up tags
573 if tags:
574 tags = set(tags)
575 else:
576 tags = set()
577
578 # make some check functions
579 if inverse:
580 has_tags = lambda test: not tags.intersection(test.keys())
581 def dict_query(test):
582 for key, value in kwargs.items():
583 if test.get(key) == value:
584 return False
585 return True
586 else:
587 has_tags = lambda test: tags.issubset(test.keys())
588 def dict_query(test):
589 for key, value in kwargs.items():
590 if test.get(key) != value:
591 return False
592 return True
593
594 # query the tests
595 tests = self.query(has_tags, dict_query, tests=tests)
596
597 # if a key is given, return only a list of that key
598 # useful for keys like 'name' or 'path'
599 if _key:
600 return [test[_key] for test in tests]
601
602 # return the tests
603 return tests
604
605 def manifests(self, tests=None):
606 """
607 return manifests in order in which they appear in the tests
608 """
609 if tests is None:
610 # Make sure to return all the manifests, even ones without tests.
611 return self.manifest_defaults.keys()
612
613 manifests = []
614 for test in tests:
615 manifest = test.get('manifest')
616 if not manifest:
617 continue
618 if manifest not in manifests:
619 manifests.append(manifest)
620 return manifests
621
622 def paths(self):
623 return [i['path'] for i in self.tests]
624
625
626 ### methods for auditing
627
628 def missing(self, tests=None):
629 """return list of tests that do not exist on the filesystem"""
630 if tests is None:
631 tests = self.tests
632 return [test for test in tests
633 if not os.path.exists(test['path'])]
634
635 def verifyDirectory(self, directories, pattern=None, extensions=None):
636 """
637 checks what is on the filesystem vs what is in a manifest
638 returns a 2-tuple of sets:
639 (missing_from_filesystem, missing_from_manifest)
640 """
641
642 files = set([])
643 if isinstance(directories, basestring):
644 directories = [directories]
645
646 # get files in directories
647 for directory in directories:
648 for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
649
650 # only add files that match a pattern
651 if pattern:
652 filenames = fnmatch.filter(filenames, pattern)
653
654 # only add files that have one of the extensions
655 if extensions:
656 filenames = [filename for filename in filenames
657 if os.path.splitext(filename)[-1] in extensions]
658
659 files.update([os.path.join(dirpath, filename) for filename in filenames])
660
661 paths = set(self.paths())
662 missing_from_filesystem = paths.difference(files)
663 missing_from_manifest = files.difference(paths)
664 return (missing_from_filesystem, missing_from_manifest)
665
666
667 ### methods for output
668
669 def write(self, fp=sys.stdout, rootdir=None,
670 global_tags=None, global_kwargs=None,
671 local_tags=None, local_kwargs=None):
672 """
673 write a manifest given a query
674 global and local options will be munged to do the query
675 globals will be written to the top of the file
676 locals (if given) will be written per test
677 """
678
679 # open file if `fp` given as string
680 close = False
681 if isinstance(fp, string):
682 fp = file(fp, 'w')
683 close = True
684
685 # root directory
686 if rootdir is None:
687 rootdir = self.rootdir
688
689 # sanitize input
690 global_tags = global_tags or set()
691 local_tags = local_tags or set()
692 global_kwargs = global_kwargs or {}
693 local_kwargs = local_kwargs or {}
694
695 # create the query
696 tags = set([])
697 tags.update(global_tags)
698 tags.update(local_tags)
699 kwargs = {}
700 kwargs.update(global_kwargs)
701 kwargs.update(local_kwargs)
702
703 # get matching tests
704 tests = self.get(tags=tags, **kwargs)
705
706 # print the .ini manifest
707 if global_tags or global_kwargs:
708 print >> fp, '[DEFAULT]'
709 for tag in global_tags:
710 print >> fp, '%s =' % tag
711 for key, value in global_kwargs.items():
712 print >> fp, '%s = %s' % (key, value)
713 print >> fp
714
715 for test in tests:
716 test = test.copy() # don't overwrite
717
718 path = test['name']
719 if not os.path.isabs(path):
720 path = test['path']
721 if self.rootdir:
722 path = relpath(test['path'], self.rootdir)
723 path = denormalize_path(path)
724 print >> fp, '[%s]' % path
725
726 # reserved keywords:
727 reserved = ['path', 'name', 'here', 'manifest', 'relpath']
728 for key in sorted(test.keys()):
729 if key in reserved:
730 continue
731 if key in global_kwargs:
732 continue
733 if key in global_tags and not test[key]:
734 continue
735 print >> fp, '%s = %s' % (key, test[key])
736 print >> fp
737
738 if close:
739 # close the created file
740 fp.close()
741
742 def __str__(self):
743 fp = StringIO()
744 self.write(fp=fp)
745 value = fp.getvalue()
746 return value
747
748 def copy(self, directory, rootdir=None, *tags, **kwargs):
749 """
750 copy the manifests and associated tests
751 - directory : directory to copy to
752 - rootdir : root directory to copy to (if not given from manifests)
753 - tags : keywords the tests must have
754 - kwargs : key, values the tests must match
755 """
756 # XXX note that copy does *not* filter the tests out of the
757 # resulting manifest; it just stupidly copies them over.
758 # ideally, it would reread the manifests and filter out the
759 # tests that don't match *tags and **kwargs
760
761 # destination
762 if not os.path.exists(directory):
763 os.path.makedirs(directory)
764 else:
765 # sanity check
766 assert os.path.isdir(directory)
767
768 # tests to copy
769 tests = self.get(tags=tags, **kwargs)
770 if not tests:
771 return # nothing to do!
772
773 # root directory
774 if rootdir is None:
775 rootdir = self.rootdir
776
777 # copy the manifests + tests
778 manifests = [relpath(manifest, rootdir) for manifest in self.manifests()]
779 for manifest in manifests:
780 destination = os.path.join(directory, manifest)
781 dirname = os.path.dirname(destination)
782 if not os.path.exists(dirname):
783 os.makedirs(dirname)
784 else:
785 # sanity check
786 assert os.path.isdir(dirname)
787 shutil.copy(os.path.join(rootdir, manifest), destination)
788 for test in tests:
789 if os.path.isabs(test['name']):
790 continue
791 source = test['path']
792 if not os.path.exists(source):
793 print >> sys.stderr, "Missing test: '%s' does not exist!" % source
794 continue
795 # TODO: should err on strict
796 destination = os.path.join(directory, relpath(test['path'], rootdir))
797 shutil.copy(source, destination)
798 # TODO: ensure that all of the tests are below the from_dir
799
800 def update(self, from_dir, rootdir=None, *tags, **kwargs):
801 """
802 update the tests as listed in a manifest from a directory
803 - from_dir : directory where the tests live
804 - rootdir : root directory to copy to (if not given from manifests)
805 - tags : keys the tests must have
806 - kwargs : key, values the tests must match
807 """
808
809 # get the tests
810 tests = self.get(tags=tags, **kwargs)
811
812 # get the root directory
813 if not rootdir:
814 rootdir = self.rootdir
815
816 # copy them!
817 for test in tests:
818 if not os.path.isabs(test['name']):
819 _relpath = relpath(test['path'], rootdir)
820 source = os.path.join(from_dir, _relpath)
821 if not os.path.exists(source):
822 # TODO err on strict
823 print >> sys.stderr, "Missing test: '%s'; skipping" % test['name']
824 continue
825 destination = os.path.join(rootdir, _relpath)
826 shutil.copy(source, destination)
827
828 ### directory importers
829
830 @classmethod
831 def _walk_directories(cls, directories, function, pattern=None, ignore=()):
832 """
833 internal function to import directories
834 """
835
836 class FilteredDirectoryContents(object):
837 """class to filter directory contents"""
838
839 sort = sorted
840
841 def __init__(self, pattern=pattern, ignore=ignore, cache=None):
842 if pattern is None:
843 pattern = set()
844 if isinstance(pattern, basestring):
845 pattern = [pattern]
846 self.patterns = pattern
847 self.ignore = set(ignore)
848
849 # cache of (dirnames, filenames) keyed on directory real path
850 # assumes volume is frozen throughout scope
851 self._cache = cache or {}
852
853 def __call__(self, directory):
854 """returns 2-tuple: dirnames, filenames"""
855 directory = os.path.realpath(directory)
856 if directory not in self._cache:
857 dirnames, filenames = self.contents(directory)
858
859 # filter out directories without progeny
860 # XXX recursive: should keep track of seen directories
861 dirnames = [ dirname for dirname in dirnames
862 if not self.empty(os.path.join(directory, dirname)) ]
863
864 self._cache[directory] = (tuple(dirnames), filenames)
865
866 # return cached values
867 return self._cache[directory]
868
869 def empty(self, directory):
870 """
871 returns if a directory and its descendents are empty
872 """
873 return self(directory) == ((), ())
874
875 def contents(self, directory, sort=None):
876 """
877 return directory contents as (dirnames, filenames)
878 with `ignore` and `pattern` applied
879 """
880
881 if sort is None:
882 sort = self.sort
883
884 # split directories and files
885 dirnames = []
886 filenames = []
887 for item in os.listdir(directory):
888 path = os.path.join(directory, item)
889 if os.path.isdir(path):
890 dirnames.append(item)
891 else:
892 # XXX not sure what to do if neither a file or directory
893 # (if anything)
894 assert os.path.isfile(path)
895 filenames.append(item)
896
897 # filter contents;
898 # this could be done in situ re the above for loop
899 # but it is really disparate in intent
900 # and could conceivably go to a separate method
901 dirnames = [dirname for dirname in dirnames
902 if dirname not in self.ignore]
903 filenames = set(filenames)
904 # we use set functionality to filter filenames
905 if self.patterns:
906 matches = set()
907 matches.update(*[fnmatch.filter(filenames, pattern)
908 for pattern in self.patterns])
909 filenames = matches
910
911 if sort is not None:
912 # sort dirnames, filenames
913 dirnames = sort(dirnames)
914 filenames = sort(filenames)
915
916 return (tuple(dirnames), tuple(filenames))
917
918 # make a filtered directory object
919 directory_contents = FilteredDirectoryContents(pattern=pattern, ignore=ignore)
920
921 # walk the directories, generating manifests
922 for index, directory in enumerate(directories):
923
924 for dirpath, dirnames, filenames in os.walk(directory):
925
926 # get the directory contents from the caching object
927 _dirnames, filenames = directory_contents(dirpath)
928 # filter out directory names
929 dirnames[:] = _dirnames
930
931 # call callback function
932 function(directory, dirpath, dirnames, filenames)
933
934 @classmethod
935 def populate_directory_manifests(cls, directories, filename, pattern=None, ignore=(), overwrite=False):
936 """
937 walks directories and writes manifests of name `filename` in-place; returns `cls` instance populated
938 with the given manifests
939
940 filename -- filename of manifests to write
941 pattern -- shell pattern (glob) or patterns of filenames to match
942 ignore -- directory names to ignore
943 overwrite -- whether to overwrite existing files of given name
944 """
945
946 manifest_dict = {}
947 seen = [] # top-level directories seen
948
949 if os.path.basename(filename) != filename:
950 raise IOError("filename should not include directory name")
951
952 # no need to hit directories more than once
953 _directories = directories
954 directories = []
955 for directory in _directories:
956 if directory not in directories:
957 directories.append(directory)
958
959 def callback(directory, dirpath, dirnames, filenames):
960 """write a manifest for each directory"""
961
962 manifest_path = os.path.join(dirpath, filename)
963 if (dirnames or filenames) and not (os.path.exists(manifest_path) and overwrite):
964 with file(manifest_path, 'w') as manifest:
965 for dirname in dirnames:
966 print >> manifest, '[include:%s]' % os.path.join(dirname, filename)
967 for _filename in filenames:
968 print >> manifest, '[%s]' % _filename
969
970 # add to list of manifests
971 manifest_dict.setdefault(directory, manifest_path)
972
973 # walk the directories to gather files
974 cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore)
975 # get manifests
976 manifests = [manifest_dict[directory] for directory in _directories]
977
978 # create a `cls` instance with the manifests
979 return cls(manifests=manifests)
980
981 @classmethod
982 def from_directories(cls, directories, pattern=None, ignore=(), write=None, relative_to=None):
983 """
984 convert directories to a simple manifest; returns ManifestParser instance
985
986 pattern -- shell pattern (glob) or patterns of filenames to match
987 ignore -- directory names to ignore
988 write -- filename or file-like object of manifests to write;
989 if `None` then a StringIO instance will be created
990 relative_to -- write paths relative to this path;
991 if false then the paths are absolute
992 """
993
994
995 # determine output
996 opened_manifest_file = None # name of opened manifest file
997 absolute = not relative_to # whether to output absolute path names as names
998 if isinstance(write, string):
999 opened_manifest_file = write
1000 write = file(write, 'w')
1001 if write is None:
1002 write = StringIO()
1003
1004 # walk the directories, generating manifests
1005 def callback(directory, dirpath, dirnames, filenames):
1006
1007 # absolute paths
1008 filenames = [os.path.join(dirpath, filename)
1009 for filename in filenames]
1010 # ensure new manifest isn't added
1011 filenames = [filename for filename in filenames
1012 if filename != opened_manifest_file]
1013 # normalize paths
1014 if not absolute and relative_to:
1015 filenames = [relpath(filename, relative_to)
1016 for filename in filenames]
1017
1018 # write to manifest
1019 print >> write, '\n'.join(['[%s]' % denormalize_path(filename)
1020 for filename in filenames])
1021
1022
1023 cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore)
1024
1025 if opened_manifest_file:
1026 # close file
1027 write.close()
1028 manifests = [opened_manifest_file]
1029 else:
1030 # manifests/write is a file-like object;
1031 # rewind buffer
1032 write.flush()
1033 write.seek(0)
1034 manifests = [write]
1035
1036
1037 # make a ManifestParser instance
1038 return cls(manifests=manifests)
1039
1040 convert = ManifestParser.from_directories
1041
1042
1043 class TestManifest(ManifestParser):
1044 """
1045 apply logic to manifests; this is your integration layer :)
1046 specific harnesses may subclass from this if they need more logic
1047 """
1048
1049 def filter(self, values, tests):
1050 """
1051 filter on a specific list tag, e.g.:
1052 run-if = os == win linux
1053 skip-if = os == mac
1054 """
1055
1056 # tags:
1057 run_tag = 'run-if'
1058 skip_tag = 'skip-if'
1059 fail_tag = 'fail-if'
1060
1061 # loop over test
1062 for test in tests:
1063 reason = None # reason to disable
1064
1065 # tagged-values to run
1066 if run_tag in test:
1067 condition = test[run_tag]
1068 if not parse(condition, **values):
1069 reason = '%s: %s' % (run_tag, condition)
1070
1071 # tagged-values to skip
1072 if skip_tag in test:
1073 condition = test[skip_tag]
1074 if parse(condition, **values):
1075 reason = '%s: %s' % (skip_tag, condition)
1076
1077 # mark test as disabled if there's a reason
1078 if reason:
1079 test.setdefault('disabled', reason)
1080
1081 # mark test as a fail if so indicated
1082 if fail_tag in test:
1083 condition = test[fail_tag]
1084 if parse(condition, **values):
1085 test['expected'] = 'fail'
1086
1087 def active_tests(self, exists=True, disabled=True, options=None, **values):
1088 """
1089 - exists : return only existing tests
1090 - disabled : whether to return disabled tests
1091 - tags : keys and values to filter on (e.g. `os = linux mac`)
1092 """
1093 tests = [i.copy() for i in self.tests] # shallow copy
1094
1095 # Filter on current subsuite
1096 if options:
1097 if options.subsuite:
1098 tests = [test for test in tests if options.subsuite == test['subsuite']]
1099 else:
1100 tests = [test for test in tests if not test['subsuite']]
1101
1102 # mark all tests as passing unless indicated otherwise
1103 for test in tests:
1104 test['expected'] = test.get('expected', 'pass')
1105
1106 # ignore tests that do not exist
1107 if exists:
1108 tests = [test for test in tests if os.path.exists(test['path'])]
1109
1110 # filter by tags
1111 self.filter(values, tests)
1112
1113 # ignore disabled tests if specified
1114 if not disabled:
1115 tests = [test for test in tests
1116 if not 'disabled' in test]
1117
1118 # return active tests
1119 return tests
1120
1121 def test_paths(self):
1122 return [test['path'] for test in self.active_tests()]
1123
1124
1125 ### command line attributes
1126
1127 class ParserError(Exception):
1128 """error for exceptions while parsing the command line"""
1129
1130 def parse_args(_args):
1131 """
1132 parse and return:
1133 --keys=value (or --key value)
1134 -tags
1135 args
1136 """
1137
1138 # return values
1139 _dict = {}
1140 tags = []
1141 args = []
1142
1143 # parse the arguments
1144 key = None
1145 for arg in _args:
1146 if arg.startswith('---'):
1147 raise ParserError("arguments should start with '-' or '--' only")
1148 elif arg.startswith('--'):
1149 if key:
1150 raise ParserError("Key %s still open" % key)
1151 key = arg[2:]
1152 if '=' in key:
1153 key, value = key.split('=', 1)
1154 _dict[key] = value
1155 key = None
1156 continue
1157 elif arg.startswith('-'):
1158 if key:
1159 raise ParserError("Key %s still open" % key)
1160 tags.append(arg[1:])
1161 continue
1162 else:
1163 if key:
1164 _dict[key] = arg
1165 continue
1166 args.append(arg)
1167
1168 # return values
1169 return (_dict, tags, args)
1170
1171
1172 ### classes for subcommands
1173
1174 class CLICommand(object):
1175 usage = '%prog [options] command'
1176 def __init__(self, parser):
1177 self._parser = parser # master parser
1178 def parser(self):
1179 return OptionParser(usage=self.usage, description=self.__doc__,
1180 add_help_option=False)
1181
1182 class Copy(CLICommand):
1183 usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
1184 def __call__(self, options, args):
1185 # parse the arguments
1186 try:
1187 kwargs, tags, args = parse_args(args)
1188 except ParserError, e:
1189 self._parser.error(e.message)
1190
1191 # make sure we have some manifests, otherwise it will
1192 # be quite boring
1193 if not len(args) == 2:
1194 HelpCLI(self._parser)(options, ['copy'])
1195 return
1196
1197 # read the manifests
1198 # TODO: should probably ensure these exist here
1199 manifests = ManifestParser()
1200 manifests.read(args[0])
1201
1202 # print the resultant query
1203 manifests.copy(args[1], None, *tags, **kwargs)
1204
1205
1206 class CreateCLI(CLICommand):
1207 """
1208 create a manifest from a list of directories
1209 """
1210 usage = '%prog [options] create directory <directory> <...>'
1211
1212 def parser(self):
1213 parser = CLICommand.parser(self)
1214 parser.add_option('-p', '--pattern', dest='pattern',
1215 help="glob pattern for files")
1216 parser.add_option('-i', '--ignore', dest='ignore',
1217 default=[], action='append',
1218 help='directories to ignore')
1219 parser.add_option('-w', '--in-place', dest='in_place',
1220 help='Write .ini files in place; filename to write to')
1221 return parser
1222
1223 def __call__(self, _options, args):
1224 parser = self.parser()
1225 options, args = parser.parse_args(args)
1226
1227 # need some directories
1228 if not len(args):
1229 parser.print_usage()
1230 return
1231
1232 # add the directories to the manifest
1233 for arg in args:
1234 assert os.path.exists(arg)
1235 assert os.path.isdir(arg)
1236 manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
1237 write=options.in_place)
1238 if manifest:
1239 print manifest
1240
1241
1242 class WriteCLI(CLICommand):
1243 """
1244 write a manifest based on a query
1245 """
1246 usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
1247 def __call__(self, options, args):
1248
1249 # parse the arguments
1250 try:
1251 kwargs, tags, args = parse_args(args)
1252 except ParserError, e:
1253 self._parser.error(e.message)
1254
1255 # make sure we have some manifests, otherwise it will
1256 # be quite boring
1257 if not args:
1258 HelpCLI(self._parser)(options, ['write'])
1259 return
1260
1261 # read the manifests
1262 # TODO: should probably ensure these exist here
1263 manifests = ManifestParser()
1264 manifests.read(*args)
1265
1266 # print the resultant query
1267 manifests.write(global_tags=tags, global_kwargs=kwargs)
1268
1269
1270 class HelpCLI(CLICommand):
1271 """
1272 get help on a command
1273 """
1274 usage = '%prog [options] help [command]'
1275
1276 def __call__(self, options, args):
1277 if len(args) == 1 and args[0] in commands:
1278 commands[args[0]](self._parser).parser().print_help()
1279 else:
1280 self._parser.print_help()
1281 print '\nCommands:'
1282 for command in sorted(commands):
1283 print ' %s : %s' % (command, commands[command].__doc__.strip())
1284
1285 class UpdateCLI(CLICommand):
1286 """
1287 update the tests as listed in a manifest from a directory
1288 """
1289 usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
1290
1291 def __call__(self, options, args):
1292 # parse the arguments
1293 try:
1294 kwargs, tags, args = parse_args(args)
1295 except ParserError, e:
1296 self._parser.error(e.message)
1297
1298 # make sure we have some manifests, otherwise it will
1299 # be quite boring
1300 if not len(args) == 2:
1301 HelpCLI(self._parser)(options, ['update'])
1302 return
1303
1304 # read the manifests
1305 # TODO: should probably ensure these exist here
1306 manifests = ManifestParser()
1307 manifests.read(args[0])
1308
1309 # print the resultant query
1310 manifests.update(args[1], None, *tags, **kwargs)
1311
1312
1313 # command -> class mapping
1314 commands = { 'create': CreateCLI,
1315 'help': HelpCLI,
1316 'update': UpdateCLI,
1317 'write': WriteCLI }
1318
1319 def main(args=sys.argv[1:]):
1320 """console_script entry point"""
1321
1322 # set up an option parser
1323 usage = '%prog [options] [command] ...'
1324 description = "%s. Use `help` to display commands" % __doc__.strip()
1325 parser = OptionParser(usage=usage, description=description)
1326 parser.add_option('-s', '--strict', dest='strict',
1327 action='store_true', default=False,
1328 help='adhere strictly to errors')
1329 parser.disable_interspersed_args()
1330
1331 options, args = parser.parse_args(args)
1332
1333 if not args:
1334 HelpCLI(parser)(options, args)
1335 parser.exit()
1336
1337 # get the command
1338 command = args[0]
1339 if command not in commands:
1340 parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
1341
1342 handler = commands[command](parser)
1343 handler(options, args[1:])
1344
1345 if __name__ == '__main__':
1346 main()

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial