The Tor Browser / file comparison
comparison: addon-sdk/source/python-lib/simplejson/scanner.py
addon-sdk/source/python-lib/simplejson/scanner.py
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:d1bacea754a0 |
|---|---|
| 1 """ | |
| 2 Iterator based sre token scanner | |
| 3 """ | |
| 4 import re | |
| 5 from re import VERBOSE, MULTILINE, DOTALL | |
| 6 import sre_parse | |
| 7 import sre_compile | |
| 8 import sre_constants | |
| 9 from sre_constants import BRANCH, SUBPATTERN | |
| 10 | |
| 11 __all__ = ['Scanner', 'pattern'] | |
| 12 | |
| 13 FLAGS = (VERBOSE | MULTILINE | DOTALL) | |
| 14 | |
| 15 class Scanner(object): | |
| 16 def __init__(self, lexicon, flags=FLAGS): | |
| 17 self.actions = [None] | |
| 18 # Combine phrases into a compound pattern | |
| 19 s = sre_parse.Pattern() | |
| 20 s.flags = flags | |
| 21 p = [] | |
| 22 for idx, token in enumerate(lexicon): | |
| 23 phrase = token.pattern | |
| 24 try: | |
| 25 subpattern = sre_parse.SubPattern(s, | |
| 26 [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) | |
| 27 except sre_constants.error: | |
| 28 raise | |
| 29 p.append(subpattern) | |
| 30 self.actions.append(token) | |
| 31 | |
| 32 s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work | |
| 33 p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) | |
| 34 self.scanner = sre_compile.compile(p) | |
| 35 | |
| 36 def iterscan(self, string, idx=0, context=None): | |
| 37 """ | |
| 38 Yield match, end_idx for each match | |
| 39 """ | |
| 40 match = self.scanner.scanner(string, idx).match | |
| 41 actions = self.actions | |
| 42 lastend = idx | |
| 43 end = len(string) | |
| 44 while True: | |
| 45 m = match() | |
| 46 if m is None: | |
| 47 break | |
| 48 matchbegin, matchend = m.span() | |
| 49 if lastend == matchend: | |
| 50 break | |
| 51 action = actions[m.lastindex] | |
| 52 if action is not None: | |
| 53 rval, next_pos = action(m, context) | |
| 54 if next_pos is not None and next_pos != matchend: | |
| 55 # "fast forward" the scanner | |
| 56 matchend = next_pos | |
| 57 match = self.scanner.scanner(string, matchend).match | |
| 58 yield rval, matchend | |
| 59 lastend = matchend | |
| 60 | |
| 61 | |
| 62 def pattern(pattern, flags=FLAGS): | |
| 63 def decorator(fn): | |
| 64 fn.pattern = pattern | |
| 65 fn.regex = re.compile(pattern, flags) | |
| 66 return fn | |
| 67 return decorator |
