addon-sdk/source/python-lib/simplejson/scanner.py@6474c204b198 (annotated)
addon-sdk/source/python-lib/simplejson/scanner.py
Wed, 31 Dec 2014 06:09:35 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:09:35 +0100
- changeset 0
- 6474c204b198
- permissions
- -rw-r--r--
Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.
| michael@0 | 1 | """ |
| michael@0 | 2 | Iterator based sre token scanner |
| michael@0 | 3 | """ |
| michael@0 | 4 | import re |
| michael@0 | 5 | from re import VERBOSE, MULTILINE, DOTALL |
| michael@0 | 6 | import sre_parse |
| michael@0 | 7 | import sre_compile |
| michael@0 | 8 | import sre_constants |
| michael@0 | 9 | from sre_constants import BRANCH, SUBPATTERN |
| michael@0 | 10 | |
| michael@0 | 11 | __all__ = ['Scanner', 'pattern'] |
| michael@0 | 12 | |
| michael@0 | 13 | FLAGS = (VERBOSE | MULTILINE | DOTALL) |
| michael@0 | 14 | |
| michael@0 | 15 | class Scanner(object): |
| michael@0 | 16 | def __init__(self, lexicon, flags=FLAGS): |
| michael@0 | 17 | self.actions = [None] |
| michael@0 | 18 | # Combine phrases into a compound pattern |
| michael@0 | 19 | s = sre_parse.Pattern() |
| michael@0 | 20 | s.flags = flags |
| michael@0 | 21 | p = [] |
| michael@0 | 22 | for idx, token in enumerate(lexicon): |
| michael@0 | 23 | phrase = token.pattern |
| michael@0 | 24 | try: |
| michael@0 | 25 | subpattern = sre_parse.SubPattern(s, |
| michael@0 | 26 | [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) |
| michael@0 | 27 | except sre_constants.error: |
| michael@0 | 28 | raise |
| michael@0 | 29 | p.append(subpattern) |
| michael@0 | 30 | self.actions.append(token) |
| michael@0 | 31 | |
| michael@0 | 32 | s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work |
| michael@0 | 33 | p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) |
| michael@0 | 34 | self.scanner = sre_compile.compile(p) |
| michael@0 | 35 | |
| michael@0 | 36 | def iterscan(self, string, idx=0, context=None): |
| michael@0 | 37 | """ |
| michael@0 | 38 | Yield match, end_idx for each match |
| michael@0 | 39 | """ |
| michael@0 | 40 | match = self.scanner.scanner(string, idx).match |
| michael@0 | 41 | actions = self.actions |
| michael@0 | 42 | lastend = idx |
| michael@0 | 43 | end = len(string) |
| michael@0 | 44 | while True: |
| michael@0 | 45 | m = match() |
| michael@0 | 46 | if m is None: |
| michael@0 | 47 | break |
| michael@0 | 48 | matchbegin, matchend = m.span() |
| michael@0 | 49 | if lastend == matchend: |
| michael@0 | 50 | break |
| michael@0 | 51 | action = actions[m.lastindex] |
| michael@0 | 52 | if action is not None: |
| michael@0 | 53 | rval, next_pos = action(m, context) |
| michael@0 | 54 | if next_pos is not None and next_pos != matchend: |
| michael@0 | 55 | # "fast forward" the scanner |
| michael@0 | 56 | matchend = next_pos |
| michael@0 | 57 | match = self.scanner.scanner(string, matchend).match |
| michael@0 | 58 | yield rval, matchend |
| michael@0 | 59 | lastend = matchend |
| michael@0 | 60 | |
| michael@0 | 61 | |
| michael@0 | 62 | def pattern(pattern, flags=FLAGS): |
| michael@0 | 63 | def decorator(fn): |
| michael@0 | 64 | fn.pattern = pattern |
| michael@0 | 65 | fn.regex = re.compile(pattern, flags) |
| michael@0 | 66 | return fn |
| michael@0 | 67 | return decorator |
