1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/addon-sdk/source/python-lib/simplejson/decoder.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,343 @@
1.4 +"""
1.5 +Implementation of JSONDecoder
1.6 +"""
1.7 +import re
1.8 +import sys
1.9 +
1.10 +from simplejson.scanner import Scanner, pattern
1.11 +try:
1.12 + from simplejson._speedups import scanstring as c_scanstring
1.13 +except ImportError:
1.14 + pass
1.15 +
1.16 +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
1.17 +
1.18 +def _floatconstants():
1.19 + import struct
1.20 + import sys
1.21 + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
1.22 + if sys.byteorder != 'big':
1.23 + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
1.24 + nan, inf = struct.unpack('dd', _BYTES)
1.25 + return nan, inf, -inf
1.26 +
1.27 +NaN, PosInf, NegInf = _floatconstants()
1.28 +
1.29 +
1.30 +def linecol(doc, pos):
1.31 + lineno = doc.count('\n', 0, pos) + 1
1.32 + if lineno == 1:
1.33 + colno = pos
1.34 + else:
1.35 + colno = pos - doc.rindex('\n', 0, pos)
1.36 + return lineno, colno
1.37 +
1.38 +
1.39 +def errmsg(msg, doc, pos, end=None):
1.40 + lineno, colno = linecol(doc, pos)
1.41 + if end is None:
1.42 + return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
1.43 + endlineno, endcolno = linecol(doc, end)
1.44 + return '%s: line %d column %d - line %d column %d (char %d - %d)' % (
1.45 + msg, lineno, colno, endlineno, endcolno, pos, end)
1.46 +
1.47 +
1.48 +_CONSTANTS = {
1.49 + '-Infinity': NegInf,
1.50 + 'Infinity': PosInf,
1.51 + 'NaN': NaN,
1.52 + 'true': True,
1.53 + 'false': False,
1.54 + 'null': None,
1.55 +}
1.56 +
1.57 +def JSONConstant(match, context, c=_CONSTANTS):
1.58 + s = match.group(0)
1.59 + fn = getattr(context, 'parse_constant', None)
1.60 + if fn is None:
1.61 + rval = c[s]
1.62 + else:
1.63 + rval = fn(s)
1.64 + return rval, None
1.65 +pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
1.66 +
1.67 +
1.68 +def JSONNumber(match, context):
1.69 + match = JSONNumber.regex.match(match.string, *match.span())
1.70 + integer, frac, exp = match.groups()
1.71 + if frac or exp:
1.72 + fn = getattr(context, 'parse_float', None) or float
1.73 + res = fn(integer + (frac or '') + (exp or ''))
1.74 + else:
1.75 + fn = getattr(context, 'parse_int', None) or int
1.76 + res = fn(integer)
1.77 + return res, None
1.78 +pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
1.79 +
1.80 +
1.81 +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
1.82 +BACKSLASH = {
1.83 + '"': u'"', '\\': u'\\', '/': u'/',
1.84 + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
1.85 +}
1.86 +
1.87 +DEFAULT_ENCODING = "utf-8"
1.88 +
1.89 +def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
1.90 + if encoding is None:
1.91 + encoding = DEFAULT_ENCODING
1.92 + chunks = []
1.93 + _append = chunks.append
1.94 + begin = end - 1
1.95 + while 1:
1.96 + chunk = _m(s, end)
1.97 + if chunk is None:
1.98 + raise ValueError(
1.99 + errmsg("Unterminated string starting at", s, begin))
1.100 + end = chunk.end()
1.101 + content, terminator = chunk.groups()
1.102 + if content:
1.103 + if not isinstance(content, unicode):
1.104 + content = unicode(content, encoding)
1.105 + _append(content)
1.106 + if terminator == '"':
1.107 + break
1.108 + elif terminator != '\\':
1.109 + if strict:
1.110 + raise ValueError(errmsg("Invalid control character %r at", s, end))
1.111 + else:
1.112 + _append(terminator)
1.113 + continue
1.114 + try:
1.115 + esc = s[end]
1.116 + except IndexError:
1.117 + raise ValueError(
1.118 + errmsg("Unterminated string starting at", s, begin))
1.119 + if esc != 'u':
1.120 + try:
1.121 + m = _b[esc]
1.122 + except KeyError:
1.123 + raise ValueError(
1.124 + errmsg("Invalid \\escape: %r" % (esc,), s, end))
1.125 + end += 1
1.126 + else:
1.127 + esc = s[end + 1:end + 5]
1.128 + next_end = end + 5
1.129 + msg = "Invalid \\uXXXX escape"
1.130 + try:
1.131 + if len(esc) != 4:
1.132 + raise ValueError
1.133 + uni = int(esc, 16)
1.134 + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
1.135 + msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
1.136 + if not s[end + 5:end + 7] == '\\u':
1.137 + raise ValueError
1.138 + esc2 = s[end + 7:end + 11]
1.139 + if len(esc2) != 4:
1.140 + raise ValueError
1.141 + uni2 = int(esc2, 16)
1.142 + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
1.143 + next_end += 6
1.144 + m = unichr(uni)
1.145 + except ValueError:
1.146 + raise ValueError(errmsg(msg, s, end))
1.147 + end = next_end
1.148 + _append(m)
1.149 + return u''.join(chunks), end
1.150 +
1.151 +
1.152 +# Use speedup
1.153 +try:
1.154 + scanstring = c_scanstring
1.155 +except NameError:
1.156 + scanstring = py_scanstring
1.157 +
1.158 +def JSONString(match, context):
1.159 + encoding = getattr(context, 'encoding', None)
1.160 + strict = getattr(context, 'strict', True)
1.161 + return scanstring(match.string, match.end(), encoding, strict)
1.162 +pattern(r'"')(JSONString)
1.163 +
1.164 +
1.165 +WHITESPACE = re.compile(r'\s*', FLAGS)
1.166 +
1.167 +def JSONObject(match, context, _w=WHITESPACE.match):
1.168 + pairs = {}
1.169 + s = match.string
1.170 + end = _w(s, match.end()).end()
1.171 + nextchar = s[end:end + 1]
1.172 + # Trivial empty object
1.173 + if nextchar == '}':
1.174 + return pairs, end + 1
1.175 + if nextchar != '"':
1.176 + raise ValueError(errmsg("Expecting property name", s, end))
1.177 + end += 1
1.178 + encoding = getattr(context, 'encoding', None)
1.179 + strict = getattr(context, 'strict', True)
1.180 + iterscan = JSONScanner.iterscan
1.181 + while True:
1.182 + key, end = scanstring(s, end, encoding, strict)
1.183 + end = _w(s, end).end()
1.184 + if s[end:end + 1] != ':':
1.185 + raise ValueError(errmsg("Expecting : delimiter", s, end))
1.186 + end = _w(s, end + 1).end()
1.187 + try:
1.188 + value, end = iterscan(s, idx=end, context=context).next()
1.189 + except StopIteration:
1.190 + raise ValueError(errmsg("Expecting object", s, end))
1.191 + pairs[key] = value
1.192 + end = _w(s, end).end()
1.193 + nextchar = s[end:end + 1]
1.194 + end += 1
1.195 + if nextchar == '}':
1.196 + break
1.197 + if nextchar != ',':
1.198 + raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
1.199 + end = _w(s, end).end()
1.200 + nextchar = s[end:end + 1]
1.201 + end += 1
1.202 + if nextchar != '"':
1.203 + raise ValueError(errmsg("Expecting property name", s, end - 1))
1.204 + object_hook = getattr(context, 'object_hook', None)
1.205 + if object_hook is not None:
1.206 + pairs = object_hook(pairs)
1.207 + return pairs, end
1.208 +pattern(r'{')(JSONObject)
1.209 +
1.210 +
1.211 +def JSONArray(match, context, _w=WHITESPACE.match):
1.212 + values = []
1.213 + s = match.string
1.214 + end = _w(s, match.end()).end()
1.215 + # Look-ahead for trivial empty array
1.216 + nextchar = s[end:end + 1]
1.217 + if nextchar == ']':
1.218 + return values, end + 1
1.219 + iterscan = JSONScanner.iterscan
1.220 + while True:
1.221 + try:
1.222 + value, end = iterscan(s, idx=end, context=context).next()
1.223 + except StopIteration:
1.224 + raise ValueError(errmsg("Expecting object", s, end))
1.225 + values.append(value)
1.226 + end = _w(s, end).end()
1.227 + nextchar = s[end:end + 1]
1.228 + end += 1
1.229 + if nextchar == ']':
1.230 + break
1.231 + if nextchar != ',':
1.232 + raise ValueError(errmsg("Expecting , delimiter", s, end))
1.233 + end = _w(s, end).end()
1.234 + return values, end
1.235 +pattern(r'\[')(JSONArray)
1.236 +
1.237 +
1.238 +ANYTHING = [
1.239 + JSONObject,
1.240 + JSONArray,
1.241 + JSONString,
1.242 + JSONConstant,
1.243 + JSONNumber,
1.244 +]
1.245 +
1.246 +JSONScanner = Scanner(ANYTHING)
1.247 +
1.248 +
1.249 +class JSONDecoder(object):
1.250 + """
1.251 + Simple JSON <http://json.org> decoder
1.252 +
1.253 + Performs the following translations in decoding by default:
1.254 +
1.255 + +---------------+-------------------+
1.256 + | JSON | Python |
1.257 + +===============+===================+
1.258 + | object | dict |
1.259 + +---------------+-------------------+
1.260 + | array | list |
1.261 + +---------------+-------------------+
1.262 + | string | unicode |
1.263 + +---------------+-------------------+
1.264 + | number (int) | int, long |
1.265 + +---------------+-------------------+
1.266 + | number (real) | float |
1.267 + +---------------+-------------------+
1.268 + | true | True |
1.269 + +---------------+-------------------+
1.270 + | false | False |
1.271 + +---------------+-------------------+
1.272 + | null | None |
1.273 + +---------------+-------------------+
1.274 +
1.275 + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
1.276 + their corresponding ``float`` values, which is outside the JSON spec.
1.277 + """
1.278 +
1.279 + _scanner = Scanner(ANYTHING)
1.280 + __all__ = ['__init__', 'decode', 'raw_decode']
1.281 +
1.282 + def __init__(self, encoding=None, object_hook=None, parse_float=None,
1.283 + parse_int=None, parse_constant=None, strict=True):
1.284 + """
1.285 + ``encoding`` determines the encoding used to interpret any ``str``
1.286 + objects decoded by this instance (utf-8 by default). It has no
1.287 + effect when decoding ``unicode`` objects.
1.288 +
1.289 + Note that currently only encodings that are a superset of ASCII work,
1.290 + strings of other encodings should be passed in as ``unicode``.
1.291 +
1.292 + ``object_hook``, if specified, will be called with the result
1.293 + of every JSON object decoded and its return value will be used in
1.294 + place of the given ``dict``. This can be used to provide custom
1.295 + deserializations (e.g. to support JSON-RPC class hinting).
1.296 +
1.297 + ``parse_float``, if specified, will be called with the string
1.298 + of every JSON float to be decoded. By default this is equivalent to
1.299 + float(num_str). This can be used to use another datatype or parser
1.300 + for JSON floats (e.g. decimal.Decimal).
1.301 +
1.302 + ``parse_int``, if specified, will be called with the string
1.303 + of every JSON int to be decoded. By default this is equivalent to
1.304 + int(num_str). This can be used to use another datatype or parser
1.305 + for JSON integers (e.g. float).
1.306 +
1.307 + ``parse_constant``, if specified, will be called with one of the
1.308 + following strings: -Infinity, Infinity, NaN, null, true, false.
1.309 + This can be used to raise an exception if invalid JSON numbers
1.310 + are encountered.
1.311 + """
1.312 + self.encoding = encoding
1.313 + self.object_hook = object_hook
1.314 + self.parse_float = parse_float
1.315 + self.parse_int = parse_int
1.316 + self.parse_constant = parse_constant
1.317 + self.strict = strict
1.318 +
1.319 + def decode(self, s, _w=WHITESPACE.match):
1.320 + """
1.321 + Return the Python representation of ``s`` (a ``str`` or ``unicode``
1.322 + instance containing a JSON document)
1.323 + """
1.324 + obj, end = self.raw_decode(s, idx=_w(s, 0).end())
1.325 + end = _w(s, end).end()
1.326 + if end != len(s):
1.327 + raise ValueError(errmsg("Extra data", s, end, len(s)))
1.328 + return obj
1.329 +
1.330 + def raw_decode(self, s, **kw):
1.331 + """
1.332 + Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
1.333 + with a JSON document) and return a 2-tuple of the Python
1.334 + representation and the index in ``s`` where the document ended.
1.335 +
1.336 + This can be used to decode a JSON document from a string that may
1.337 + have extraneous data at the end.
1.338 + """
1.339 + kw.setdefault('context', self)
1.340 + try:
1.341 + obj, end = self._scanner.iterscan(s, **kw).next()
1.342 + except StopIteration:
1.343 + raise ValueError("No JSON object could be decoded")
1.344 + return obj, end
1.345 +
1.346 +__all__ = ['JSONDecoder']
