Ticket #3320: ticket-3320.diff

File ticket-3320.diff, 33.9 KB (added by mir@…, 9 years ago)

basically the same patches, but with full paths. Still doesn't work.

  • django/utils/simplejson/LICENSE.txt

    a b  
     1Copyright (c) 2006 Bob Ippolito
     2
     3Permission is hereby granted, free of charge, to any person obtaining a copy of
     4this software and associated documentation files (the "Software"), to deal in
     5the Software without restriction, including without limitation the rights to
     6use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
     7of the Software, and to permit persons to whom the Software is furnished to do
     8so, subject to the following conditions:
     9
     10The above copyright notice and this permission notice shall be included in all
     11copies or substantial portions of the Software.
     12
     13THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     14IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     15FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
     16AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     17LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     18OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
     19SOFTWARE.
  • django/utils/simplejson/__init__.py

    diff --git a/django/utils/simplejson/__init__.py b/django/utils/simplejson/__init__.py
    index f88329b..5c79ccf 100644
    a b  
     1r"""
     2A simple, fast, extensible JSON encoder and decoder
     3
     4JSON (JavaScript Object Notation) <http://json.org> is a subset of
     5JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
     6interchange format.
     7
     8simplejson exposes an API familiar to uses of the standard library
     9marshal and pickle modules.
     10
     11Encoding basic Python object hierarchies::
     12   
     13    >>> import simplejson
     14    >>> simplejson.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
     15    '["foo", {"bar": ["baz", null, 1.0, 2]}]'
     16    >>> print simplejson.dumps("\"foo\bar")
     17    "\"foo\bar"
     18    >>> print simplejson.dumps(u'\u1234')
     19    "\u1234"
     20    >>> print simplejson.dumps('\\')
     21    "\\"
     22    >>> print simplejson.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)
     23    {"a": 0, "b": 0, "c": 0}
     24    >>> from StringIO import StringIO
     25    >>> io = StringIO()
     26    >>> simplejson.dump(['streaming API'], io)
     27    >>> io.getvalue()
     28    '["streaming API"]'
     29
     30Compact encoding::
     31
     32    >>> import simplejson
     33    >>> simplejson.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
     34    '[1,2,3,{"4":5,"6":7}]'
     35
     36Pretty printing::
     37
     38    >>> import simplejson
     39    >>> print simplejson.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
     40    {
     41        "4": 5,
     42        "6": 7
     43    }
     44
     45Decoding JSON::
     46   
     47    >>> import simplejson
     48    >>> simplejson.loads('["foo", {"bar":["baz", null, 1.0, 2]}]')
     49    [u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
     50    >>> simplejson.loads('"\\"foo\\bar"')
     51    u'"foo\x08ar'
     52    >>> from StringIO import StringIO
     53    >>> io = StringIO('["streaming API"]')
     54    >>> simplejson.load(io)
     55    [u'streaming API']
     56
     57Specializing JSON object decoding::
     58
     59    >>> import simplejson
     60    >>> def as_complex(dct):
     61    ...     if '__complex__' in dct:
     62    ...         return complex(dct['real'], dct['imag'])
     63    ...     return dct
     64    ...
     65    >>> simplejson.loads('{"__complex__": true, "real": 1, "imag": 2}',
     66    ...     object_hook=as_complex)
     67    (1+2j)
     68
     69Extending JSONEncoder::
     70   
     71    >>> import simplejson
     72    >>> class ComplexEncoder(simplejson.JSONEncoder):
     73    ...     def default(self, obj):
     74    ...         if isinstance(obj, complex):
     75    ...             return [obj.real, obj.imag]
     76    ...         return simplejson.JSONEncoder.default(self, obj)
     77    ...
     78    >>> dumps(2 + 1j, cls=ComplexEncoder)
     79    '[2.0, 1.0]'
     80    >>> ComplexEncoder().encode(2 + 1j)
     81    '[2.0, 1.0]'
     82    >>> list(ComplexEncoder().iterencode(2 + 1j))
     83    ['[', '2.0', ', ', '1.0', ']']
     84   
     85
     86Note that the JSON produced by this module's default settings
     87is a subset of YAML, so it may be used as a serializer for that as well.
     88"""
     89__version__ = '1.5'
     90__all__ = [
     91    'dump', 'dumps', 'load', 'loads',
     92    'JSONDecoder', 'JSONEncoder',
     93]
     94
     95from decoder import JSONDecoder
     96from encoder import JSONEncoder
     97
     98def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     99        allow_nan=True, cls=None, indent=None, **kw):
     100    """
     101    Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
     102    ``.write()``-supporting file-like object).
     103
     104    If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types
     105    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
     106    will be skipped instead of raising a ``TypeError``.
     107
     108    If ``ensure_ascii`` is ``False``, then the some chunks written to ``fp``
     109    may be ``unicode`` instances, subject to normal Python ``str`` to
     110    ``unicode`` coercion rules.  Unless ``fp.write()`` explicitly
     111    understands ``unicode`` (as in ``codecs.getwriter()``) this is likely
     112    to cause an error.
     113
     114    If ``check_circular`` is ``False``, then the circular reference check
     115    for container types will be skipped and a circular reference will
     116    result in an ``OverflowError`` (or worse).
     117
     118    If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to
     119    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
     120    in strict compliance of the JSON specification, instead of using the
     121    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
     122
     123    If ``indent`` is a non-negative integer, then JSON array elements and object
     124    members will be pretty-printed with that indent level.  An indent level
     125    of 0 will only insert newlines.  ``None`` is the most compact representation.
     126
     127    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     128    ``.default()`` method to serialize additional types), specify it with
     129    the ``cls`` kwarg.
     130    """
     131    if cls is None:
     132        cls = JSONEncoder
     133    iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
     134        check_circular=check_circular, allow_nan=allow_nan, indent=indent,
     135        **kw).iterencode(obj)
     136    # could accelerate with writelines in some versions of Python, at
     137    # a debuggability cost
     138    for chunk in iterable:
     139        fp.write(chunk)
     140
     141def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
     142        allow_nan=True, cls=None, indent=None, separators=None, **kw):
     143    """
     144    Serialize ``obj`` to a JSON formatted ``str``.
     145
     146    If ``skipkeys`` is ``True`` then ``dict`` keys that are not basic types
     147    (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
     148    will be skipped instead of raising a ``TypeError``.
     149
     150    If ``ensure_ascii`` is ``False``, then the return value will be a
     151    ``unicode`` instance subject to normal Python ``str`` to ``unicode``
     152    coercion rules instead of being escaped to an ASCII ``str``.
     153
     154    If ``check_circular`` is ``False``, then the circular reference check
     155    for container types will be skipped and a circular reference will
     156    result in an ``OverflowError`` (or worse).
     157
     158    If ``allow_nan`` is ``False``, then it will be a ``ValueError`` to
     159    serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
     160    strict compliance of the JSON specification, instead of using the
     161    JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
     162
     163    If ``indent`` is a non-negative integer, then JSON array elements and
     164    object members will be pretty-printed with that indent level.  An indent
     165    level of 0 will only insert newlines.  ``None`` is the most compact
     166    representation.
     167
     168    If ``separators`` is an ``(item_separator, dict_separator)`` tuple
     169    then it will be used instead of the default ``(', ', ': ')`` separators.
     170    ``(',', ':')`` is the most compact JSON representation.
     171
     172    To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     173    ``.default()`` method to serialize additional types), specify it with
     174    the ``cls`` kwarg.
     175    """
     176    if cls is None:
     177        cls = JSONEncoder
     178    return cls(
     179        skipkeys=skipkeys, ensure_ascii=ensure_ascii,
     180        check_circular=check_circular, allow_nan=allow_nan, indent=indent,
     181        separators=separators,
     182        **kw).encode(obj)
     183
     184def load(fp, encoding=None, cls=None, object_hook=None, **kw):
     185    """
     186    Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
     187    a JSON document) to a Python object.
     188
     189    If the contents of ``fp`` is encoded with an ASCII based encoding other
     190    than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must
     191    be specified.  Encodings that are not ASCII based (such as UCS-2) are
     192    not allowed, and should be wrapped with
     193    ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode``
     194    object and passed to ``loads()``
     195
     196    ``object_hook`` is an optional function that will be called with the
     197    result of any object literal decode (a ``dict``).  The return value of
     198    ``object_hook`` will be used instead of the ``dict``.  This feature
     199    can be used to implement custom decoders (e.g. JSON-RPC class hinting).
     200   
     201    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
     202    kwarg.
     203    """
     204    if cls is None:
     205        cls = JSONDecoder
     206    if object_hook is not None:
     207        kw['object_hook'] = object_hook
     208    return cls(encoding=encoding, **kw).decode(fp.read())
     209
     210def loads(s, encoding=None, cls=None, object_hook=None, **kw):
     211    """
     212    Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
     213    document) to a Python object.
     214
     215    If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding
     216    other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name
     217    must be specified.  Encodings that are not ASCII based (such as UCS-2)
     218    are not allowed and should be decoded to ``unicode`` first.
     219
     220    ``object_hook`` is an optional function that will be called with the
     221    result of any object literal decode (a ``dict``).  The return value of
     222    ``object_hook`` will be used instead of the ``dict``.  This feature
     223    can be used to implement custom decoders (e.g. JSON-RPC class hinting).
     224
     225    To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
     226    kwarg.
     227    """
     228    if cls is None:
     229        cls = JSONDecoder
     230    if object_hook is not None:
     231        kw['object_hook'] = object_hook
     232    return cls(encoding=encoding, **kw).decode(s)
     233
     234def read(s):
     235    """
     236    json-py API compatibility hook.  Use loads(s) instead.
     237    """
     238    import warnings
     239    warnings.warn("simplejson.loads(s) should be used instead of read(s)",
     240        DeprecationWarning)
     241    return loads(s)
     242
     243def write(obj):
     244    """
     245    json-py API compatibility hook.  Use dumps(s) instead.
     246    """
     247    import warnings
     248    warnings.warn("simplejson.dumps(s) should be used instead of write(s)",
     249        DeprecationWarning)
     250    return dumps(obj)
     251
     252
  • django/utils/simplejson/decoder.py

    diff --git a/django/utils/simplejson/decoder.py b/django/utils/simplejson/decoder.py
    index 684af8c..a1b53b2 100644
    a b  
     1"""
     2Implementation of JSONDecoder
     3"""
     4import re
     5
     6from simplejson.scanner import Scanner, pattern
     7
     8FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
     9
     10def _floatconstants():
     11    import struct
     12    import sys
     13    _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
     14    if sys.byteorder != 'big':
     15        _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
     16    nan, inf = struct.unpack('dd', _BYTES)
     17    return nan, inf, -inf
     18
     19NaN, PosInf, NegInf = _floatconstants()
     20
     21def linecol(doc, pos):
     22    lineno = doc.count('\n', 0, pos) + 1
     23    if lineno == 1:
     24        colno = pos
     25    else:
     26        colno = pos - doc.rindex('\n', 0, pos)
     27    return lineno, colno
     28
     29def errmsg(msg, doc, pos, end=None):
     30    lineno, colno = linecol(doc, pos)
     31    if end is None:
     32        return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
     33    endlineno, endcolno = linecol(doc, end)
     34    return '%s: line %d column %d - line %d column %d (char %d - %d)' % (
     35        msg, lineno, colno, endlineno, endcolno, pos, end)
     36
     37_CONSTANTS = {
     38    '-Infinity': NegInf,
     39    'Infinity': PosInf,
     40    'NaN': NaN,
     41    'true': True,
     42    'false': False,
     43    'null': None,
     44}
     45
     46def JSONConstant(match, context, c=_CONSTANTS):
     47    return c[match.group(0)], None
     48pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
     49
     50def JSONNumber(match, context):
     51    match = JSONNumber.regex.match(match.string, *match.span())
     52    integer, frac, exp = match.groups()
     53    if frac or exp:
     54        res = float(integer + (frac or '') + (exp or ''))
     55    else:
     56        res = int(integer)
     57    return res, None
     58pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
     59
     60STRINGCHUNK = re.compile(r'(.*?)(["\\])', FLAGS)
     61BACKSLASH = {
     62    '"': u'"', '\\': u'\\', '/': u'/',
     63    'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
     64}
     65
     66DEFAULT_ENCODING = "utf-8"
     67
     68def scanstring(s, end, encoding=None, _b=BACKSLASH, _m=STRINGCHUNK.match):
     69    if encoding is None:
     70        encoding = DEFAULT_ENCODING
     71    chunks = []
     72    _append = chunks.append
     73    begin = end - 1
     74    while 1:
     75        chunk = _m(s, end)
     76        if chunk is None:
     77            raise ValueError(
     78                errmsg("Unterminated string starting at", s, begin))
     79        end = chunk.end()
     80        content, terminator = chunk.groups()
     81        if content:
     82            if not isinstance(content, unicode):
     83                content = unicode(content, encoding)
     84            _append(content)
     85        if terminator == '"':
     86            break
     87        try:
     88            esc = s[end]
     89        except IndexError:
     90            raise ValueError(
     91                errmsg("Unterminated string starting at", s, begin))
     92        if esc != 'u':
     93            try:
     94                m = _b[esc]
     95            except KeyError:
     96                raise ValueError(
     97                    errmsg("Invalid \\escape: %r" % (esc,), s, end))
     98            end += 1
     99        else:
     100            esc = s[end + 1:end + 5]
     101            try:
     102                m = unichr(int(esc, 16))
     103                if len(esc) != 4 or not esc.isalnum():
     104                    raise ValueError
     105            except ValueError:
     106                raise ValueError(errmsg("Invalid \\uXXXX escape", s, end))
     107            end += 5
     108        _append(m)
     109    return u''.join(chunks), end
     110
     111def JSONString(match, context):
     112    encoding = getattr(context, 'encoding', None)
     113    return scanstring(match.string, match.end(), encoding)
     114pattern(r'"')(JSONString)
     115
     116WHITESPACE = re.compile(r'\s*', FLAGS)
     117
     118def JSONObject(match, context, _w=WHITESPACE.match):
     119    pairs = {}
     120    s = match.string
     121    end = _w(s, match.end()).end()
     122    nextchar = s[end:end + 1]
     123    # trivial empty object
     124    if nextchar == '}':
     125        return pairs, end + 1
     126    if nextchar != '"':
     127        raise ValueError(errmsg("Expecting property name", s, end))
     128    end += 1
     129    encoding = getattr(context, 'encoding', None)
     130    iterscan = JSONScanner.iterscan
     131    while True:
     132        key, end = scanstring(s, end, encoding)
     133        end = _w(s, end).end()
     134        if s[end:end + 1] != ':':
     135            raise ValueError(errmsg("Expecting : delimiter", s, end))
     136        end = _w(s, end + 1).end()
     137        try:
     138            value, end = iterscan(s, idx=end, context=context).next()
     139        except StopIteration:
     140            raise ValueError(errmsg("Expecting object", s, end))
     141        pairs[key] = value
     142        end = _w(s, end).end()
     143        nextchar = s[end:end + 1]
     144        end += 1
     145        if nextchar == '}':
     146            break
     147        if nextchar != ',':
     148            raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
     149        end = _w(s, end).end()
     150        nextchar = s[end:end + 1]
     151        end += 1
     152        if nextchar != '"':
     153            raise ValueError(errmsg("Expecting property name", s, end - 1))
     154    object_hook = getattr(context, 'object_hook', None)
     155    if object_hook is not None:
     156        pairs = object_hook(pairs)
     157    return pairs, end
     158pattern(r'{')(JSONObject)
     159           
     160def JSONArray(match, context, _w=WHITESPACE.match):
     161    values = []
     162    s = match.string
     163    end = _w(s, match.end()).end()
     164    # look-ahead for trivial empty array
     165    nextchar = s[end:end + 1]
     166    if nextchar == ']':
     167        return values, end + 1
     168    iterscan = JSONScanner.iterscan
     169    while True:
     170        try:
     171            value, end = iterscan(s, idx=end, context=context).next()
     172        except StopIteration:
     173            raise ValueError(errmsg("Expecting object", s, end))
     174        values.append(value)
     175        end = _w(s, end).end()
     176        nextchar = s[end:end + 1]
     177        end += 1
     178        if nextchar == ']':
     179            break
     180        if nextchar != ',':
     181            raise ValueError(errmsg("Expecting , delimiter", s, end))
     182        end = _w(s, end).end()
     183    return values, end
     184pattern(r'\[')(JSONArray)
     185 
     186ANYTHING = [
     187    JSONObject,
     188    JSONArray,
     189    JSONString,
     190    JSONConstant,
     191    JSONNumber,
     192]
     193
     194JSONScanner = Scanner(ANYTHING)
     195
     196class JSONDecoder(object):
     197    """
     198    Simple JSON <http://json.org> decoder
     199
     200    Performs the following translations in decoding:
     201   
     202    +---------------+-------------------+
     203    | JSON          | Python            |
     204    +===============+===================+
     205    | object        | dict              |
     206    +---------------+-------------------+
     207    | array         | list              |
     208    +---------------+-------------------+
     209    | string        | unicode           |
     210    +---------------+-------------------+
     211    | number (int)  | int, long         |
     212    +---------------+-------------------+
     213    | number (real) | float             |
     214    +---------------+-------------------+
     215    | true          | True              |
     216    +---------------+-------------------+
     217    | false         | False             |
     218    +---------------+-------------------+
     219    | null          | None              |
     220    +---------------+-------------------+
     221
     222    It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
     223    their corresponding ``float`` values, which is outside the JSON spec.
     224    """
     225
     226    _scanner = Scanner(ANYTHING)
     227    __all__ = ['__init__', 'decode', 'raw_decode']
     228
     229    def __init__(self, encoding=None, object_hook=None):
     230        """
     231        ``encoding`` determines the encoding used to interpret any ``str``
     232        objects decoded by this instance (utf-8 by default).  It has no
     233        effect when decoding ``unicode`` objects.
     234       
     235        Note that currently only encodings that are a superset of ASCII work,
     236        strings of other encodings should be passed in as ``unicode``.
     237
     238        ``object_hook``, if specified, will be called with the result
     239        of every JSON object decoded and its return value will be used in
     240        place of the given ``dict``.  This can be used to provide custom
     241        deserializations (e.g. to support JSON-RPC class hinting).
     242        """
     243        self.encoding = encoding
     244        self.object_hook = object_hook
     245
     246    def decode(self, s, _w=WHITESPACE.match):
     247        """
     248        Return the Python representation of ``s`` (a ``str`` or ``unicode``
     249        instance containing a JSON document)
     250        """
     251        obj, end = self.raw_decode(s, idx=_w(s, 0).end())
     252        end = _w(s, end).end()
     253        if end != len(s):
     254            raise ValueError(errmsg("Extra data", s, end, len(s)))
     255        return obj
     256
     257    def raw_decode(self, s, **kw):
     258        """
     259        Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
     260        with a JSON document) and return a 2-tuple of the Python
     261        representation and the index in ``s`` where the document ended.
     262
     263        This can be used to decode a JSON document from a string that may
     264        have extraneous data at the end.
     265        """
     266        kw.setdefault('context', self)
     267        try:
     268            obj, end = self._scanner.iterscan(s, **kw).next()
     269        except StopIteration:
     270            raise ValueError("No JSON object could be decoded")
     271        return obj, end
     272
     273__all__ = ['JSONDecoder']
  • django/utils/simplejson/encoder.py

    diff --git a/django/utils/simplejson/encoder.py b/django/utils/simplejson/encoder.py
    index bb1aba0..c83c687 100644
    a b  
     1"""
     2Implementation of JSONEncoder
     3"""
     4import re
     5
     6ESCAPE = re.compile(r'[\x00-\x19\\"\b\f\n\r\t]')
     7ESCAPE_ASCII = re.compile(r'([\\"/]|[^\ -~])')
     8ESCAPE_DCT = {
     9    # escape all forward slashes to prevent </script> attack
     10    '/': '\\/',
     11    '\\': '\\\\',
     12    '"': '\\"',
     13    '\b': '\\b',
     14    '\f': '\\f',
     15    '\n': '\\n',
     16    '\r': '\\r',
     17    '\t': '\\t',
     18}
     19for i in range(0x20):
     20    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
     21
     22# assume this produces an infinity on all machines (probably not guaranteed)
     23INFINITY = float('1e66666')
     24
     25def floatstr(o, allow_nan=True):
     26    # Check for specials.  Note that this type of test is processor- and/or
     27    # platform-specific, so do tests which don't depend on the internals.
     28
     29    if o != o:
     30        text = 'NaN'
     31    elif o == INFINITY:
     32        text = 'Infinity'
     33    elif o == -INFINITY:
     34        text = '-Infinity'
     35    else:
     36        return str(o)
     37
     38    if not allow_nan:
     39        raise ValueError("Out of range float values are not JSON compliant: %r"
     40            % (o,))
     41
     42    return text
     43
     44
     45def encode_basestring(s):
     46    """
     47    Return a JSON representation of a Python string
     48    """
     49    def replace(match):
     50        return ESCAPE_DCT[match.group(0)]
     51    return '"' + ESCAPE.sub(replace, s) + '"'
     52
     53def encode_basestring_ascii(s):
     54    def replace(match):
     55        s = match.group(0)
     56        try:
     57            return ESCAPE_DCT[s]
     58        except KeyError:
     59            return '\\u%04x' % (ord(s),)
     60    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
     61       
     62
     63class JSONEncoder(object):
     64    """
     65    Extensible JSON <http://json.org> encoder for Python data structures.
     66
     67    Supports the following objects and types by default:
     68   
     69    +-------------------+---------------+
     70    | Python            | JSON          |
     71    +===================+===============+
     72    | dict              | object        |
     73    +-------------------+---------------+
     74    | list, tuple       | array         |
     75    +-------------------+---------------+
     76    | str, unicode      | string        |
     77    +-------------------+---------------+
     78    | int, long, float  | number        |
     79    +-------------------+---------------+
     80    | True              | true          |
     81    +-------------------+---------------+
     82    | False             | false         |
     83    +-------------------+---------------+
     84    | None              | null          |
     85    +-------------------+---------------+
     86
     87    To extend this to recognize other objects, subclass and implement a
     88    ``.default()`` method with another method that returns a serializable
     89    object for ``o`` if possible, otherwise it should call the superclass
     90    implementation (to raise ``TypeError``).
     91    """
     92    __all__ = ['__init__', 'default', 'encode', 'iterencode']
     93    item_separator = ', '
     94    key_separator = ': '
     95    def __init__(self, skipkeys=False, ensure_ascii=True,
     96            check_circular=True, allow_nan=True, sort_keys=False,
     97            indent=None, separators=None):
     98        """
     99        Constructor for JSONEncoder, with sensible defaults.
     100
     101        If skipkeys is False, then it is a TypeError to attempt
     102        encoding of keys that are not str, int, long, float or None.  If
     103        skipkeys is True, such items are simply skipped.
     104
     105        If ensure_ascii is True, the output is guaranteed to be str
     106        objects with all incoming unicode characters escaped.  If
     107        ensure_ascii is false, the output will be unicode object.
     108
     109        If check_circular is True, then lists, dicts, and custom encoded
     110        objects will be checked for circular references during encoding to
     111        prevent an infinite recursion (which would cause an OverflowError).
     112        Otherwise, no such check takes place.
     113
     114        If allow_nan is True, then NaN, Infinity, and -Infinity will be
     115        encoded as such.  This behavior is not JSON specification compliant,
     116        but is consistent with most JavaScript based encoders and decoders.
     117        Otherwise, it will be a ValueError to encode such floats.
     118
     119        If sort_keys is True, then the output of dictionaries will be
     120        sorted by key; this is useful for regression tests to ensure
     121        that JSON serializations can be compared on a day-to-day basis.
     122
     123        If indent is a non-negative integer, then JSON array
     124        elements and object members will be pretty-printed with that
     125        indent level.  An indent level of 0 will only insert newlines.
     126        None is the most compact representation.
     127
     128        If specified, separators should be a (item_separator, key_separator)
     129        tuple. The default is (', ', ': '). To get the most compact JSON
     130        representation you should specify (',', ':') to eliminate whitespace.
     131        """
     132
     133        self.skipkeys = skipkeys
     134        self.ensure_ascii = ensure_ascii
     135        self.check_circular = check_circular
     136        self.allow_nan = allow_nan
     137        self.sort_keys = sort_keys
     138        self.indent = indent
     139        self.current_indent_level = 0
     140        if separators is not None:
     141            self.item_separator, self.key_separator = separators
     142
     143    def _newline_indent(self):
     144        return '\n' + (' ' * (self.indent * self.current_indent_level))
     145
     146    def _iterencode_list(self, lst, markers=None):
     147        if not lst:
     148            yield '[]'
     149            return
     150        if markers is not None:
     151            markerid = id(lst)
     152            if markerid in markers:
     153                raise ValueError("Circular reference detected")
     154            markers[markerid] = lst
     155        yield '['
     156        if self.indent is not None:
     157            self.current_indent_level += 1
     158            newline_indent = self._newline_indent()
     159            separator = self.item_separator + newline_indent
     160            yield newline_indent
     161        else:
     162            newline_indent = None
     163            separator = self.item_separator
     164        first = True
     165        for value in lst:
     166            if first:
     167                first = False
     168            else:
     169                yield separator
     170            for chunk in self._iterencode(value, markers):
     171                yield chunk
     172        if newline_indent is not None:
     173            self.current_indent_level -= 1
     174            yield self._newline_indent()
     175        yield ']'
     176        if markers is not None:
     177            del markers[markerid]
     178
     179    def _iterencode_dict(self, dct, markers=None):
     180        if not dct:
     181            yield '{}'
     182            return
     183        if markers is not None:
     184            markerid = id(dct)
     185            if markerid in markers:
     186                raise ValueError("Circular reference detected")
     187            markers[markerid] = dct
     188        yield '{'
     189        key_separator = self.key_separator
     190        if self.indent is not None:
     191            self.current_indent_level += 1
     192            newline_indent = self._newline_indent()
     193            item_separator = self.item_separator + newline_indent
     194            yield newline_indent
     195        else:
     196            newline_indent = None
     197            item_separator = self.item_separator
     198        first = True
     199        if self.ensure_ascii:
     200            encoder = encode_basestring_ascii
     201        else:
     202            encoder = encode_basestring
     203        allow_nan = self.allow_nan
     204        if self.sort_keys:
     205            keys = dct.keys()
     206            keys.sort()
     207            items = [(k, dct[k]) for k in keys]
     208        else:
     209            items = dct.iteritems()
     210        for key, value in items:
     211            if isinstance(key, basestring):
     212                pass
     213            # JavaScript is weakly typed for these, so it makes sense to
     214            # also allow them.  Many encoders seem to do something like this.
     215            elif isinstance(key, float):
     216                key = floatstr(key, allow_nan)
     217            elif isinstance(key, (int, long)):
     218                key = str(key)
     219            elif key is True:
     220                key = 'true'
     221            elif key is False:
     222                key = 'false'
     223            elif key is None:
     224                key = 'null'
     225            elif self.skipkeys:
     226                continue
     227            else:
     228                raise TypeError("key %r is not a string" % (key,))
     229            if first:
     230                first = False
     231            else:
     232                yield item_separator
     233            yield encoder(key)
     234            yield key_separator
     235            for chunk in self._iterencode(value, markers):
     236                yield chunk
     237        if newline_indent is not None:
     238            self.current_indent_level -= 1
     239            yield self._newline_indent()
     240        yield '}'
     241        if markers is not None:
     242            del markers[markerid]
     243
     244    def _iterencode(self, o, markers=None):
     245        if isinstance(o, basestring):
     246            if self.ensure_ascii:
     247                encoder = encode_basestring_ascii
     248            else:
     249                encoder = encode_basestring
     250            yield encoder(o)
     251        elif o is None:
     252            yield 'null'
     253        elif o is True:
     254            yield 'true'
     255        elif o is False:
     256            yield 'false'
     257        elif isinstance(o, (int, long)):
     258            yield str(o)
     259        elif isinstance(o, float):
     260            yield floatstr(o, self.allow_nan)
     261        elif isinstance(o, (list, tuple)):
     262            for chunk in self._iterencode_list(o, markers):
     263                yield chunk
     264        elif isinstance(o, dict):
     265            for chunk in self._iterencode_dict(o, markers):
     266                yield chunk
     267        else:
     268            if markers is not None:
     269                markerid = id(o)
     270                if markerid in markers:
     271                    raise ValueError("Circular reference detected")
     272                markers[markerid] = o
     273            for chunk in self._iterencode_default(o, markers):
     274                yield chunk
     275            if markers is not None:
     276                del markers[markerid]
     277
     278    def _iterencode_default(self, o, markers=None):
     279        newobj = self.default(o)
     280        return self._iterencode(newobj, markers)
     281
     282    def default(self, o):
     283        """
     284        Implement this method in a subclass such that it returns
     285        a serializable object for ``o``, or calls the base implementation
     286        (to raise a ``TypeError``).
     287
     288        For example, to support arbitrary iterators, you could
     289        implement default like this::
     290           
     291            def default(self, o):
     292                try:
     293                    iterable = iter(o)
     294                except TypeError:
     295                    pass
     296                else:
     297                    return list(iterable)
     298                return JSONEncoder.default(self, o)
     299        """
     300        raise TypeError("%r is not JSON serializable" % (o,))
     301
     302    def encode(self, o):
     303        """
     304        Return a JSON string representation of a Python data structure.
     305
     306        >>> JSONEncoder().encode({"foo": ["bar", "baz"]})
     307        '{"foo":["bar", "baz"]}'
     308        """
     309        # This doesn't pass the iterator directly to ''.join() because it
     310        # sucks at reporting exceptions.  It's going to do this internally
     311        # anyway because it uses PySequence_Fast or similar.
     312        chunks = list(self.iterencode(o))
     313        return ''.join(chunks)
     314
     315    def iterencode(self, o):
     316        """
     317        Encode the given object and yield each string
     318        representation as available.
     319       
     320        For example::
     321           
     322            for chunk in JSONEncoder().iterencode(bigobject):
     323                mysocket.write(chunk)
     324        """
     325        if self.check_circular:
     326            markers = {}
     327        else:
     328            markers = None
     329        return self._iterencode(o, markers)
     330
     331__all__ = ['JSONEncoder']
  • django/utils/simplejson/scanner.py

    diff --git a/django/utils/simplejson/scanner.py b/django/utils/simplejson/scanner.py
    index b9244cf..64f4999 100644
    a b  
     1"""
     2Iterator based sre token scanner
     3"""
     4import sre_parse, sre_compile, sre_constants
     5from sre_constants import BRANCH, SUBPATTERN
     6from re import VERBOSE, MULTILINE, DOTALL
     7import re
     8
     9__all__ = ['Scanner', 'pattern']
     10
     11FLAGS = (VERBOSE | MULTILINE | DOTALL)
     12class Scanner(object):
     13    def __init__(self, lexicon, flags=FLAGS):
     14        self.actions = [None]
     15        # combine phrases into a compound pattern
     16        s = sre_parse.Pattern()
     17        s.flags = flags
     18        p = []
     19        for idx, token in enumerate(lexicon):
     20            phrase = token.pattern
     21            try:
     22                subpattern = sre_parse.SubPattern(s,
     23                    [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
     24            except sre_constants.error:
     25                raise
     26            p.append(subpattern)
     27            self.actions.append(token)
     28
     29        p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
     30        self.scanner = sre_compile.compile(p)
     31
     32
     33    def iterscan(self, string, idx=0, context=None):
     34        """
     35        Yield match, end_idx for each match
     36        """
     37        match = self.scanner.scanner(string, idx).match
     38        actions = self.actions
     39        lastend = idx
     40        end = len(string)
     41        while True:
     42            m = match()
     43            if m is None:
     44                break
     45            matchbegin, matchend = m.span()
     46            if lastend == matchend:
     47                break
     48            action = actions[m.lastindex]
     49            if action is not None:
     50                rval, next_pos = action(m, context)
     51                if next_pos is not None and next_pos != matchend:
     52                    # "fast forward" the scanner
     53                    matchend = next_pos
     54                    match = self.scanner.scanner(string, matchend).match
     55                yield rval, matchend
     56            lastend = matchend
     57           
     58def pattern(pattern, flags=FLAGS):
     59    def decorator(fn):
     60        fn.pattern = pattern
     61        fn.regex = re.compile(pattern, flags)
     62        return fn
     63    return decorator
Back to Top