upgrade simplejson to v3.8.0

wakatime/packages/simplejson/__init__.py

@@ -5,9 +5,8 @@ interchange format.
 :mod:`simplejson` exposes an API familiar to users of the standard library
 :mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
 version of the :mod:`json` library contained in Python 2.6, but maintains
-compatibility with Python 2.4 and Python 2.5 and (currently) has
+compatibility back to Python 2.5 and (currently) has significant performance
-significant performance advantages, even without using the optional C
+advantages, even without using the optional C extension for speedups.
-extension for speedups.
 
 Encoding basic Python object hierarchies::
 
@@ -98,7 +97,7 @@ Using simplejson.tool from the shell to validate and pretty-print::
     Expecting property name: line 1 column 3 (char 2)
 """
 from __future__ import absolute_import
-__version__ = '3.6.5'
+__version__ = '3.8.0'
 __all__ = [
     'dump', 'dumps', 'load', 'loads',
     'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
@@ -140,6 +139,7 @@ _default_encoder = JSONEncoder(
     use_decimal=True,
     namedtuple_as_object=True,
     tuple_as_array=True,
+    iterable_as_array=False,
     bigint_as_string=False,
     item_sort_key=None,
     for_json=False,
@@ -152,7 +152,8 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
          encoding='utf-8', default=None, use_decimal=True,
          namedtuple_as_object=True, tuple_as_array=True,
          bigint_as_string=False, sort_keys=False, item_sort_key=None,
-         for_json=False, ignore_nan=False, int_as_string_bitcount=None, **kw):
+         for_json=False, ignore_nan=False, int_as_string_bitcount=None,
+         iterable_as_array=False, **kw):
     """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
     ``.write()``-supporting file-like object).
 
@@ -204,6 +205,10 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     If *tuple_as_array* is true (default: ``True``),
     :class:`tuple` (and subclasses) will be encoded as JSON arrays.
 
+    If *iterable_as_array* is true (default: ``False``),
+    any object not in the above table that implements ``__iter__()``
+    will be encoded as a JSON array.
+
     If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher
     or lower than -2**53 will be encoded as strings. This is to avoid the
     rounding that happens in Javascript otherwise. Note that this is still a
@@ -242,7 +247,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
         check_circular and allow_nan and
         cls is None and indent is None and separators is None and
         encoding == 'utf-8' and default is None and use_decimal
-        and namedtuple_as_object and tuple_as_array
+        and namedtuple_as_object and tuple_as_array and not iterable_as_array
         and not bigint_as_string and not sort_keys
         and not item_sort_key and not for_json
         and not ignore_nan and int_as_string_bitcount is None
@@ -258,6 +263,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
             default=default, use_decimal=use_decimal,
             namedtuple_as_object=namedtuple_as_object,
             tuple_as_array=tuple_as_array,
+            iterable_as_array=iterable_as_array,
             bigint_as_string=bigint_as_string,
             sort_keys=sort_keys,
             item_sort_key=item_sort_key,
@@ -276,7 +282,8 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
           encoding='utf-8', default=None, use_decimal=True,
           namedtuple_as_object=True, tuple_as_array=True,
           bigint_as_string=False, sort_keys=False, item_sort_key=None,
-          for_json=False, ignore_nan=False, int_as_string_bitcount=None, **kw):
+          for_json=False, ignore_nan=False, int_as_string_bitcount=None,
+          iterable_as_array=False, **kw):
     """Serialize ``obj`` to a JSON formatted ``str``.
 
     If ``skipkeys`` is false then ``dict`` keys that are not basic types
@@ -324,6 +331,10 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
     If *tuple_as_array* is true (default: ``True``),
     :class:`tuple` (and subclasses) will be encoded as JSON arrays.
 
+    If *iterable_as_array* is true (default: ``False``),
+    any object not in the above table that implements ``__iter__()``
+    will be encoded as a JSON array.
+
     If *bigint_as_string* is true (not the default), ints 2**53 and higher
     or lower than -2**53 will be encoded as strings. This is to avoid the
     rounding that happens in Javascript otherwise.
@@ -356,12 +367,11 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
 
     """
     # cached encoder
-    if (
+    if (not skipkeys and ensure_ascii and
-        not skipkeys and ensure_ascii and
         check_circular and allow_nan and
         cls is None and indent is None and separators is None and
         encoding == 'utf-8' and default is None and use_decimal
-        and namedtuple_as_object and tuple_as_array
+        and namedtuple_as_object and tuple_as_array and not iterable_as_array
         and not bigint_as_string and not sort_keys
         and not item_sort_key and not for_json
         and not ignore_nan and int_as_string_bitcount is None
@@ -377,6 +387,7 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
         use_decimal=use_decimal,
         namedtuple_as_object=namedtuple_as_object,
         tuple_as_array=tuple_as_array,
+        iterable_as_array=iterable_as_array,
         bigint_as_string=bigint_as_string,
         sort_keys=sort_keys,
         item_sort_key=item_sort_key,

wakatime/packages/simplejson/_speedups.c

@@ -10,6 +10,7 @@
 #define PyString_AS_STRING PyBytes_AS_STRING
 #define PyString_FromStringAndSize PyBytes_FromStringAndSize
 #define PyInt_Check(obj) 0
+#define PyInt_CheckExact(obj) 0
 #define JSON_UNICHR Py_UCS4
 #define JSON_InternFromString PyUnicode_InternFromString
 #define JSON_Intern_GET_SIZE PyUnicode_GET_SIZE
@@ -168,6 +169,7 @@ typedef struct _PyEncoderObject {
     int use_decimal;
     int namedtuple_as_object;
     int tuple_as_array;
+    int iterable_as_array;
     PyObject *max_long_size;
     PyObject *min_long_size;
     PyObject *item_sort_key;
@@ -660,8 +662,21 @@ encoder_stringify_key(PyEncoderObject *s, PyObject *key)
         return _encoded_const(key);
     }
     else if (PyInt_Check(key) || PyLong_Check(key)) {
+        if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) {
+            /* See #118, do not trust custom str/repr */
+            PyObject *res;
+            PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, key, NULL);
+            if (tmp == NULL) {
+                return NULL;
+            }
+            res = PyObject_Str(tmp);
+            Py_DECREF(tmp);
+            return res;
+        }
+        else {
             return PyObject_Str(key);
         }
+    }
     else if (s->use_decimal && PyObject_TypeCheck(key, (PyTypeObject *)s->Decimal)) {
         return PyObject_Str(key);
     }
@@ -2567,7 +2582,6 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 static int
 encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
 {
-    /* initialize Encoder object */
     static char *kwlist[] = {
         "markers",
         "default",
@@ -2582,30 +2596,32 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
         "use_decimal",
         "namedtuple_as_object",
         "tuple_as_array",
+        "iterable_as_array"
         "int_as_string_bitcount",
         "item_sort_key",
         "encoding",
         "for_json",
         "ignore_nan",
         "Decimal",
+        "iterable_as_array",
         NULL};
 
     PyEncoderObject *s;
     PyObject *markers, *defaultfn, *encoder, *indent, *key_separator;
     PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo;
-    PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array;
+    PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array, *iterable_as_array;
     PyObject *int_as_string_bitcount, *item_sort_key, *encoding, *for_json;
     PyObject *ignore_nan, *Decimal;
 
     assert(PyEncoder_Check(self));
     s = (PyEncoderObject *)self;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOO:make_encoder", kwlist,
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOOO:make_encoder", kwlist,
         &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator,
         &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal,
         &namedtuple_as_object, &tuple_as_array,
         &int_as_string_bitcount, &item_sort_key, &encoding, &for_json,
-        &ignore_nan, &Decimal))
+        &ignore_nan, &Decimal, &iterable_as_array))
         return -1;
 
     Py_INCREF(markers);
@@ -2635,9 +2651,10 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds)
     s->use_decimal = PyObject_IsTrue(use_decimal);
     s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object);
     s->tuple_as_array = PyObject_IsTrue(tuple_as_array);
+    s->iterable_as_array = PyObject_IsTrue(iterable_as_array);
     if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) {
         static const unsigned int long_long_bitsize = SIZEOF_LONG_LONG * 8;
-        int int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount);
+        int int_as_string_bitcount_val = (int)PyLong_AsLong(int_as_string_bitcount);
         if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < long_long_bitsize) {
             s->max_long_size = PyLong_FromUnsignedLongLong(1ULL << int_as_string_bitcount_val);
             s->min_long_size = PyLong_FromLongLong(-1LL << int_as_string_bitcount_val);
@@ -2800,8 +2817,21 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj)
         }
     }
     /* Use a better float format here? */
+    if (PyFloat_CheckExact(obj)) {
         return PyObject_Repr(obj);
     }
+    else {
+        /* See #118, do not trust custom str/repr */
+        PyObject *res;
+        PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyFloat_Type, obj, NULL);
+        if (tmp == NULL) {
+            return NULL;
+        }
+        res = PyObject_Repr(tmp);
+        Py_DECREF(tmp);
+        return res;
+    }
+}
 
 static PyObject *
 encoder_encode_string(PyEncoderObject *s, PyObject *obj)
@@ -2840,7 +2870,21 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss
                 rv = _steal_accumulate(rval, encoded);
         }
         else if (PyInt_Check(obj) || PyLong_Check(obj)) {
-            PyObject *encoded = PyObject_Str(obj);
+            PyObject *encoded;
+            if (PyInt_CheckExact(obj) || PyLong_CheckExact(obj)) {
+                encoded = PyObject_Str(obj);
+            }
+            else {
+                /* See #118, do not trust custom str/repr */
+                PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, obj, NULL);
+                if (tmp == NULL) {
+                    encoded = NULL;
+                }
+                else {
+                    encoded = PyObject_Str(tmp);
+                    Py_DECREF(tmp);
+                }
+            }
             if (encoded != NULL) {
                 encoded = maybe_quote_bigint(s, encoded, obj);
                 if (encoded == NULL)
@@ -2895,6 +2939,16 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss
         else {
             PyObject *ident = NULL;
             PyObject *newobj;
+            if (s->iterable_as_array) {
+                newobj = PyObject_GetIter(obj);
+                if (newobj == NULL)
+                    PyErr_Clear();
+                else {
+                    rv = encoder_listencode_list(s, rval, newobj, indent_level);
+                    Py_DECREF(newobj);
+                    break;
+                }
+            }
             if (s->markers != Py_None) {
                 int has_key;
                 ident = PyLong_FromVoidPtr(obj);

wakatime/packages/simplejson/encoder.py

@@ -3,7 +3,8 @@
 from __future__ import absolute_import
 import re
 from operator import itemgetter
-from decimal import Decimal
+# Do not import Decimal directly to avoid reload issues
+import decimal
 from .compat import u, unichr, binary_type, string_types, integer_types, PY3
 def _import_speedups():
     try:
@@ -123,7 +124,7 @@ class JSONEncoder(object):
                  use_decimal=True, namedtuple_as_object=True,
                  tuple_as_array=True, bigint_as_string=False,
                  item_sort_key=None, for_json=False, ignore_nan=False,
-                 int_as_string_bitcount=None):
+                 int_as_string_bitcount=None, iterable_as_array=False):
         """Constructor for JSONEncoder, with sensible defaults.
 
         If skipkeys is false, then it is a TypeError to attempt
@@ -178,6 +179,10 @@ class JSONEncoder(object):
         If tuple_as_array is true (the default), tuple (and subclasses) will
         be encoded as JSON arrays.
 
+        If *iterable_as_array* is true (default: ``False``),
+        any object not in the above table that implements ``__iter__()``
+        will be encoded as a JSON array.
+
         If bigint_as_string is true (not the default), ints 2**53 and higher
         or lower than -2**53 will be encoded as strings. This is to avoid the
         rounding that happens in Javascript otherwise.
@@ -209,6 +214,7 @@ class JSONEncoder(object):
         self.use_decimal = use_decimal
         self.namedtuple_as_object = namedtuple_as_object
         self.tuple_as_array = tuple_as_array
+        self.iterable_as_array = iterable_as_array
         self.bigint_as_string = bigint_as_string
         self.item_sort_key = item_sort_key
         self.for_json = for_json
@@ -311,6 +317,9 @@ class JSONEncoder(object):
             elif o == _neginf:
                 text = '-Infinity'
             else:
+                if type(o) != float:
+                    # See #118, do not trust custom str/repr
+                    o = float(o)
                 return _repr(o)
 
             if ignore_nan:
@@ -334,7 +343,7 @@ class JSONEncoder(object):
                 self.namedtuple_as_object, self.tuple_as_array,
                 int_as_string_bitcount,
                 self.item_sort_key, self.encoding, self.for_json,
-                self.ignore_nan, Decimal)
+                self.ignore_nan, decimal.Decimal, self.iterable_as_array)
         else:
             _iterencode = _make_iterencode(
                 markers, self.default, _encoder, self.indent, floatstr,
@@ -343,7 +352,7 @@ class JSONEncoder(object):
                 self.namedtuple_as_object, self.tuple_as_array,
                 int_as_string_bitcount,
                 self.item_sort_key, self.encoding, self.for_json,
-                Decimal=Decimal)
+                self.iterable_as_array, Decimal=decimal.Decimal)
         try:
             return _iterencode(o, 0)
         finally:
@@ -382,11 +391,12 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         _use_decimal, _namedtuple_as_object, _tuple_as_array,
         _int_as_string_bitcount, _item_sort_key,
         _encoding,_for_json,
+        _iterable_as_array,
         ## HACK: hand-optimized bytecode; turn globals into locals
         _PY3=PY3,
         ValueError=ValueError,
         string_types=string_types,
-        Decimal=Decimal,
+        Decimal=None,
         dict=dict,
         float=float,
         id=id,
@@ -395,7 +405,10 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         list=list,
         str=str,
         tuple=tuple,
+        iter=iter,
     ):
+    if _use_decimal and Decimal is None:
+        Decimal = decimal.Decimal
     if _item_sort_key and not callable(_item_sort_key):
         raise TypeError("item_sort_key must be None or callable")
     elif _sort_keys and not _item_sort_key:
@@ -412,6 +425,9 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
             or
             _int_as_string_bitcount < 1
         )
+        if type(value) not in integer_types:
+            # See #118, do not trust custom str/repr
+            value = int(value)
         if (
             skip_quoting or
             (-1 << _int_as_string_bitcount)
@@ -501,6 +517,9 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
         elif key is None:
             key = 'null'
         elif isinstance(key, integer_types):
+            if type(key) not in integer_types:
+                # See #118, do not trust custom str/repr
+                key = int(key)
             key = str(key)
         elif _use_decimal and isinstance(key, Decimal):
             key = str(key)
@@ -634,6 +653,16 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
                 elif _use_decimal and isinstance(o, Decimal):
                     yield str(o)
                 else:
+                    while _iterable_as_array:
+                        # Markers are not checked here because it is valid for
+                        # an iterable to return self.
+                        try:
+                            o = iter(o)
+                        except TypeError:
+                            break
+                        for chunk in _iterencode_list(o, _current_indent_level):
+                            yield chunk
+                        return
                     if markers is not None:
                         markerid = id(o)
                         if markerid in markers:

wakatime/packages/simplejson/tests/__init__.py

@@ -62,6 +62,7 @@ def all_tests_suite():
                 'simplejson.tests.test_namedtuple',
                 'simplejson.tests.test_tool',
                 'simplejson.tests.test_for_json',
+                'simplejson.tests.test_subclass',
             ]))
     suite = get_suite()
     import simplejson

wakatime/packages/simplejson/tests/test_iterable.py

@@ -0,0 +1,31 @@
+import unittest
+from StringIO import StringIO
+
+import simplejson as json
+
+def iter_dumps(obj, **kw):
+    return ''.join(json.JSONEncoder(**kw).iterencode(obj))
+
+def sio_dump(obj, **kw):
+    sio = StringIO()
+    json.dumps(obj, **kw)
+    return sio.getvalue()
+
+class TestIterable(unittest.TestCase):
+    def test_iterable(self):
+        l = [1, 2, 3]
+        for dumps in (json.dumps, iter_dumps, sio_dump):
+            expect = dumps(l)
+            default_expect = dumps(sum(l))
+            # Default is False
+            self.assertRaises(TypeError, dumps, iter(l))
+            self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False)
+            self.assertEqual(expect, dumps(iter(l), iterable_as_array=True))
+            # Ensure that the "default" gets called
+            self.assertEqual(default_expect, dumps(iter(l), default=sum))
+            self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum))
+            # Ensure that the "default" does not get called
+            self.assertEqual(
+                default_expect,
+                dumps(iter(l), iterable_as_array=True, default=sum))
+        

wakatime/packages/simplejson/tests/test_subclass.py

@@ -0,0 +1,37 @@
+from unittest import TestCase
+import simplejson as json
+
+from decimal import Decimal
+
+class AlternateInt(int):
+    def __repr__(self):
+        return 'invalid json'
+    __str__ = __repr__
+
+
+class AlternateFloat(float):
+    def __repr__(self):
+        return 'invalid json'
+    __str__ = __repr__
+
+
+# class AlternateDecimal(Decimal):
+#     def __repr__(self):
+#         return 'invalid json'
+
+
+class TestSubclass(TestCase):
+    def test_int(self):
+        self.assertEqual(json.dumps(AlternateInt(1)), '1')
+        self.assertEqual(json.dumps(AlternateInt(-1)), '-1')
+        self.assertEqual(json.loads(json.dumps({AlternateInt(1): 1})), {'1': 1})
+
+    def test_float(self):
+        self.assertEqual(json.dumps(AlternateFloat(1.0)), '1.0')
+        self.assertEqual(json.dumps(AlternateFloat(-1.0)), '-1.0')
+        self.assertEqual(json.loads(json.dumps({AlternateFloat(1.0): 1})), {'1.0': 1})
+
+    # NOTE: Decimal subclasses are not supported as-is
+    # def test_decimal(self):
+    #     self.assertEqual(json.dumps(AlternateDecimal('1.0')), '1.0')
+    #     self.assertEqual(json.dumps(AlternateDecimal('-1.0')), '-1.0')

wakatime/packages/simplejson/tests/test_tuple.py

@@ -45,7 +45,3 @@ class TestTuples(unittest.TestCase):
         self.assertEqual(
             json.dumps(repr(t)),
             sio.getvalue())
-
-class TestNamedTuple(unittest.TestCase):
-    def test_namedtuple_dump(self):
-        pass