From 337e4bae5bd9398317a431f8e3dac33f39e61496 Mon Sep 17 00:00:00 2001 From: Alan Hamlett Date: Tue, 11 Aug 2015 12:21:27 -0700 Subject: [PATCH] upgrade simplejson to v3.8.0 --- wakatime/packages/simplejson/__init__.py | 31 +++++--- wakatime/packages/simplejson/_speedups.c | 70 ++++++++++++++++--- wakatime/packages/simplejson/encoder.py | 39 +++++++++-- .../packages/simplejson/tests/__init__.py | 1 + .../simplejson/tests/test_iterable.py | 31 ++++++++ .../simplejson/tests/test_subclass.py | 37 ++++++++++ .../packages/simplejson/tests/test_tuple.py | 4 -- 7 files changed, 186 insertions(+), 27 deletions(-) create mode 100644 wakatime/packages/simplejson/tests/test_iterable.py create mode 100644 wakatime/packages/simplejson/tests/test_subclass.py diff --git a/wakatime/packages/simplejson/__init__.py b/wakatime/packages/simplejson/__init__.py index 8c0b698..ac1e5cb 100644 --- a/wakatime/packages/simplejson/__init__.py +++ b/wakatime/packages/simplejson/__init__.py @@ -5,9 +5,8 @@ interchange format. :mod:`simplejson` exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. It is the externally maintained version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. +compatibility back to Python 2.5 and (currently) has significant performance +advantages, even without using the optional C extension for speedups. Encoding basic Python object hierarchies:: @@ -98,7 +97,7 @@ Using simplejson.tool from the shell to validate and pretty-print:: Expecting property name: line 1 column 3 (char 2) """ from __future__ import absolute_import -__version__ = '3.6.5' +__version__ = '3.8.0' __all__ = [ 'dump', 'dumps', 'load', 'loads', 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', @@ -140,6 +139,7 @@ _default_encoder = JSONEncoder( use_decimal=True, namedtuple_as_object=True, tuple_as_array=True, + iterable_as_array=False, bigint_as_string=False, item_sort_key=None, for_json=False, @@ -152,7 +152,8 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, encoding='utf-8', default=None, use_decimal=True, namedtuple_as_object=True, tuple_as_array=True, bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, **kw): + for_json=False, ignore_nan=False, int_as_string_bitcount=None, + iterable_as_array=False, **kw): """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). @@ -204,6 +205,10 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, If *tuple_as_array* is true (default: ``True``), :class:`tuple` (and subclasses) will be encoded as JSON arrays. + If *iterable_as_array* is true (default: ``False``), + any object not in the above table that implements ``__iter__()`` + will be encoded as a JSON array. + If *bigint_as_string* is true (default: ``False``), ints 2**53 and higher or lower than -2**53 will be encoded as strings. This is to avoid the rounding that happens in Javascript otherwise. Note that this is still a @@ -242,7 +247,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, check_circular and allow_nan and cls is None and indent is None and separators is None and encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array + and namedtuple_as_object and tuple_as_array and not iterable_as_array and not bigint_as_string and not sort_keys and not item_sort_key and not for_json and not ignore_nan and int_as_string_bitcount is None @@ -258,6 +263,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, default=default, use_decimal=use_decimal, namedtuple_as_object=namedtuple_as_object, tuple_as_array=tuple_as_array, + iterable_as_array=iterable_as_array, bigint_as_string=bigint_as_string, sort_keys=sort_keys, item_sort_key=item_sort_key, @@ -276,7 +282,8 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, encoding='utf-8', default=None, use_decimal=True, namedtuple_as_object=True, tuple_as_array=True, bigint_as_string=False, sort_keys=False, item_sort_key=None, - for_json=False, ignore_nan=False, int_as_string_bitcount=None, **kw): + for_json=False, ignore_nan=False, int_as_string_bitcount=None, + iterable_as_array=False, **kw): """Serialize ``obj`` to a JSON formatted ``str``. If ``skipkeys`` is false then ``dict`` keys that are not basic types @@ -324,6 +331,10 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, If *tuple_as_array* is true (default: ``True``), :class:`tuple` (and subclasses) will be encoded as JSON arrays. + If *iterable_as_array* is true (default: ``False``), + any object not in the above table that implements ``__iter__()`` + will be encoded as a JSON array. + If *bigint_as_string* is true (not the default), ints 2**53 and higher or lower than -2**53 will be encoded as strings. This is to avoid the rounding that happens in Javascript otherwise. @@ -356,12 +367,11 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, """ # cached encoder - if ( - not skipkeys and ensure_ascii and + if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and encoding == 'utf-8' and default is None and use_decimal - and namedtuple_as_object and tuple_as_array + and namedtuple_as_object and tuple_as_array and not iterable_as_array and not bigint_as_string and not sort_keys and not item_sort_key and not for_json and not ignore_nan and int_as_string_bitcount is None @@ -377,6 +387,7 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, use_decimal=use_decimal, namedtuple_as_object=namedtuple_as_object, tuple_as_array=tuple_as_array, + iterable_as_array=iterable_as_array, bigint_as_string=bigint_as_string, sort_keys=sort_keys, item_sort_key=item_sort_key, diff --git a/wakatime/packages/simplejson/_speedups.c b/wakatime/packages/simplejson/_speedups.c index 0b2d81c..fb68e35 100644 --- a/wakatime/packages/simplejson/_speedups.c +++ b/wakatime/packages/simplejson/_speedups.c @@ -10,6 +10,7 @@ #define PyString_AS_STRING PyBytes_AS_STRING #define PyString_FromStringAndSize PyBytes_FromStringAndSize #define PyInt_Check(obj) 0 +#define PyInt_CheckExact(obj) 0 #define JSON_UNICHR Py_UCS4 #define JSON_InternFromString PyUnicode_InternFromString #define JSON_Intern_GET_SIZE PyUnicode_GET_SIZE @@ -168,6 +169,7 @@ typedef struct _PyEncoderObject { int use_decimal; int namedtuple_as_object; int tuple_as_array; + int iterable_as_array; PyObject *max_long_size; PyObject *min_long_size; PyObject *item_sort_key; @@ -660,7 +662,20 @@ encoder_stringify_key(PyEncoderObject *s, PyObject *key) return _encoded_const(key); } else if (PyInt_Check(key) || PyLong_Check(key)) { - return PyObject_Str(key); + if (!(PyInt_CheckExact(key) || PyLong_CheckExact(key))) { + /* See #118, do not trust custom str/repr */ + PyObject *res; + PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, key, NULL); + if (tmp == NULL) { + return NULL; + } + res = PyObject_Str(tmp); + Py_DECREF(tmp); + return res; + } + else { + return PyObject_Str(key); + } } else if (s->use_decimal && PyObject_TypeCheck(key, (PyTypeObject *)s->Decimal)) { return PyObject_Str(key); @@ -2567,7 +2582,6 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int encoder_init(PyObject *self, PyObject *args, PyObject *kwds) { - /* initialize Encoder object */ static char *kwlist[] = { "markers", "default", @@ -2582,30 +2596,32 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) "use_decimal", "namedtuple_as_object", "tuple_as_array", + "iterable_as_array" "int_as_string_bitcount", "item_sort_key", "encoding", "for_json", "ignore_nan", "Decimal", + "iterable_as_array", NULL}; PyEncoderObject *s; PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo; - PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array; + PyObject *use_decimal, *namedtuple_as_object, *tuple_as_array, *iterable_as_array; PyObject *int_as_string_bitcount, *item_sort_key, *encoding, *for_json; PyObject *ignore_nan, *Decimal; assert(PyEncoder_Check(self)); s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOO:make_encoder", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOOOOOOOOOOO:make_encoder", kwlist, &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal, &namedtuple_as_object, &tuple_as_array, &int_as_string_bitcount, &item_sort_key, &encoding, &for_json, - &ignore_nan, &Decimal)) + &ignore_nan, &Decimal, &iterable_as_array)) return -1; Py_INCREF(markers); @@ -2635,9 +2651,10 @@ encoder_init(PyObject *self, PyObject *args, PyObject *kwds) s->use_decimal = PyObject_IsTrue(use_decimal); s->namedtuple_as_object = PyObject_IsTrue(namedtuple_as_object); s->tuple_as_array = PyObject_IsTrue(tuple_as_array); + s->iterable_as_array = PyObject_IsTrue(iterable_as_array); if (PyInt_Check(int_as_string_bitcount) || PyLong_Check(int_as_string_bitcount)) { static const unsigned int long_long_bitsize = SIZEOF_LONG_LONG * 8; - int int_as_string_bitcount_val = PyLong_AsLong(int_as_string_bitcount); + int int_as_string_bitcount_val = (int)PyLong_AsLong(int_as_string_bitcount); if (int_as_string_bitcount_val > 0 && int_as_string_bitcount_val < long_long_bitsize) { s->max_long_size = PyLong_FromUnsignedLongLong(1ULL << int_as_string_bitcount_val); s->min_long_size = PyLong_FromLongLong(-1LL << int_as_string_bitcount_val); @@ -2800,7 +2817,20 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj) } } /* Use a better float format here? */ - return PyObject_Repr(obj); + if (PyFloat_CheckExact(obj)) { + return PyObject_Repr(obj); + } + else { + /* See #118, do not trust custom str/repr */ + PyObject *res; + PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyFloat_Type, obj, NULL); + if (tmp == NULL) { + return NULL; + } + res = PyObject_Repr(tmp); + Py_DECREF(tmp); + return res; + } } static PyObject * @@ -2840,7 +2870,21 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss rv = _steal_accumulate(rval, encoded); } else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded = PyObject_Str(obj); + PyObject *encoded; + if (PyInt_CheckExact(obj) || PyLong_CheckExact(obj)) { + encoded = PyObject_Str(obj); + } + else { + /* See #118, do not trust custom str/repr */ + PyObject *tmp = PyObject_CallFunctionObjArgs((PyObject *)&PyLong_Type, obj, NULL); + if (tmp == NULL) { + encoded = NULL; + } + else { + encoded = PyObject_Str(tmp); + Py_DECREF(tmp); + } + } if (encoded != NULL) { encoded = maybe_quote_bigint(s, encoded, obj); if (encoded == NULL) @@ -2895,6 +2939,16 @@ encoder_listencode_obj(PyEncoderObject *s, JSON_Accu *rval, PyObject *obj, Py_ss else { PyObject *ident = NULL; PyObject *newobj; + if (s->iterable_as_array) { + newobj = PyObject_GetIter(obj); + if (newobj == NULL) + PyErr_Clear(); + else { + rv = encoder_listencode_list(s, rval, newobj, indent_level); + Py_DECREF(newobj); + break; + } + } if (s->markers != Py_None) { int has_key; ident = PyLong_FromVoidPtr(obj); diff --git a/wakatime/packages/simplejson/encoder.py b/wakatime/packages/simplejson/encoder.py index db18244..d771bb4 100644 --- a/wakatime/packages/simplejson/encoder.py +++ b/wakatime/packages/simplejson/encoder.py @@ -3,7 +3,8 @@ from __future__ import absolute_import import re from operator import itemgetter -from decimal import Decimal +# Do not import Decimal directly to avoid reload issues +import decimal from .compat import u, unichr, binary_type, string_types, integer_types, PY3 def _import_speedups(): try: @@ -123,7 +124,7 @@ class JSONEncoder(object): use_decimal=True, namedtuple_as_object=True, tuple_as_array=True, bigint_as_string=False, item_sort_key=None, for_json=False, ignore_nan=False, - int_as_string_bitcount=None): + int_as_string_bitcount=None, iterable_as_array=False): """Constructor for JSONEncoder, with sensible defaults. If skipkeys is false, then it is a TypeError to attempt @@ -178,6 +179,10 @@ class JSONEncoder(object): If tuple_as_array is true (the default), tuple (and subclasses) will be encoded as JSON arrays. + If *iterable_as_array* is true (default: ``False``), + any object not in the above table that implements ``__iter__()`` + will be encoded as a JSON array. + If bigint_as_string is true (not the default), ints 2**53 and higher or lower than -2**53 will be encoded as strings. This is to avoid the rounding that happens in Javascript otherwise. @@ -209,6 +214,7 @@ class JSONEncoder(object): self.use_decimal = use_decimal self.namedtuple_as_object = namedtuple_as_object self.tuple_as_array = tuple_as_array + self.iterable_as_array = iterable_as_array self.bigint_as_string = bigint_as_string self.item_sort_key = item_sort_key self.for_json = for_json @@ -311,6 +317,9 @@ class JSONEncoder(object): elif o == _neginf: text = '-Infinity' else: + if type(o) != float: + # See #118, do not trust custom str/repr + o = float(o) return _repr(o) if ignore_nan: @@ -334,7 +343,7 @@ class JSONEncoder(object): self.namedtuple_as_object, self.tuple_as_array, int_as_string_bitcount, self.item_sort_key, self.encoding, self.for_json, - self.ignore_nan, Decimal) + self.ignore_nan, decimal.Decimal, self.iterable_as_array) else: _iterencode = _make_iterencode( markers, self.default, _encoder, self.indent, floatstr, @@ -343,7 +352,7 @@ class JSONEncoder(object): self.namedtuple_as_object, self.tuple_as_array, int_as_string_bitcount, self.item_sort_key, self.encoding, self.for_json, - Decimal=Decimal) + self.iterable_as_array, Decimal=decimal.Decimal) try: return _iterencode(o, 0) finally: @@ -382,11 +391,12 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _use_decimal, _namedtuple_as_object, _tuple_as_array, _int_as_string_bitcount, _item_sort_key, _encoding,_for_json, + _iterable_as_array, ## HACK: hand-optimized bytecode; turn globals into locals _PY3=PY3, ValueError=ValueError, string_types=string_types, - Decimal=Decimal, + Decimal=None, dict=dict, float=float, id=id, @@ -395,7 +405,10 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, list=list, str=str, tuple=tuple, + iter=iter, ): + if _use_decimal and Decimal is None: + Decimal = decimal.Decimal if _item_sort_key and not callable(_item_sort_key): raise TypeError("item_sort_key must be None or callable") elif _sort_keys and not _item_sort_key: @@ -412,6 +425,9 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, or _int_as_string_bitcount < 1 ) + if type(value) not in integer_types: + # See #118, do not trust custom str/repr + value = int(value) if ( skip_quoting or (-1 << _int_as_string_bitcount) @@ -501,6 +517,9 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, elif key is None: key = 'null' elif isinstance(key, integer_types): + if type(key) not in integer_types: + # See #118, do not trust custom str/repr + key = int(key) key = str(key) elif _use_decimal and isinstance(key, Decimal): key = str(key) @@ -634,6 +653,16 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, elif _use_decimal and isinstance(o, Decimal): yield str(o) else: + while _iterable_as_array: + # Markers are not checked here because it is valid for + # an iterable to return self. + try: + o = iter(o) + except TypeError: + break + for chunk in _iterencode_list(o, _current_indent_level): + yield chunk + return if markers is not None: markerid = id(o) if markerid in markers: diff --git a/wakatime/packages/simplejson/tests/__init__.py b/wakatime/packages/simplejson/tests/__init__.py index c7551e8..8c1a4f1 100644 --- a/wakatime/packages/simplejson/tests/__init__.py +++ b/wakatime/packages/simplejson/tests/__init__.py @@ -62,6 +62,7 @@ def all_tests_suite(): 'simplejson.tests.test_namedtuple', 'simplejson.tests.test_tool', 'simplejson.tests.test_for_json', + 'simplejson.tests.test_subclass', ])) suite = get_suite() import simplejson diff --git a/wakatime/packages/simplejson/tests/test_iterable.py b/wakatime/packages/simplejson/tests/test_iterable.py new file mode 100644 index 0000000..4b37d00 --- /dev/null +++ b/wakatime/packages/simplejson/tests/test_iterable.py @@ -0,0 +1,31 @@ +import unittest +from StringIO import StringIO + +import simplejson as json + +def iter_dumps(obj, **kw): + return ''.join(json.JSONEncoder(**kw).iterencode(obj)) + +def sio_dump(obj, **kw): + sio = StringIO() + json.dumps(obj, **kw) + return sio.getvalue() + +class TestIterable(unittest.TestCase): + def test_iterable(self): + l = [1, 2, 3] + for dumps in (json.dumps, iter_dumps, sio_dump): + expect = dumps(l) + default_expect = dumps(sum(l)) + # Default is False + self.assertRaises(TypeError, dumps, iter(l)) + self.assertRaises(TypeError, dumps, iter(l), iterable_as_array=False) + self.assertEqual(expect, dumps(iter(l), iterable_as_array=True)) + # Ensure that the "default" gets called + self.assertEqual(default_expect, dumps(iter(l), default=sum)) + self.assertEqual(default_expect, dumps(iter(l), iterable_as_array=False, default=sum)) + # Ensure that the "default" does not get called + self.assertEqual( + default_expect, + dumps(iter(l), iterable_as_array=True, default=sum)) + \ No newline at end of file diff --git a/wakatime/packages/simplejson/tests/test_subclass.py b/wakatime/packages/simplejson/tests/test_subclass.py new file mode 100644 index 0000000..2bae3b6 --- /dev/null +++ b/wakatime/packages/simplejson/tests/test_subclass.py @@ -0,0 +1,37 @@ +from unittest import TestCase +import simplejson as json + +from decimal import Decimal + +class AlternateInt(int): + def __repr__(self): + return 'invalid json' + __str__ = __repr__ + + +class AlternateFloat(float): + def __repr__(self): + return 'invalid json' + __str__ = __repr__ + + +# class AlternateDecimal(Decimal): +# def __repr__(self): +# return 'invalid json' + + +class TestSubclass(TestCase): + def test_int(self): + self.assertEqual(json.dumps(AlternateInt(1)), '1') + self.assertEqual(json.dumps(AlternateInt(-1)), '-1') + self.assertEqual(json.loads(json.dumps({AlternateInt(1): 1})), {'1': 1}) + + def test_float(self): + self.assertEqual(json.dumps(AlternateFloat(1.0)), '1.0') + self.assertEqual(json.dumps(AlternateFloat(-1.0)), '-1.0') + self.assertEqual(json.loads(json.dumps({AlternateFloat(1.0): 1})), {'1.0': 1}) + + # NOTE: Decimal subclasses are not supported as-is + # def test_decimal(self): + # self.assertEqual(json.dumps(AlternateDecimal('1.0')), '1.0') + # self.assertEqual(json.dumps(AlternateDecimal('-1.0')), '-1.0') diff --git a/wakatime/packages/simplejson/tests/test_tuple.py b/wakatime/packages/simplejson/tests/test_tuple.py index a6a9910..4ad7b0e 100644 --- a/wakatime/packages/simplejson/tests/test_tuple.py +++ b/wakatime/packages/simplejson/tests/test_tuple.py @@ -45,7 +45,3 @@ class TestTuples(unittest.TestCase): self.assertEqual( json.dumps(repr(t)), sio.getvalue()) - -class TestNamedTuple(unittest.TestCase): - def test_namedtuple_dump(self): - pass