Skip to content

Commit 08e550c

Browse files
gh-69643: Fix sorting keys of different types in json
json.dumps() and json.dump() with sort_keys=True failed for keys of different basic types (str, int, float, bool and None), and for unsupported keys skipped due to skipkeys. Now keys of mixed types are sorted by groups: strings, numbers (booleans are numbers too) and None. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 5141621 commit 08e550c

6 files changed

Lines changed: 165 additions & 7 deletions

File tree

Doc/library/json.rst

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,10 @@ Basic Usage
243243
.. versionchanged:: 3.6
244244
All optional parameters are now :ref:`keyword-only <keyword-only_parameter>`.
245245

246+
.. versionchanged:: next
247+
*sort_keys* no longer fails for keys of different basic types
248+
or for unsupported keys skipped due to *skipkeys*.
249+
246250

247251
.. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \
248252
check_circular=True, allow_nan=True, cls=None, \
@@ -536,6 +540,11 @@ Encoders and Decoders
536540
If *sort_keys* is true (default: ``False``), then the output of dictionaries
537541
will be sorted by key; this is useful for regression tests to ensure that
538542
JSON serializations can be compared on a day-to-day basis.
543+
Keys of mixed types are sorted by groups: strings, numbers and ``None``.
544+
545+
.. versionchanged:: next
546+
*sort_keys* no longer fails for keys of different basic types
547+
or for unsupported keys skipped due to *skipkeys*.
539548

540549
If *indent* is a non-negative integer or string, then JSON array elements and
541550
object members will be pretty-printed with that indent level. An indent level

Lib/json/encoder.py

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,32 @@ def floatstr(o, allow_nan=self.allow_nan,
261261
self.skipkeys, _one_shot)
262262
return _iterencode(o, 0)
263263

264+
def _sort_items(items, skipkeys):
265+
"""Sort (key, value) pairs in separate groups, because keys of
266+
different types are not comparable: strings, numbers and ``None``.
267+
268+
Unsupported keys are skipped if *skipkeys* is true and reported
269+
otherwise.
270+
"""
271+
strings = []
272+
nones = []
273+
numbers = []
274+
for item in items:
275+
key, value = item
276+
if isinstance(key, str):
277+
strings.append(item)
278+
elif key is None:
279+
nones.append(item)
280+
elif isinstance(key, (int, float)): # includes bool
281+
numbers.append(item)
282+
elif not skipkeys:
283+
raise TypeError(f'keys must be str, int, float, bool or None, '
284+
f'not {key.__class__.__name__}')
285+
strings.sort()
286+
numbers.sort()
287+
return strings + numbers + nones
288+
289+
264290
def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
265291
_key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
266292
):
@@ -343,7 +369,12 @@ def _iterencode_dict(dct, _current_indent_level):
343369
item_separator = _item_separator
344370
first = True
345371
if _sort_keys:
346-
items = sorted(dct.items())
372+
items = list(dct.items())
373+
try:
374+
items.sort()
375+
except TypeError:
376+
# Keys of different types are not comparable.
377+
items = _sort_items(items, _skipkeys)
347378
else:
348379
items = dct.items()
349380
for key, value in items:

Lib/test/test_json/test_dump.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,43 @@ def test_skipkeys_indent(self):
4242
v = {b'invalid_key': False, 'valid_key': True}
4343
self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}')
4444

45+
def test_dump_sort_keys_mixed_types(self):
46+
# Keys of different types are sorted in separate groups.
47+
self.assertEqual(
48+
self.dumps({1: 'a', 'z': 'b', 'a': 'c'}, sort_keys=True),
49+
'{"a": "c", "z": "b", "1": "a"}')
50+
self.assertEqual(
51+
self.dumps({None: 0, True: 1, False: 4, 2: 2, 'a': 3},
52+
sort_keys=True),
53+
'{"a": 3, "false": 4, "true": 1, "2": 2, "null": 0}')
54+
# Numbers are still sorted as numbers, and adding a string key
55+
# does not change their order.
56+
self.assertEqual(
57+
self.dumps({10: 1, 2: 2}, sort_keys=True),
58+
'{"2": 2, "10": 1}')
59+
self.assertEqual(
60+
self.dumps({10: 1, 2: 2, 'a': 3}, sort_keys=True),
61+
'{"a": 3, "2": 2, "10": 1}')
62+
# Unsupported keys are still reported, or skipped.
63+
with self.assertRaises(TypeError):
64+
self.dumps({(1, 2): 'x', 'z': 'b'}, sort_keys=True)
65+
self.assertEqual(
66+
self.dumps({(1, 2): 'x', 'z': 'b'}, skipkeys=True, sort_keys=True),
67+
'{"z": "b"}')
68+
69+
def test_dump_sort_keys_unsupported(self):
70+
# Unsupported keys are reported or skipped, whether or not they are
71+
# comparable with each other.
72+
for d in ({(2,): 1, (1,): 2}, # comparable
73+
{(2,): 1, (1,): 2, 'z': 3},
74+
{(1,): 1, ('a',): 2, 'z': 3}): # not comparable
75+
with self.subTest(d=d):
76+
with self.assertRaises(TypeError):
77+
self.dumps(d, sort_keys=True)
78+
self.assertEqual(
79+
self.dumps(d, skipkeys=True, sort_keys=True),
80+
'{"z": 3}' if 'z' in d else '{}')
81+
4582
def test_encode_truefalse(self):
4683
self.assertEqual(self.dumps(
4784
{True: False, False: True}, sort_keys=True),

Lib/test/test_json/test_speedups.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,6 @@ def test(name):
7878
self.assertRaises(ZeroDivisionError, test, 'allow_nan')
7979
self.assertRaises(ZeroDivisionError, test, 'sort_keys')
8080

81-
def test_unsortable_keys(self):
82-
with self.assertRaises(TypeError):
83-
self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'})
84-
8581
def test_current_indent_level(self):
8682
enc = self.json.encoder.c_make_encoder(
8783
markers=None,
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
:func:`json.dump` and :func:`json.dumps` with ``sort_keys=True`` no longer
2+
fail for keys of different basic types or for unsupported keys skipped due
3+
to *skipkeys*. Keys of mixed types are sorted by groups: strings, numbers
4+
and ``None``.

Modules/_json.c

Lines changed: 83 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1793,6 +1793,76 @@ _encoder_iterate_dict_lock_held(PyEncoderObject *s, PyUnicodeWriter *writer,
17931793
return 0;
17941794
}
17951795

1796+
/* Sort the (key, value) pairs in separate groups, because keys of
1797+
different types are not comparable: strings, numbers and None.
1798+
Unsupported keys are skipped if skipkeys is true and reported otherwise.
1799+
Return a new list, or NULL on error. */
1800+
static PyObject *
1801+
encoder_sort_items(PyObject *items, int skipkeys)
1802+
{
1803+
enum {STRINGS, NUMBERS, NONES, NGROUPS};
1804+
PyObject *groups[NGROUPS] = {NULL};
1805+
PyObject *result = NULL;
1806+
1807+
for (int i = 0; i < NGROUPS; i++) {
1808+
groups[i] = PyList_New(0);
1809+
if (groups[i] == NULL) {
1810+
goto done;
1811+
}
1812+
}
1813+
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) {
1814+
PyObject *item = PyList_GET_ITEM(items, i);
1815+
if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) {
1816+
PyErr_SetString(PyExc_ValueError, "items must return 2-tuples");
1817+
goto done;
1818+
}
1819+
PyObject *key = PyTuple_GET_ITEM(item, 0);
1820+
int group;
1821+
if (PyUnicode_Check(key)) {
1822+
group = STRINGS;
1823+
}
1824+
else if (key == Py_None) {
1825+
group = NONES;
1826+
}
1827+
else if (PyLong_Check(key) || PyFloat_Check(key)) { // includes bool
1828+
group = NUMBERS;
1829+
}
1830+
else if (skipkeys) {
1831+
continue;
1832+
}
1833+
else {
1834+
PyErr_Format(PyExc_TypeError,
1835+
"keys must be str, int, float, bool or None, "
1836+
"not %.100s", Py_TYPE(key)->tp_name);
1837+
goto done;
1838+
}
1839+
if (PyList_Append(groups[group], item) < 0) {
1840+
goto done;
1841+
}
1842+
}
1843+
/* There is at most one None key. */
1844+
if (PyList_Sort(groups[STRINGS]) < 0 ||
1845+
PyList_Sort(groups[NUMBERS]) < 0)
1846+
{
1847+
goto done;
1848+
}
1849+
result = groups[STRINGS];
1850+
groups[STRINGS] = NULL;
1851+
for (int i = STRINGS + 1; i < NGROUPS; i++) {
1852+
Py_ssize_t size = PyList_GET_SIZE(result);
1853+
if (PyList_SetSlice(result, size, size, groups[i]) < 0) {
1854+
Py_CLEAR(result);
1855+
goto done;
1856+
}
1857+
}
1858+
1859+
done:
1860+
for (int i = 0; i < NGROUPS; i++) {
1861+
Py_XDECREF(groups[i]);
1862+
}
1863+
return result;
1864+
}
1865+
17961866
static int
17971867
encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,
17981868
PyObject *dct,
@@ -1837,10 +1907,21 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer,
18371907

18381908
if (s->sort_keys || !PyAnyDict_CheckExact(dct)) {
18391909
PyObject *items = PyMapping_Items(dct);
1840-
if (items == NULL || (s->sort_keys && PyList_Sort(items) < 0)) {
1841-
Py_XDECREF(items);
1910+
if (items == NULL) {
18421911
goto bail;
18431912
}
1913+
if (s->sort_keys && PyList_Sort(items) < 0) {
1914+
if (!PyErr_ExceptionMatches(PyExc_TypeError)) {
1915+
Py_DECREF(items);
1916+
goto bail;
1917+
}
1918+
/* Keys of different types are not comparable. */
1919+
PyErr_Clear();
1920+
Py_SETREF(items, encoder_sort_items(items, s->skipkeys));
1921+
if (items == NULL) {
1922+
goto bail;
1923+
}
1924+
}
18441925

18451926
int result;
18461927
Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(items);

0 commit comments

Comments
 (0)