From 1a728acbf29482220ddabb033e434638bfa5f12c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 11 Sep 2026 11:57:41 +0300 Subject: [PATCH 1/3] gh-156955: Speed up csv.writer by caching the set of special characters Cache in the dialect a 128-bit set of ASCII characters which need quoting or escaping (delimiter, quotechar, escapechar, '\r', '\n' and characters of lineterminator) and a flag whether any of them is non-ASCII. Testing a character is now one bit test instead of five comparisons and a call to PyUnicode_FindChar(). Co-Authored-By: Claude Opus 5 (1M context) --- Lib/test/test_csv.py | 23 ++++++- ...-09-11-12-00-00.gh-issue-156955.bLtmAp.rst | 3 + Modules/_csv.c | 66 ++++++++++++++++--- 3 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 73e282d1abf7177..36fa7e3572a5ac2 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -227,6 +227,18 @@ def test_write_quoting(self): quoting = csv.QUOTE_STRINGS) self._write_test(['a','',None,1], '"a","",,"1"', quoting = csv.QUOTE_NOTNULL) + # FULLWIDTH QUOTATION MARK + self._write_test(['a', 1, 'p,q', 'r＂s', 'x！y'], + 'a,1,＂p,q＂,＂r＂＂s＂,x！y', + quotechar='＂') + + def test_write_delimiter(self): + self._write_test(['a', 1, 'p,q', 'x;y'], 'a,1,"p,q",x;y') + self._write_test(['a', 1, 'p;q', 'x,y'], 'a;1;"p;q";x,y', delimiter=';') + self._write_test(['a', 1, 'p\0q', 'x,y'], 'a\x001\0"p\0q"\0x,y', + delimiter='\0') + self._write_test(['a', 1, 'p🍌q', 'x🍍y'], 'a🍌1🍌"p🍌q"🍌x🍍y', + delimiter='🍌') def test_write_escape(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"', @@ -258,19 +270,26 @@ def test_write_escape(self): escapechar='\\', quoting=csv.QUOTE_MINIMAL) self._write_test(['C\\', '6', '7', 'X"'], 'C\\\\,6,7,"X"""', escapechar='\\', quoting=csv.QUOTE_MINIMAL) + # SYMBOL FOR ESCAPE + self._write_test(['a', 1, 'p,q', 'r\u241bs', 'x\u241ay'], + 'a,1,p\u241b,q,r\u241b\u241bs,x\u241ay', + escapechar='\u241b', quoting=csv.QUOTE_NONE) def test_write_lineterminator(self): - for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': + for lineterminator in ('\r\n', '\n', '\r', '!@#', '\0', + '\x85', '\u2028', '\U0001f600'): with self.subTest(lineterminator=lineterminator): with StringIO() as sio: writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) writer.writerow(['\r', '\n']) + writer.writerow([f'a{lineterminator[-1]}b', 'c']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' f'1,2{lineterminator}' - f'"\r","\n"{lineterminator}') + f'"\r","\n"{lineterminator}' + f'"a{lineterminator[-1]}b",c{lineterminator}') def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') diff --git a/Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst b/Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst new file mode 100644 index 000000000000000..2a452a2a6e16526 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-11-12-00-00.gh-issue-156955.bLtmAp.rst @@ -0,0 +1,3 @@ +Speed up :func:`csv.writer` by caching the set of characters that need +quoting or escaping in the dialect. Writing long fields is now up to 5 times +faster. diff --git a/Modules/_csv.c b/Modules/_csv.c index c640f2d36a84647..e383c2e3d115358 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -117,7 +117,12 @@ typedef struct { Py_UCS4 quotechar; /* quote character */ Py_UCS4 escapechar; /* escape character */ PyObject *lineterminator; /* string to write between records */ - + /* Cache for the writer: bit c is set if the ASCII character c needs + quoting or escaping (delimiter, quotechar, escapechar, '\r', '\n' + and the characters of lineterminator). */ + uint64_t special_chars[2]; + /* Whether any of the special characters is non-ASCII. */ + bool nonascii_special; } DialectObj; typedef struct { @@ -332,6 +337,55 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) return 0; } +static void +dialect_add_special_char(DialectObj *self, Py_UCS4 c) +{ + if (c == NOT_SET) { + return; + } + if (c < 128) { + self->special_chars[c / 64] |= (uint64_t)1 << (c % 64); + } + else { + self->nonascii_special = true; + } +} + +/* Fill the cache of special characters used by the writer. */ +static void +dialect_init_special_chars(DialectObj *self) +{ + self->special_chars[0] = self->special_chars[1] = 0; + self->nonascii_special = false; + dialect_add_special_char(self, self->delimiter); + dialect_add_special_char(self, self->quotechar); + dialect_add_special_char(self, self->escapechar); + dialect_add_special_char(self, '\r'); + dialect_add_special_char(self, '\n'); + PyObject *lt = self->lineterminator; + for (Py_ssize_t i = 0; i < PyUnicode_GET_LENGTH(lt); i++) { + dialect_add_special_char(self, PyUnicode_READ_CHAR(lt, i)); + } +} + +/* Whether the character needs quoting or escaping by the writer. */ +static inline int +dialect_is_special_char(DialectObj *self, Py_UCS4 c) +{ + if (c < 128) { + return (self->special_chars[c / 64] >> (c % 64)) & 1; + } + if (!self->nonascii_special) { + return 0; + } + return (c == self->delimiter || + c == self->quotechar || + c == self->escapechar || + PyUnicode_FindChar(self->lineterminator, c, 0, + PyUnicode_GET_LENGTH(self->lineterminator), + 1) >= 0); +} + static int dialect_check_quoting(int quoting) { @@ -558,6 +612,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { goto err; } + dialect_init_special_chars(self); ret = Py_NewRef(self); err: @@ -1208,14 +1263,7 @@ join_append_data(WriterObj *self, int field_kind, const void *field_data, Py_UCS4 c = PyUnicode_READ(field_kind, field_data, i); int want_escape = 0; - if (c == dialect->delimiter || - c == dialect->escapechar || - c == dialect->quotechar || - c == '\n' || - c == '\r' || - PyUnicode_FindChar( - dialect->lineterminator, c, 0, - PyUnicode_GET_LENGTH(dialect->lineterminator), 1) >= 0) { + if (dialect_is_special_char(dialect, c)) { if (dialect->quoting == QUOTE_NONE) want_escape = 1; else { From 79b2415d2f7bcda07bc306f96f6b6e5ba5c1d825 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 13 Sep 2026 10:47:21 +0300 Subject: [PATCH 2/3] Rename dialect_init_special_chars() to dialect_init_special_chars_cache() --- Modules/_csv.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Modules/_csv.c b/Modules/_csv.c index e383c2e3d115358..e3b6de37cc806ba 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -351,9 +351,8 @@ dialect_add_special_char(DialectObj *self, Py_UCS4 c) } } -/* Fill the cache of special characters used by the writer. */ static void -dialect_init_special_chars(DialectObj *self) +dialect_init_special_chars_cache(DialectObj *self) { self->special_chars[0] = self->special_chars[1] = 0; self->nonascii_special = false; @@ -612,7 +611,7 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { goto err; } - dialect_init_special_chars(self); + dialect_init_special_chars_cache(self); ret = Py_NewRef(self); err: From 38e579c4ea7f3019c34367a2a57b99ace1dba318 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 13 Sep 2026 10:59:20 +0300 Subject: [PATCH 3/3] Make dialect_is_special_char() return bool --- Modules/_csv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_csv.c b/Modules/_csv.c index e3b6de37cc806ba..6af66c3f09a03b1 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -368,14 +368,14 @@ dialect_init_special_chars_cache(DialectObj *self) } /* Whether the character needs quoting or escaping by the writer. */ -static inline int +static inline bool dialect_is_special_char(DialectObj *self, Py_UCS4 c) { if (c < 128) { return (self->special_chars[c / 64] >> (c % 64)) & 1; } if (!self->nonascii_special) { - return 0; + return false; } return (c == self->delimiter || c == self->quotechar ||