Skip to content

Commit 01ada39

Browse files
vstinnervstinner
committed
Issue python#25267: The UTF-8 encoder is now up to 75 times as fast for error
handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. Patch co-written with Serhiy Storchaka.
1 parent 29a1445 commit 01ada39

File tree

5 files changed

+135
-63
lines changed

5 files changed

+135
-63
lines changed

‎Doc/whatsnew/3.6.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@ Optimizations
120120
* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error
121121
error ``surrogateescape``.
122122

123+
* The UTF-8 encoder is now up to 75 times as fast for error handlers:
124+
``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
125+
123126

124127
Build and C API Changes
125128
=======================

‎Lib/test/test_codecs.py‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,12 @@ def test_lone_surrogates(self):
361361
self.assertEqual("[\uDC80]".encode(self.encoding, "replace"),
362362
"[?]".encode(self.encoding))
363363

364+
# sequential surrogate characters
365+
self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"),
366+
"[]".encode(self.encoding))
367+
self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"),
368+
"[??]".encode(self.encoding))
369+
364370
bom = "".encode(self.encoding)
365371
for before, after in [("\U00010fff", "A"), ("[", "]"),
366372
("A", "\U00010fff")]:
@@ -753,6 +759,7 @@ class UTF8Test(ReadTest, unittest.TestCase):
753759
encoding = "utf-8"
754760
ill_formed_sequence = b"\xed\xb2\x80"
755761
ill_formed_sequence_replace = "\ufffd" * 3
762+
BOM = b''
756763

757764
def test_partial(self):
758765
self.check_partial(
@@ -785,23 +792,32 @@ def test_lone_surrogates(self):
785792
super().test_lone_surrogates()
786793
# not sure if this is making sense for
787794
# UTF-16 and UTF-32
788-
self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"),
789-
b'[\x80]')
795+
self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"),
796+
self.BOM + b'[\x80]')
797+
798+
with self.assertRaises(UnicodeEncodeError) as cm:
799+
"[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape")
800+
exc = cm.exception
801+
self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF')
790802

791803
def test_surrogatepass_handler(self):
792-
self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"),
793-
b"abc\xed\xa0\x80def")
794-
self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"),
804+
self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"),
805+
self.BOM + b"abc\xed\xa0\x80def")
806+
self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"),
807+
self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80")
808+
self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"),
809+
self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]')
810+
811+
self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"),
795812
"abc\ud800def")
796-
self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"),
797-
b"\xf0\x90\xbf\xbf\xed\xa0\x80")
798-
self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"),
813+
self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"),
799814
"\U00010fff\uD800")
815+
800816
self.assertTrue(codecs.lookup_error("surrogatepass"))
801817
with self.assertRaises(UnicodeDecodeError):
802-
b"abc\xed\xa0".decode("utf-8", "surrogatepass")
818+
b"abc\xed\xa0".decode(self.encoding, "surrogatepass")
803819
with self.assertRaises(UnicodeDecodeError):
804-
b"abc\xed\xa0z".decode("utf-8", "surrogatepass")
820+
b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
805821

806822

807823
@unittest.skipUnless(sys.platform == 'win32',
@@ -1008,6 +1024,7 @@ def test_bad_args(self):
10081024

10091025
class UTF8SigTest(UTF8Test, unittest.TestCase):
10101026
encoding = "utf-8-sig"
1027+
BOM = codecs.BOM_UTF8
10111028

10121029
def test_partial(self):
10131030
self.check_partial(

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ Release date: XXXX-XX-XX
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error
14+
handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``.
15+
Patch co-written with Serhiy Storchaka.
16+
1317
- Issue #25280: Import trace messages emitted in verbose (-v) mode are no
1418
longer formatted twice.
1519

‎Objects/stringlib/codecs.h‎

Lines changed: 96 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -268,9 +268,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
268268
Py_ssize_t nallocated; /* number of result bytes allocated */
269269
Py_ssize_t nneeded; /* number of result bytes needed */
270270
#if STRINGLIB_SIZEOF_CHAR > 1
271-
PyObject *errorHandler = NULL;
271+
PyObject *error_handler_obj = NULL;
272272
PyObject *exc = NULL;
273273
PyObject *rep = NULL;
274+
_Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
274275
#endif
275276
#if STRINGLIB_SIZEOF_CHAR == 1
276277
const Py_ssize_t max_char_size = 2;
@@ -326,72 +327,116 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
326327
}
327328
#if STRINGLIB_SIZEOF_CHAR > 1
328329
else if (Py_UNICODE_IS_SURROGATE(ch)) {
329-
Py_ssize_t newpos;
330-
Py_ssize_t repsize, k, startpos;
330+
Py_ssize_t startpos, endpos, newpos;
331+
Py_ssize_t repsize, k;
332+
if (error_handler == _Py_ERROR_UNKNOWN)
333+
error_handler = get_error_handler(errors);
334+
331335
startpos = i-1;
332-
rep = unicode_encode_call_errorhandler(
333-
errors, &errorHandler, "utf-8", "surrogates not allowed",
334-
unicode, &exc, startpos, startpos+1, &newpos);
335-
if (!rep)
336-
goto error;
337-
338-
if (PyBytes_Check(rep))
339-
repsize = PyBytes_GET_SIZE(rep);
340-
else
341-
repsize = PyUnicode_GET_LENGTH(rep);
336+
endpos = startpos+1;
337+
338+
while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos]))
339+
endpos++;
340+
341+
switch (error_handler)
342+
{
343+
case _Py_ERROR_REPLACE:
344+
memset(p, '?', endpos - startpos);
345+
p += (endpos - startpos);
346+
/* fall through the ignore handler */
347+
case _Py_ERROR_IGNORE:
348+
i += (endpos - startpos - 1);
349+
break;
342350

343-
if (repsize > max_char_size) {
344-
Py_ssize_t offset;
345351

346-
if (result == NULL)
347-
offset = p - stackbuf;
348-
else
349-
offset = p - PyBytes_AS_STRING(result);
352+
case _Py_ERROR_SURROGATEPASS:
353+
for (k=startpos; k<endpos; k++) {
354+
ch = data[k];
355+
*p++ = (char)(0xe0 | (ch >> 12));
356+
*p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
357+
*p++ = (char)(0x80 | (ch & 0x3f));
358+
}
359+
i += (endpos - startpos - 1);
360+
break;
350361

351-
if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
352-
/* integer overflow */
353-
PyErr_NoMemory();
354-
goto error;
362+
case _Py_ERROR_SURROGATEESCAPE:
363+
for (k=startpos; k<endpos; k++) {
364+
ch = data[k];
365+
if (!(0xDC80 <= ch && ch <= 0xDCFF))
366+
break;
367+
*p++ = (char)(ch & 0xff);
355368
}
356-
nallocated += repsize - max_char_size;
357-
if (result != NULL) {
358-
if (_PyBytes_Resize(&result, nallocated) < 0)
359-
goto error;
360-
} else {
361-
result = PyBytes_FromStringAndSize(NULL, nallocated);
369+
if (k >= endpos) {
370+
i += (endpos - startpos - 1);
371+
break;
372+
}
373+
startpos = k;
374+
assert(startpos < endpos);
375+
/* fall through the default handler */
376+
377+
default:
378+
rep = unicode_encode_call_errorhandler(
379+
errors, &error_handler_obj, "utf-8", "surrogates not allowed",
380+
unicode, &exc, startpos, endpos, &newpos);
381+
if (!rep)
382+
goto error;
383+
384+
if (PyBytes_Check(rep))
385+
repsize = PyBytes_GET_SIZE(rep);
386+
else
387+
repsize = PyUnicode_GET_LENGTH(rep);
388+
389+
if (repsize > max_char_size) {
390+
Py_ssize_t offset;
391+
362392
if (result == NULL)
393+
offset = p - stackbuf;
394+
else
395+
offset = p - PyBytes_AS_STRING(result);
396+
397+
if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) {
398+
/* integer overflow */
399+
PyErr_NoMemory();
363400
goto error;
364-
Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
401+
}
402+
nallocated += repsize - max_char_size;
403+
if (result != NULL) {
404+
if (_PyBytes_Resize(&result, nallocated) < 0)
405+
goto error;
406+
} else {
407+
result = PyBytes_FromStringAndSize(NULL, nallocated);
408+
if (result == NULL)
409+
goto error;
410+
Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
411+
}
412+
p = PyBytes_AS_STRING(result) + offset;
365413
}
366-
p = PyBytes_AS_STRING(result) + offset;
367-
}
368-
369-
if (PyBytes_Check(rep)) {
370-
char *prep = PyBytes_AS_STRING(rep);
371-
for(k = repsize; k > 0; k--)
372-
*p++ = *prep++;
373-
} else /* rep is unicode */ {
374-
enum PyUnicode_Kind repkind;
375-
void *repdata;
376414

377-
if (PyUnicode_READY(rep) < 0)
378-
goto error;
379-
repkind = PyUnicode_KIND(rep);
380-
repdata = PyUnicode_DATA(rep);
415+
if (PyBytes_Check(rep)) {
416+
memcpy(p, PyBytes_AS_STRING(rep), repsize);
417+
p += repsize;
418+
}
419+
else {
420+
/* rep is unicode */
421+
if (PyUnicode_READY(rep) < 0)
422+
goto error;
381423

382-
for(k=0; k<repsize; k++) {
383-
Py_UCS4 c = PyUnicode_READ(repkind, repdata, k);
384-
if (0x80 <= c) {
424+
if (!PyUnicode_IS_ASCII(rep)) {
385425
raise_encode_exception(&exc, "utf-8",
386426
unicode,
387427
i-1, i,
388428
"surrogates not allowed");
389429
goto error;
390430
}
391-
*p++ = (char)c;
431+
432+
assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND);
433+
memcpy(p, PyUnicode_DATA(rep), repsize);
434+
p += repsize;
392435
}
436+
Py_CLEAR(rep);
437+
438+
i = newpos;
393439
}
394-
Py_CLEAR(rep);
395440
}
396441
else
397442
#if STRINGLIB_SIZEOF_CHAR > 2
@@ -430,15 +475,15 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
430475
}
431476

432477
#if STRINGLIB_SIZEOF_CHAR > 1
433-
Py_XDECREF(errorHandler);
478+
Py_XDECREF(error_handler_obj);
434479
Py_XDECREF(exc);
435480
#endif
436481
return result;
437482

438483
#if STRINGLIB_SIZEOF_CHAR > 1
439484
error:
440485
Py_XDECREF(rep);
441-
Py_XDECREF(errorHandler);
486+
Py_XDECREF(error_handler_obj);
442487
Py_XDECREF(exc);
443488
Py_XDECREF(result);
444489
return NULL;

‎Objects/unicodeobject.c‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,7 @@ typedef enum {
297297
_Py_ERROR_UNKNOWN=0,
298298
_Py_ERROR_STRICT,
299299
_Py_ERROR_SURROGATEESCAPE,
300+
_Py_ERROR_SURROGATEPASS,
300301
_Py_ERROR_REPLACE,
301302
_Py_ERROR_IGNORE,
302303
_Py_ERROR_XMLCHARREFREPLACE,
@@ -312,6 +313,8 @@ get_error_handler(const char *errors)
312313
return _Py_ERROR_STRICT;
313314
if (strcmp(errors, "surrogateescape") == 0)
314315
return _Py_ERROR_SURROGATEESCAPE;
316+
if (strcmp(errors, "surrogatepass") == 0)
317+
return _Py_ERROR_SURROGATEPASS;
315318
if (strcmp(errors, "ignore") == 0)
316319
return _Py_ERROR_IGNORE;
317320
if (strcmp(errors, "replace") == 0)
@@ -6479,8 +6482,8 @@ unicode_encode_ucs1(PyObject *unicode,
64796482
goto onError;
64806483

64816484
case _Py_ERROR_REPLACE:
6482-
while (collstart++ < collend)
6483-
*str++ = '?';
6485+
memset(str, '?', collend - collstart);
6486+
str += (collend - collstart);
64846487
/* fall through ignore error handler */
64856488
case _Py_ERROR_IGNORE:
64866489
pos = collend;

0 commit comments

Comments
 (0)