diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 0684820ccc6916..53934dc698f19e 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -443,13 +443,30 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.login_cram_md5(user, password) Force use of ``CRAM-MD5`` authentication when identifying the client to protect - the password. Will only work if the server ``CAPABILITY`` response includes the - phrase ``AUTH=CRAM-MD5``. + the password. It will only work if the server ``CAPABILITY`` response includes + the phrase ``AUTH=CRAM-MD5``. .. versionchanged:: 3.15 An :exc:`IMAP4.error` is raised if MD5 support is not available. +.. method:: IMAP4.login_plain(user, password) + + Authenticate using the ``PLAIN`` SASL mechanism (:rfc:`4616`). + + This is a plaintext authentication mechanism that can be used instead + of :meth:`login` when UTF-8 support is required (see :rfc:`6855`). + Since the credentials are only base64-encoded, not encrypted, this + method should only be used over a TLS-protected connection, such as + :class:`IMAP4_SSL` or after :meth:`starttls`. + + It will only work if the server supports the ``PLAIN`` mechanism, + which it need not advertise as ``AUTH=PLAIN`` in its ``CAPABILITY`` + response. + + .. versionadded:: next + + .. method:: IMAP4.logout() Shutdown connection to server. Returns server ``BYE`` response. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index ef48addaba03e9..3ce20311c42c84 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -60,11 +60,16 @@ or on combining URL components into a URL string. *missing_as_none* is true. Not defined component are represented an empty string (by default) or ``None`` if *missing_as_none* is true. - The components are not broken up - into smaller parts (for example, the network location is a single string), and % - escapes are not expanded. The delimiters as shown above are not part of the - result, except for a leading slash in the *path* component, which is retained if - present. For example: + The delimiters as shown above are not part of the result, except for a + leading slash in the *path* component, which is retained if present. + + Additionally, the netloc property is broken down into these additional + attributes added to the returned object: username, password, hostname, and + port. + + Percent-encoded sequences are not decoded. + + For example: .. doctest:: :options: +NORMALIZE_WHITESPACE diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index e8c530e19d2b53..e8c75d2f571061 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -239,6 +239,11 @@ imaplib a wrapper for the ``MOVE`` command (:rfc:`6851`). (Contributed by Serhiy Storchaka in :gh:`77508`.) +* Add the :meth:`~imaplib.IMAP4.login_plain` method, which authenticates + using the ``PLAIN`` SASL mechanism (:rfc:`4616`) and supports non-ASCII + user names and passwords. + (Contributed by Przemysław Buczkowski and Serhiy Storchaka in :gh:`89869`.) + logging ------- diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 40d2b7a309b640..b1cf8872314d97 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -762,6 +762,30 @@ def login(self, user, password): return typ, dat + def login_plain(self, user, password): + """Authenticate using the PLAIN SASL mechanism (RFC 4616). + + This is a plaintext authentication mechanism that can be used + instead of login() when UTF-8 support is required. Since the + credentials are only base64-encoded, not encrypted, it should + only be used over a TLS-protected connection. + + 'user' and 'password' can be strings (encoded to UTF-8) or + bytes-like objects. + """ + if isinstance(user, str): + user = user.encode('utf-8') + if isinstance(password, str): + password = password.encode('utf-8') + if b'\0' in user or b'\0' in password: + raise ValueError("NUL is not allowed in user name or password") + # An empty authorization identity (RFC 4616) makes the server + # derive it from the authentication identity; a non-empty one + # requests proxy authorization and is often rejected. + response = b'\0' + bytes(user) + b'\0' + bytes(password) + return self.authenticate("PLAIN", lambda _: response) + + def login_cram_md5(self, user, password): """ Force use of CRAM-MD5 authentication. diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 91bcf243e78e5b..11c6e21de90e69 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1294,9 +1294,9 @@ class SyncManager(BaseManager): class _SharedMemoryTracker: "Manages one or more shared memory segments." - def __init__(self, name, segment_names=[]): + def __init__(self, name, segment_names=None): self.shared_memory_context_name = name - self.segment_names = segment_names + self.segment_names = [] if segment_names is None else segment_names def register_segment(self, segment_name): "Adds the supplied shared memory block name to tracker." diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py index 742da6f8c788b4..e0bd7e211e7302 100644 --- a/Lib/test/test_free_threading/test_io.py +++ b/Lib/test/test_free_threading/test_io.py @@ -122,6 +122,35 @@ def sizeof(barrier, b, *ignore): class CBytesIOTest(ThreadSafetyMixin, TestCase): ioclass = io.BytesIO + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_concurrent_whole_buffer_read_and_resize(self): + shared = self.ioclass(b"x" * 64) + writers = 2 + readers = 8 + loops = 2000 + barrier = threading.Barrier(writers + readers) + + def writer(): + barrier.wait() + for i in range(loops): + shared.seek(0) + shared.write(b"a" * (64 + (i & 63))) + + def reader(): + barrier.wait() + for _ in range(loops): + shared.seek(0) + shared.read() + shared.seek(0) + shared.peek() + shared.getvalue() + + threads = [threading.Thread(target=writer) for _ in range(writers)] + threads += [threading.Thread(target=reader) for _ in range(readers)] + with threading_helper.start_threads(threads): + pass + class PyBytesIOTest(ThreadSafetyMixin, TestCase): ioclass = pyio.BytesIO diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index aba3f5e44f2566..a2ddbbf9121808 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -420,6 +420,15 @@ def cmd_AUTHENTICATE(self, tag, args): self._send_tagged(tag, 'NO', 'No access') +class AuthHandler_PLAIN(SimpleIMAPHandler): + capabilities = 'LOGINDISABLED AUTH=PLAIN' + def cmd_AUTHENTICATE(self, tag, args): + self.server.auth_args = args + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'Logged in.') + + class NewIMAPTestsMixin: client = None @@ -692,6 +701,35 @@ def test_login_cram_md5_blocked(self): with self.assertRaisesRegex(imaplib.IMAP4.error, msg): client.login_cram_md5("tim", b"tanstaaftanstaaf") + def test_login_plain_ascii(self): + client, server = self._setup(AuthHandler_PLAIN) + self.assertIn('AUTH=PLAIN', client.capabilities) + ret, _ = client.login_plain("prem", "pass") + self.assertEqual(ret, "OK") + self.assertEqual(server.auth_args, ['PLAIN']) + self.assertEqual(server.response, b'AHByZW0AcGFzcw==\r\n') + + def test_login_plain_utf8(self): + client, server = self._setup(AuthHandler_PLAIN) + self.assertIn('AUTH=PLAIN', client.capabilities) + ret, _ = client.login_plain("pręm", "żółć") + self.assertEqual(ret, "OK") + self.assertEqual(server.response, b'AHByxJltAMW8w7PFgsSH\r\n') + + def test_login_plain_bytes(self): + # bytes are accepted and sent verbatim (not re-encoded). + client, server = self._setup(AuthHandler_PLAIN) + ret, _ = client.login_plain(b'pr\xc4\x99m', b'\xc5\xbc\xc3\xb3\xc5\x82\xc4\x87') + self.assertEqual(ret, "OK") + self.assertEqual(server.response, b'AHByxJltAMW8w7PFgsSH\r\n') + + def test_login_plain_nul(self): + client, _ = self._setup(AuthHandler_PLAIN) + with self.assertRaises(ValueError): + client.login_plain('user\0', 'pass') + with self.assertRaises(ValueError): + client.login_plain('user', b'pa\0ss') + def test_aborted_authentication(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 5a93d263458046..f53525d77b7dba 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -939,7 +939,10 @@ def test_setstate(self): @support.cpython_only def test_sizeof(self): - basesize = support.calcobjsize('P2n2Pn') + if support.Py_GIL_DISABLED: + basesize = support.calcobjsize('P2n2Pni') + else: + basesize = support.calcobjsize('P2n2Pn') check = self.check_sizeof self.assertEqual(object.__sizeof__(io.BytesIO()), basesize) check(io.BytesIO(), basesize ) diff --git a/Misc/ACKS b/Misc/ACKS index 1f2049da56c2da..68fed1f68de593 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -260,6 +260,7 @@ Stan Bubrouski Brandt Bucher Curtis Bucher Colm Buckley +Przemysław Buczkowski Erik de Bueger Jan-Hein Bührman Marc Bürg diff --git a/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst b/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst new file mode 100644 index 00000000000000..ef61a7d27dd07d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst @@ -0,0 +1,3 @@ +Fix a data race in :class:`io.BytesIO` in free-threaded builds when +whole-buffer reads or peeks, or :meth:`~io.BytesIO.getvalue`, share the +internal buffer with concurrent writes. diff --git a/Misc/NEWS.d/next/Library/2026-07-05-21-40-00.gh-issue-89869.XG7aHz.rst b/Misc/NEWS.d/next/Library/2026-07-05-21-40-00.gh-issue-89869.XG7aHz.rst new file mode 100644 index 00000000000000..4bf319f7f68152 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-21-40-00.gh-issue-89869.XG7aHz.rst @@ -0,0 +1,3 @@ +Add :meth:`imaplib.IMAP4.login_plain`, which authenticates using the +``PLAIN`` SASL mechanism (:rfc:`4616`). Unlike :meth:`~imaplib.IMAP4.login`, +it supports non-ASCII user names and passwords. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index ffe82150be1b51..3d14ec3f8f94a9 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -22,6 +22,9 @@ typedef struct { PyObject *dict; PyObject *weakreflist; Py_ssize_t exports; +#ifdef Py_GIL_DISABLED + int buf_shared; +#endif } bytesio; #define bytesio_CAST(op) ((bytesio *)(op)) @@ -71,7 +74,45 @@ check_exports(bytesio *self) return NULL; \ } +#ifdef Py_GIL_DISABLED +#define SHARED_BUF(self) ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf)) +#else #define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) +#endif + +static inline void +set_shared_buf(bytesio *self) +{ +#ifdef Py_GIL_DISABLED + self->buf_shared = 1; +#endif +} + +static inline void +clear_shared_buf(bytesio *self) +{ +#ifdef Py_GIL_DISABLED + self->buf_shared = 0; +#endif +} + +static int +resize_unshared_buffer_lock_held(bytesio *self, Py_ssize_t size) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + +#ifdef Py_GIL_DISABLED + /* If the internal bytes object escaped via a zero-copy getvalue(), read(), + or peek(), resizing it would mutate an object visible to Python code. + Callers must detach first. */ + assert(!self->buf_shared); +#endif + int ret = _PyBytes_Resize(&self->buf, size); + if (ret == 0) { + clear_shared_buf(self); + } + return ret; +} /* Internal routine to get a line from the buffer of a BytesIO @@ -128,6 +169,7 @@ unshare_buffer_lock_held(bytesio *self, size_t size) memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf), self->string_size); Py_SETREF(self->buf, new_buf); + clear_shared_buf(self); return 0; } @@ -173,7 +215,7 @@ resize_buffer_lock_held(bytesio *self, size_t size) return -1; } else { - if (_PyBytes_Resize(&self->buf, alloc) < 0) + if (resize_unshared_buffer_lock_held(self, alloc) < 0) return -1; } @@ -381,10 +423,11 @@ _io_BytesIO_getvalue_impl(bytesio *self) return NULL; } else { - if (_PyBytes_Resize(&self->buf, self->string_size) < 0) + if (resize_unshared_buffer_lock_held(self, self->string_size) < 0) return NULL; } } + set_shared_buf(self); return Py_NewRef(self->buf); } @@ -433,6 +476,7 @@ peek_bytes_lock_held(bytesio *self, Py_ssize_t size) if (size > 1 && self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) && FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { + set_shared_buf(self); return Py_NewRef(self->buf); } @@ -1091,6 +1135,7 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) if (initvalue && initvalue != Py_None) { if (PyBytes_CheckExact(initvalue)) { Py_XSETREF(self->buf, Py_NewRef(initvalue)); + clear_shared_buf(self); self->string_size = PyBytes_GET_SIZE(initvalue); } else { diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 7d03b909226f24..e742a25e4891fc 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -738,6 +738,7 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->charsize = charsize; state->match_all = 0; state->must_advance = 0; + state->save_marks = 0; state->debug = ((pattern->flags & SRE_FLAG_DEBUG) != 0); state->beginning = ptr; diff --git a/Modules/_sre/sre.h b/Modules/_sre/sre.h index 42681c2addf3c2..bec4ae21d1e10f 100644 --- a/Modules/_sre/sre.h +++ b/Modules/_sre/sre.h @@ -97,6 +97,10 @@ typedef struct { int lastmark; int lastindex; const void** mark; + int save_marks; /* if nonzero, save and restore mark values on + backtracking instead of only rewinding the lastmark + index; counts enclosing repeat contexts and + possessive bodies */ /* dynamically allocated stuff */ char* data_stack; size_t data_stack_size; diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index 6e6ae46f05a50f..444cd39d2fed28 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -863,7 +863,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* <0=skip> code ... */ TRACE(("|%p|%p|BRANCH\n", pattern, ptr)); LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); for (; pattern[0]; pattern += pattern[0]) { if (pattern[1] == SRE_OP_LITERAL && @@ -878,16 +878,16 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) state->ptr = ptr; DO_JUMP(JUMP_BRANCH, jump_branch, pattern+1); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_FAILURE; @@ -933,7 +933,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); if (pattern[pattern[0]] == SRE_OP_LITERAL) { @@ -952,19 +952,19 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, pattern+pattern[0]); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ptr--; ctx->count--; } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); } else { /* general case */ @@ -973,19 +973,19 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, pattern+pattern[0]); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); ptr--; ctx->count--; } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; @@ -1035,7 +1035,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } else { /* general case */ LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); while ((Py_ssize_t)pattern[2] == SRE_MAXREPEAT @@ -1044,12 +1044,12 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, pattern+pattern[0]); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (state->repeat) + if (state->save_marks) MARK_POP_KEEP(ctx->lastmark); LASTMARK_RESTORE(); @@ -1063,7 +1063,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) ptr++; ctx->count++; } - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); } RETURN_FAILURE; @@ -1140,10 +1140,12 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) ctx->u.rep->prev = state->repeat; ctx->u.rep->last_ptr = NULL; state->repeat = ctx->u.rep; + state->save_marks++; state->ptr = ptr; DO_JUMP(JUMP_REPEAT, jump_repeat, pattern+pattern[0]); state->repeat = ctx->u.rep->prev; + state->save_marks--; repeat_pool_free(state, ctx->u.rep); if (ret) { @@ -1212,8 +1214,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* cannot match more repeated items here. make sure the tail matches */ state->repeat = ctx->u.rep->prev; + state->save_marks--; DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, pattern); state->repeat = ctx->u.rep; // restore repeat before return + state->save_marks++; RETURN_ON_SUCCESS(ret); state->ptr = ptr; @@ -1250,22 +1254,26 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) /* see if the tail matches */ state->repeat = ctx->u.rep->prev; + state->save_marks--; LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, pattern); - SRE_REPEAT *repeat_of_tail = state->repeat; + /* save_marks is balanced across the jump, so this equals the + value tested for MARK_PUSH above */ + int pushed = state->save_marks != 0; state->repeat = ctx->u.rep; // restore repeat before return + state->save_marks++; if (ret) { - if (repeat_of_tail) + if (pushed) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - if (repeat_of_tail) + if (pushed) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); @@ -1302,16 +1310,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) pointer */ state->ptr = ptr; - /* Set state->repeat to non-NULL */ - ctx->u.rep = repeat_pool_malloc(state); - if (!ctx->u.rep) { - RETURN_ERROR(SRE_ERROR_MEMORY); - } - ctx->u.rep->count = -1; - ctx->u.rep->pattern = NULL; - ctx->u.rep->prev = state->repeat; - ctx->u.rep->last_ptr = NULL; - state->repeat = ctx->u.rep; + /* Capture groups in the body can be revisited on backtracking + between iterations, so their marks must be saved and restored, + as is done inside a repeat. */ + state->save_marks++; /* Initialize Count to 0 */ ctx->count = 0; @@ -1327,9 +1329,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } else { state->ptr = ptr; - /* Restore state->repeat */ - state->repeat = ctx->u.rep->prev; - repeat_pool_free(state, ctx->u.rep); + state->save_marks--; RETURN_FAILURE; } } @@ -1402,9 +1402,7 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) } } - /* Restore state->repeat */ - state->repeat = ctx->u.rep->prev; - repeat_pool_free(state, ctx->u.rep); + state->save_marks--; /* Evaluate Tail */ /* Jump to end of pattern indicated by skip, and then skip @@ -1586,17 +1584,17 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) if ((uintptr_t)(ptr - (SRE_CHAR *)state->beginning) >= pattern[1]) { state->ptr = ptr - pattern[1]; LASTMARK_SAVE(); - if (state->repeat) + if (state->save_marks) MARK_PUSH(ctx->lastmark); DO_JUMP0(JUMP_ASSERT_NOT, jump_assert_not, pattern+2); if (ret) { - if (state->repeat) + if (state->save_marks) MARK_POP_DISCARD(ctx->lastmark); RETURN_ON_ERROR(ret); RETURN_FAILURE; } - if (state->repeat) + if (state->save_marks) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); }