From 4c60aaaf25f58cd73723624a0086f56098a29b7a Mon Sep 17 00:00:00 2001 From: youknowdot Date: Wed, 14 Oct 2026 22:48:48 +0900 Subject: [PATCH 1/3] Preserve truncating FileIO mode and sync canonical IO compatibility Keep truncation metadata in both host backends so writable update streams report wb+. Synchronize the complete final CPython _pyio module and two complete canonical mode test definitions through scripts/update_lib. Assisted-by: OpenAI Codex: model identifier unavailable --- Lib/_pyio.py | 96 +++++++++++++++++---------- Lib/test/test_fileio.py | 4 +- Lib/test/test_io.py | 4 +- crates/host_env/src/io.rs | 10 ++- crates/host_env/src/io_unsupported.rs | 10 ++- 5 files changed, 80 insertions(+), 44 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index cc336d29394..342a50724d4 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -550,7 +550,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -624,7 +624,7 @@ def read(self, size=-1): if n < 0 or n > len(b): raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -632,7 +632,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -882,16 +882,28 @@ class BytesIO(BufferedIOBase): _buffer = None def __init__(self, initial_bytes=None): + # Use to keep self._buffer and self._pos consistent. + self._lock = Lock() + buf = bytearray() if initial_bytes is not None: buf += initial_bytes - self._buffer = buf - self._pos = 0 + + with self._lock: + self._buffer = buf + self._pos = 0 def __getstate__(self): if self.closed: raise ValueError("__getstate__ on closed file") - return self.__dict__.copy() + with self._lock: + state = self.__dict__.copy() + del state['_lock'] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self._lock = Lock() def getvalue(self): """Return the bytes value (contents) of the buffer @@ -924,14 +936,16 @@ def read(self, size=-1): raise TypeError(f"{size!r} is not an integer") else: size = size_index() - if size < 0: - size = len(self._buffer) - if len(self._buffer) <= self._pos: - return b"" - newpos = min(len(self._buffer), self._pos + size) - b = self._buffer[self._pos : newpos] - self._pos = newpos - return bytes(b) + + with self._lock: + if size < 0: + size = len(self._buffer) + if len(self._buffer) <= self._pos: + return b"" + newpos = min(len(self._buffer), self._pos + size) + b = self._buffer[self._pos : newpos] + self._pos = newpos + return bytes(b) def read1(self, size=-1): """This is the same as read. @@ -949,13 +963,14 @@ def write(self, b): if n == 0: return 0 - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = view - self._pos += n - return n + with self._lock: + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n + return n def seek(self, pos, whence=0): if self.closed: @@ -971,9 +986,11 @@ def seek(self, pos, whence=0): raise ValueError("negative seek position %r" % (pos,)) self._pos = pos elif whence == 1: - self._pos = max(0, self._pos + pos) + with self._lock: + self._pos = max(0, self._pos + pos) elif whence == 2: - self._pos = max(0, len(self._buffer) + pos) + with self._lock: + self._pos = max(0, len(self._buffer) + pos) else: raise ValueError("unsupported whence value") return self._pos @@ -986,18 +1003,20 @@ def tell(self): def truncate(self, pos=None): if self.closed: raise ValueError("truncate on closed file") - if pos is None: - pos = self._pos - else: - try: - pos_index = pos.__index__ - except AttributeError: - raise TypeError(f"{pos!r} is not an integer") + + with self._lock: + if pos is None: + pos = self._pos else: - pos = pos_index() - if pos < 0: - raise ValueError("negative truncate position %r" % (pos,)) - del self._buffer[pos:] + try: + pos_index = pos.__index__ + except AttributeError: + raise TypeError(f"{pos!r} is not an integer") + else: + pos = pos_index() + if pos < 0: + raise ValueError("negative truncate position %r" % (pos,)) + del self._buffer[pos:] return pos def readable(self): @@ -1486,6 +1505,7 @@ class FileIO(RawIOBase): _writable = False _appending = False _seekable = None + _truncate = False _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): @@ -1546,6 +1566,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): flags = 0 elif 'w' in mode: self._writable = True + self._truncate = True flags = os.O_CREAT | os.O_TRUNC elif 'a' in mode: self._writable = True @@ -1727,7 +1748,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" @@ -1871,7 +1892,10 @@ def mode(self): return 'ab' elif self._readable: if self._writable: - return 'rb+' + if self._truncate: + return 'wb+' + else: + return 'rb+' else: return 'rb' else: diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 4f195cf045c..b65848f9d77 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -568,8 +568,8 @@ def testModeStrings(self): # test that the mode attribute is correct for various mode strings # given as init args try: - for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'), - ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'), + for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'wb+'), + ('w+b', 'wb+'), ('a', 'ab'), ('ab', 'ab'), ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'), ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]: # read modes are last so that TESTFN will exist first diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 212cbaa324f..4522fbce128 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4385,8 +4385,8 @@ def test_attributes(self): f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") self.assertEqual(f.mode, "w+") - self.assertEqual(f.buffer.mode, "rb+") # Does it really matter? - self.assertEqual(f.buffer.raw.mode, "rb+") + self.assertEqual(f.buffer.mode, "wb+") + self.assertEqual(f.buffer.raw.mode, "wb+") g = self.open(f.fileno(), "wb", closefd=False) self.assertEqual(g.mode, "wb") diff --git a/crates/host_env/src/io.rs b/crates/host_env/src/io.rs index 134e99df14d..3d8e67779ea 100644 --- a/crates/host_env/src/io.rs +++ b/crates/host_env/src/io.rs @@ -17,6 +17,7 @@ bitflagset::bitflag! { Readable = 1, Writable = 2, Appending = 3, + Truncating = 4, } } @@ -30,6 +31,7 @@ impl FileMode { pub const READABLE: Self = Self::from_element(FileModeFlag::Readable); pub const WRITABLE: Self = Self::from_element(FileModeFlag::Writable); pub const APPENDING: Self = Self::from_element(FileModeFlag::Appending); + pub const TRUNCATING: Self = Self::from_element(FileModeFlag::Truncating); } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -72,7 +74,11 @@ impl FileMode { } } else if self.contains(&FileModeFlag::Readable) { if self.contains(&FileModeFlag::Writable) { - "rb+" + if self.contains(&FileModeFlag::Truncating) { + "wb+" + } else { + "rb+" + } } else { "rb" } @@ -109,7 +115,7 @@ pub fn parse_fileio_mode(mode_str: &str) -> Result { diff --git a/crates/host_env/src/io_unsupported.rs b/crates/host_env/src/io_unsupported.rs index a779d6f19d1..17eba9ff795 100644 --- a/crates/host_env/src/io_unsupported.rs +++ b/crates/host_env/src/io_unsupported.rs @@ -13,6 +13,7 @@ bitflagset::bitflag! { Readable = 1, Writable = 2, Appending = 3, + Truncating = 4, } } @@ -26,6 +27,7 @@ impl FileMode { pub const READABLE: Self = Self::from_element(FileModeFlag::Readable); pub const WRITABLE: Self = Self::from_element(FileModeFlag::Writable); pub const APPENDING: Self = Self::from_element(FileModeFlag::Appending); + pub const TRUNCATING: Self = Self::from_element(FileModeFlag::Truncating); } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -68,7 +70,11 @@ impl FileMode { } } else if self.contains(&FileModeFlag::Readable) { if self.contains(&FileModeFlag::Writable) { - "rb+" + if self.contains(&FileModeFlag::Truncating) { + "wb+" + } else { + "rb+" + } } else { "rb" } @@ -106,7 +112,7 @@ pub fn parse_fileio_mode(mode_str: &str) -> Result { From b314260ae4ccf493c51fb702ad81ead6cc806f03 Mon Sep 17 00:00:00 2001 From: youknowdot <336763669+youknowdot@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:33:54 +0900 Subject: [PATCH 2/3] Update gzip, tarfile, and shutil from CPython 3.15 Regenerate six Lib files using scripts/update_lib from CPython d220409a13b77de7a6c30abe16ea90db01528edc. Preserve upstream tests without agent-added marks. Assisted-by: OpenAI Codex:model version unavailable --- Lib/gzip.py | 93 ++++++++++++++++++--------- Lib/shutil.py | 30 ++++++--- Lib/tarfile.py | 119 +++++++++++++++++++++++------------ Lib/test/test_gzip.py | 58 +++++++++++++++-- Lib/test/test_shutil.py | 59 ++++++++++++++++- Lib/test/test_tarfile.py | 133 ++++++++++++++++++++++++++++++++++----- 6 files changed, 395 insertions(+), 97 deletions(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index f3f9fa91c81..c7b4563fdfe 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -30,7 +30,7 @@ _WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE -def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, +def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, encoding=None, errors=None, newline=None): """Open a gzip-compressed file in binary or text mode. @@ -158,7 +158,7 @@ class GzipFile(_streams.BaseStream): myfileobj = None def __init__(self, filename=None, mode=None, - compresslevel=_COMPRESS_LEVEL_BEST, fileobj=None, mtime=None): + compresslevel=_COMPRESS_LEVEL_TRADEOFF, fileobj=None, mtime=None): """Constructor for the GzipFile class. At least one of fileobj and filename must be given a @@ -189,9 +189,10 @@ def __init__(self, filename=None, mode=None, The optional mtime argument is the timestamp requested by gzip. The time is in Unix format, i.e., seconds since 00:00:00 UTC, - January 1, 1970. If mtime is omitted or None, the current time - is used. Use mtime = 0 to generate a compressed stream that does - not depend on creation time. + January 1, 1970. Set mtime to 0 to generate a compressed stream + that does not depend on creation time. If mtime is omitted or None, + the current time is used. If the resulting mtime is outside the + range 0 to 2**32-1, then the value 0 is used instead. """ @@ -270,8 +271,6 @@ def _init_write(self, filename): self.name = filename self.crc = zlib.crc32(b"") self.size = 0 - self.writebuf = [] - self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc def tell(self): @@ -299,6 +298,8 @@ def _write_gzip_header(self, compresslevel): mtime = self._write_mtime if mtime is None: mtime = time.time() + if not 0 <= mtime < 2**32: + mtime = 0 write32u(self.fileobj, int(mtime)) if compresslevel == _COMPRESS_LEVEL_BEST: xfl = b'\002' @@ -488,40 +489,73 @@ def _read_exact(fp, n): return data +def _read_until_null(fp, crc=None): + '''Read until the first encountered null byte in fp. + If crc is not None, update and return the CRC. + ''' + if crc is None: + while True: + s = fp.read(1) + if not s or s == b'\000': + break + else: + while True: + s = fp.read(1) + crc = zlib.crc32(s, crc) + if not s or s == b'\000': + break + return crc + + def _read_gzip_header(fp): '''Read a gzip header from `fp` and progress to the end of the header. Returns last mtime if header was present or None otherwise. ''' magic = fp.read(2) - if magic == b'': + if not magic: return None if magic != b'\037\213': raise BadGzipFile('Not a gzipped file (%r)' % magic) - - (method, flag, last_mtime) = struct.unpack(" Date: Thu, 15 Oct 2026 04:05:35 +0900 Subject: [PATCH 3/3] Sync canonical spooled tempfile mode expectation Copy the complete TestSpooledTemporaryFile.test_properties definition from CPython 3.15.0 final d220409a using scripts/update_lib copy-test. Preserve all surrounding test content and existing markers. Assisted-by: OpenAI Codex:model version unavailable --- Lib/test/test_tempfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 6fac921f629..cf8b1037194 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1409,7 +1409,7 @@ def test_properties(self): f.write(b'x') self.assertTrue(f._rolled) - self.assertEqual(f.mode, 'rb+') + self.assertEqual(f.mode, 'wb+') self.assertIsNotNone(f.name) with self.assertRaises(AttributeError): f.newlines