diff --git a/Lib/_pyio.py b/Lib/_pyio.py index cc336d29394..342a50724d4 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -550,7 +550,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -624,7 +624,7 @@ def read(self, size=-1): if n < 0 or n > len(b): raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -632,7 +632,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -882,16 +882,28 @@ class BytesIO(BufferedIOBase): _buffer = None def __init__(self, initial_bytes=None): + # Use to keep self._buffer and self._pos consistent. + self._lock = Lock() + buf = bytearray() if initial_bytes is not None: buf += initial_bytes - self._buffer = buf - self._pos = 0 + + with self._lock: + self._buffer = buf + self._pos = 0 def __getstate__(self): if self.closed: raise ValueError("__getstate__ on closed file") - return self.__dict__.copy() + with self._lock: + state = self.__dict__.copy() + del state['_lock'] + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self._lock = Lock() def getvalue(self): """Return the bytes value (contents) of the buffer @@ -924,14 +936,16 @@ def read(self, size=-1): raise TypeError(f"{size!r} is not an integer") else: size = size_index() - if size < 0: - size = len(self._buffer) - if len(self._buffer) <= self._pos: - return b"" - newpos = min(len(self._buffer), self._pos + size) - b = self._buffer[self._pos : newpos] - self._pos = newpos - return bytes(b) + + with self._lock: + if size < 0: + size = len(self._buffer) + if len(self._buffer) <= self._pos: + return b"" + newpos = min(len(self._buffer), self._pos + size) + b = self._buffer[self._pos : newpos] + self._pos = newpos + return bytes(b) def read1(self, size=-1): """This is the same as read. @@ -949,13 +963,14 @@ def write(self, b): if n == 0: return 0 - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = view - self._pos += n - return n + with self._lock: + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n + return n def seek(self, pos, whence=0): if self.closed: @@ -971,9 +986,11 @@ def seek(self, pos, whence=0): raise ValueError("negative seek position %r" % (pos,)) self._pos = pos elif whence == 1: - self._pos = max(0, self._pos + pos) + with self._lock: + self._pos = max(0, self._pos + pos) elif whence == 2: - self._pos = max(0, len(self._buffer) + pos) + with self._lock: + self._pos = max(0, len(self._buffer) + pos) else: raise ValueError("unsupported whence value") return self._pos @@ -986,18 +1003,20 @@ def tell(self): def truncate(self, pos=None): if self.closed: raise ValueError("truncate on closed file") - if pos is None: - pos = self._pos - else: - try: - pos_index = pos.__index__ - except AttributeError: - raise TypeError(f"{pos!r} is not an integer") + + with self._lock: + if pos is None: + pos = self._pos else: - pos = pos_index() - if pos < 0: - raise ValueError("negative truncate position %r" % (pos,)) - del self._buffer[pos:] + try: + pos_index = pos.__index__ + except AttributeError: + raise TypeError(f"{pos!r} is not an integer") + else: + pos = pos_index() + if pos < 0: + raise ValueError("negative truncate position %r" % (pos,)) + del self._buffer[pos:] return pos def readable(self): @@ -1486,6 +1505,7 @@ class FileIO(RawIOBase): _writable = False _appending = False _seekable = None + _truncate = False _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): @@ -1546,6 +1566,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None): flags = 0 elif 'w' in mode: self._writable = True + self._truncate = True flags = os.O_CREAT | os.O_TRUNC elif 'a' in mode: self._writable = True @@ -1727,7 +1748,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" @@ -1871,7 +1892,10 @@ def mode(self): return 'ab' elif self._readable: if self._writable: - return 'rb+' + if self._truncate: + return 'wb+' + else: + return 'rb+' else: return 'rb' else: diff --git a/Lib/gzip.py b/Lib/gzip.py index f3f9fa91c81..c7b4563fdfe 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -30,7 +30,7 @@ _WRITE_BUFFER_SIZE = 4 * io.DEFAULT_BUFFER_SIZE -def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, +def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, encoding=None, errors=None, newline=None): """Open a gzip-compressed file in binary or text mode. @@ -158,7 +158,7 @@ class GzipFile(_streams.BaseStream): myfileobj = None def __init__(self, filename=None, mode=None, - compresslevel=_COMPRESS_LEVEL_BEST, fileobj=None, mtime=None): + compresslevel=_COMPRESS_LEVEL_TRADEOFF, fileobj=None, mtime=None): """Constructor for the GzipFile class. At least one of fileobj and filename must be given a @@ -189,9 +189,10 @@ def __init__(self, filename=None, mode=None, The optional mtime argument is the timestamp requested by gzip. The time is in Unix format, i.e., seconds since 00:00:00 UTC, - January 1, 1970. If mtime is omitted or None, the current time - is used. Use mtime = 0 to generate a compressed stream that does - not depend on creation time. + January 1, 1970. Set mtime to 0 to generate a compressed stream + that does not depend on creation time. If mtime is omitted or None, + the current time is used. If the resulting mtime is outside the + range 0 to 2**32-1, then the value 0 is used instead. """ @@ -270,8 +271,6 @@ def _init_write(self, filename): self.name = filename self.crc = zlib.crc32(b"") self.size = 0 - self.writebuf = [] - self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc def tell(self): @@ -299,6 +298,8 @@ def _write_gzip_header(self, compresslevel): mtime = self._write_mtime if mtime is None: mtime = time.time() + if not 0 <= mtime < 2**32: + mtime = 0 write32u(self.fileobj, int(mtime)) if compresslevel == _COMPRESS_LEVEL_BEST: xfl = b'\002' @@ -488,40 +489,73 @@ def _read_exact(fp, n): return data +def _read_until_null(fp, crc=None): + '''Read until the first encountered null byte in fp. + If crc is not None, update and return the CRC. + ''' + if crc is None: + while True: + s = fp.read(1) + if not s or s == b'\000': + break + else: + while True: + s = fp.read(1) + crc = zlib.crc32(s, crc) + if not s or s == b'\000': + break + return crc + + def _read_gzip_header(fp): '''Read a gzip header from `fp` and progress to the end of the header. Returns last mtime if header was present or None otherwise. ''' magic = fp.read(2) - if magic == b'': + if not magic: return None if magic != b'\037\213': raise BadGzipFile('Not a gzipped file (%r)' % magic) - - (method, flag, last_mtime) = struct.unpack(" Result { diff --git a/crates/host_env/src/io_unsupported.rs b/crates/host_env/src/io_unsupported.rs index a779d6f19d1..17eba9ff795 100644 --- a/crates/host_env/src/io_unsupported.rs +++ b/crates/host_env/src/io_unsupported.rs @@ -13,6 +13,7 @@ bitflagset::bitflag! { Readable = 1, Writable = 2, Appending = 3, + Truncating = 4, } } @@ -26,6 +27,7 @@ impl FileMode { pub const READABLE: Self = Self::from_element(FileModeFlag::Readable); pub const WRITABLE: Self = Self::from_element(FileModeFlag::Writable); pub const APPENDING: Self = Self::from_element(FileModeFlag::Appending); + pub const TRUNCATING: Self = Self::from_element(FileModeFlag::Truncating); } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -68,7 +70,11 @@ impl FileMode { } } else if self.contains(&FileModeFlag::Readable) { if self.contains(&FileModeFlag::Writable) { - "rb+" + if self.contains(&FileModeFlag::Truncating) { + "wb+" + } else { + "rb+" + } } else { "rb" } @@ -106,7 +112,7 @@ pub fn parse_fileio_mode(mode_str: &str) -> Result {