From 72b0543603e28b0c3c1c748e306f5970071446d7 Mon Sep 17 00:00:00 2001 From: Kwak Byoung Min Date: Thu, 23 Jul 2026 11:51:51 +0900 Subject: [PATCH 1/3] Keep StringIO text operations CPython-compatible Constraint: email.feedparser depends on StringIO(newline='') recognizing CR, LF, and CRLF boundaries. Rejected: an email-specific parser workaround | StringIO is the shared root cause. Confidence: high Scope-risk: moderate Directive: Keep internal storage byte-based; convert only at StringIO API boundaries. Tested: prek run --all-files; test_memoryio; test_email; full workspace verification by contributor Assisted-by: Codex:gpt-5.6-sol --- Lib/test/test_email/test_email.py | 4 - crates/vm/src/stdlib/_io.rs | 155 ++++++++++++++++++++++++++---- 2 files changed, 134 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 49cdc95021a..671bc487bbf 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3812,7 +3812,6 @@ def test_typed_subpart_iterator_default_type(self): -Me """) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_pushCR_LF(self): '''FeedParser BufferedSubFile.push() assumed it received complete line endings. A CR ending one push() followed by a LF starting @@ -3843,7 +3842,6 @@ def test_pushCR_LF(self): self.assertEqual(len(om), nt) self.assertEqual(''.join([il for il, n in imt]), ''.join(om)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_push_random(self): from email.feedparser import BufferedSubFile, NeedMoreData @@ -3877,7 +3875,6 @@ def test_empty_header_name_handled(self): self.assertEqual(msg['First'], 'val') self.assertEqual(msg['Second'], 'val') - @unittest.expectedFailure # TODO: RUSTPYTHON; Feedparser.feed -> Feedparser._input.push, Feedparser._call_parse -> Feedparser._parse does not keep _input state between calls def test_newlines(self): m = self.parse(['a:\nb:\rc:\r\nd:\n']) self.assertEqual(m.keys(), ['a', 'b', 'c', 'd']) @@ -3896,7 +3893,6 @@ def test_newlines(self): m = self.parse(['a:\r', 'b:\x85', 'c:\n']) self.assertEqual(m.items(), [('a', ''), ('b', '\x85c:')]) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_long_lines(self): # Expected peak memory use on 32-bit platform: 6*N*M bytes. M, N = 1000, 20000 diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 4cffb909fa7..3e1adad242a 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -4380,6 +4380,7 @@ mod _io { struct StringIO { _base: _TextIOBase, buffer: PyRwLock, + newline: AtomicCell, closed: AtomicCell, } @@ -4388,10 +4389,8 @@ mod _io { #[pyarg(positional, optional)] object: OptionalOption, - // TODO: use this #[pyarg(any, default)] - #[allow(dead_code)] - newline: Newlines, + newline: OptionalOption, } impl Constructor for StringIO { @@ -4401,6 +4400,7 @@ mod _io { Ok(Self { _base: Default::default(), buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), + newline: AtomicCell::new(Newlines::Lf), closed: AtomicCell::new(false), }) } @@ -4409,7 +4409,6 @@ mod _io { impl Initializer for StringIO { type Args = StringIONewArgs; - #[allow(unused_variables)] fn init( zelf: PyRef, Self::Args { object, newline }: Self::Args, @@ -4419,6 +4418,12 @@ mod _io { .flatten() .map_or_else(Vec::new, |v| v.as_bytes().to_vec()); *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); + let newline = match newline { + OptionalArg::Missing => Newlines::Lf, + OptionalArg::Present(None) => Newlines::Universal, + OptionalArg::Present(Some(newline)) => newline, + }; + zelf.newline.store(newline); Ok(()) } } @@ -4431,6 +4436,54 @@ mod _io { Err(io_closed_error(vm)) } } + + fn char_offset_to_byte( + bytes: &[u8], + char_offset: usize, + vm: &VirtualMachine, + ) -> PyResult { + let text = Wtf8::from_bytes(bytes) + .ok_or_else(|| vm.new_value_error("Error Retrieving Value"))?; + let char_len = text.code_points().count(); + if char_offset >= char_len { + return Ok(bytes.len() + (char_offset - char_len)); + } + Ok(crate::common::str::codepoint_range_end(text, char_offset) + .expect("char_offset is within the string")) + } + + fn byte_offset_to_char( + bytes: &[u8], + byte_offset: usize, + vm: &VirtualMachine, + ) -> PyResult { + let content_len = bytes.len(); + let in_content = byte_offset.min(content_len); + let text = Wtf8::from_bytes(&bytes[..in_content]) + .ok_or_else(|| vm.new_value_error("Error Retrieving Value"))?; + Ok(text.code_points().count() + byte_offset.saturating_sub(content_len)) + } + + fn read_size( + buffer: &BufferedIO, + size: Option, + newline: Option, + vm: &VirtualMachine, + ) -> PyResult { + let position = buffer.tell() as usize; + let bytes = buffer.cursor.get_ref().get(position..).unwrap_or_default(); + let text = Wtf8::from_bytes(bytes) + .ok_or_else(|| vm.new_value_error("Error Retrieving Value"))?; + let size_end = size + .and_then(|size| crate::common::str::codepoint_range_end(text, size)) + .unwrap_or(bytes.len()); + Ok(match newline { + Some(newline) => match newline.find_newline(text) { + Ok(line_end) | Err(line_end) => line_end.min(size_end), + }, + None => size_end, + }) + } } #[pyclass(flags(BASETYPE, HAS_DICT, HAS_WEAKREF), with(Constructor, Initializer))] @@ -4466,7 +4519,8 @@ mod _io { let bytes = data.as_bytes(); self.buffer(vm)? .write(bytes) - .ok_or_else(|| vm.new_type_error("Error Writing String")) + .ok_or_else(|| vm.new_type_error("Error Writing String"))?; + Ok(data.char_len() as u64) } // return the entire contents of the underlying @@ -4484,9 +4538,38 @@ mod _io { how: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - self.buffer(vm)? - .seek(seekfrom(vm, offset, how)?) - .map_err(|err| os_err(vm, err)) + let offset: isize = ArgSize::try_from_object(vm, offset)?.into(); + let how = how.unwrap_or(0); + let char_offset = match how { + 0 if offset >= 0 => offset as usize, + 0 => return Err(vm.new_value_error(format!("negative seek position {offset}"))), + 1 if offset == 0 => { + let buffer = self.buffer(vm)?; + Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)? + } + 1 => return Err(vm.new_os_error("can't do nonzero cur-relative seeks")), + 2 if offset == 0 => { + let buffer = self.buffer(vm)?; + Self::byte_offset_to_char( + buffer.cursor.get_ref(), + buffer.cursor.get_ref().len(), + vm, + )? + } + 2 => return Err(vm.new_os_error("can't do nonzero end-relative seeks")), + _ => { + return Err( + vm.new_value_error(format!("invalid whence ({how}, should be 0, 1 or 2)")) + ); + } + }; + + let mut buffer = self.buffer(vm)?; + let byte_offset = Self::char_offset_to_byte(buffer.cursor.get_ref(), char_offset, vm)?; + buffer + .seek(SeekFrom::Start(byte_offset as u64)) + .map_err(|err| os_err(vm, err))?; + Ok(char_offset as u64) } // Read k bytes from the object and return. @@ -4494,7 +4577,9 @@ mod _io { // This also increments the stream position by the value of k #[pymethod] fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { - let data = self.buffer(vm)?.read(size.to_usize()).unwrap_or_default(); + let mut buffer = self.buffer(vm)?; + let size = Self::read_size(&buffer, size.to_usize(), None, vm)?; + let data = buffer.read(Some(size)).unwrap_or_default(); let value = Wtf8Buf::from_bytes(data) .map_err(|_| vm.new_value_error("Error Retrieving Value"))?; @@ -4503,22 +4588,37 @@ mod _io { #[pymethod] fn tell(&self, vm: &VirtualMachine) -> PyResult { - Ok(self.buffer(vm)?.tell()) + let buffer = self.buffer(vm)?; + Ok( + Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)? + as u64, + ) } #[pymethod] fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { - // TODO size should correspond to the number of characters, at the moments its the number of - // bytes. - let input = self.buffer(vm)?.readline(size.to_usize(), vm)?; + let mut buffer = self.buffer(vm)?; + let newline = match self.newline.load() { + Newlines::Passthrough => Newlines::Passthrough, + _ => Newlines::Lf, + }; + let size = Self::read_size(&buffer, size.to_usize(), Some(newline), vm)?; + let input = buffer.read(Some(size)).unwrap_or_default(); Wtf8Buf::from_bytes(input).map_err(|_| vm.new_value_error("Error Retrieving Value")) } #[pymethod] fn truncate(&self, pos: OptionalSize, vm: &VirtualMachine) -> PyResult { let mut buffer = self.buffer(vm)?; - let pos = pos.try_usize(vm)?; - Ok(buffer.truncate(pos)) + let pos = match pos.try_usize(vm)? { + Some(pos) => pos, + None => { + Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)? + } + }; + let byte_pos = Self::char_offset_to_byte(buffer.cursor.get_ref(), pos, vm)?; + buffer.truncate(Some(byte_pos)); + Ok(pos) } #[pygetset] @@ -4531,7 +4631,8 @@ mod _io { let buffer = zelf.buffer(vm)?; let content = Wtf8Buf::from_bytes(buffer.getvalue()) .map_err(|_| vm.new_value_error("Error Retrieving Value"))?; - let pos = buffer.tell(); + let pos = + Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)?; drop(buffer); // Get __dict__ if it exists and is non-empty @@ -4540,11 +4641,18 @@ mod _io { _ => vm.ctx.none(), }; + let newline = match zelf.newline.load() { + Newlines::Universal => vm.ctx.none(), + Newlines::Passthrough => vm.ctx.new_str("").into(), + Newlines::Lf => vm.ctx.new_str("\n").into(), + Newlines::Cr => vm.ctx.new_str("\r").into(), + Newlines::Crlf => vm.ctx.new_str("\r\n").into(), + }; + // Return (content, newline, position, dict) - // TODO: store actual newline setting when it's implemented Ok(vm.ctx.new_tuple(vec![ vm.ctx.new_str(content).into(), - vm.ctx.new_str("\n").into(), + newline, vm.ctx.new_int(pos).into(), dict_obj, ])) @@ -4564,18 +4672,23 @@ mod _io { } let content: PyStrRef = state[0].clone().try_into_value(vm)?; - // state[1] is newline - TODO: use when newline handling is implemented - let pos: u64 = state[2].clone().try_into_value(vm)?; + let newline = Newlines::try_from_object(vm, state[1].clone())?; + let pos: isize = ArgSize::try_from_object(vm, state[2].clone())?.into(); + if pos < 0 { + return Err(vm.new_value_error("negative seek position")); + } let dict = &state[3]; // Set content and position let raw_bytes = content.as_bytes().to_vec(); let mut buffer = zelf.buffer.write(); *buffer = BufferedIO::new(Cursor::new(raw_bytes)); + let byte_pos = Self::char_offset_to_byte(buffer.cursor.get_ref(), pos as usize, vm)?; buffer - .seek(SeekFrom::Start(pos)) + .seek(SeekFrom::Start(byte_pos as u64)) .map_err(|err| os_err(vm, err))?; drop(buffer); + zelf.newline.store(newline); // Set __dict__ if provided if !vm.is_none(dict) { From b87b7896c989d824cedf8a5150218d2ef252c75f Mon Sep 17 00:00:00 2001 From: Kwak Byoung Min Date: Thu, 23 Jul 2026 11:51:56 +0900 Subject: [PATCH 2/3] Run restored StringIO conformance tests Constraint: upstream StringIO tests already cover the repaired behavior. Rejected: new RustPython-only regression tests | existing CPython coverage is sufficient. Confidence: high Scope-risk: narrow Directive: Remove expected-failure markers only while the inherited tests pass. Tested: prek run --all-files; test_memoryio; test_email Assisted-by: Codex:gpt-5.6-sol --- Lib/test/test_memoryio.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index f0c21e4ae11..2d7881faec9 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -940,7 +940,6 @@ class CStringIOTest(PyStringIOTest): # XXX: For the Python version of io.StringIO, this is highly # dependent on the encoding used for the underlying buffer. - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 8 != 2 def test_widechar(self): buf = self.buftype("\U0002030a\U00020347") memio = self.ioclass(buf) @@ -965,7 +964,6 @@ def test_getstate(self): memio.close() self.assertRaises(ValueError, memio.__getstate__) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by __setstate__ def test_setstate(self): # This checks whether __setstate__ does proper input validation. memio = self.ioclass() @@ -1006,18 +1004,10 @@ def __str__(self): def test_issue5265(self): return super().test_issue5265() - @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ - def test_newline_empty(self): - return super().test_newline_empty() - @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ def test_newline_none(self): return super().test_newline_none() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by seek - def test_relative_seek(self): - return super().test_relative_seek() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by writable def test_flags(self): return super().test_flags() @@ -1048,18 +1038,10 @@ def __init__(self, *args, **kwargs): def test_issue5265(self): return super().test_issue5265() - @unittest.expectedFailure # TODO: RUSTPYTHON; ? ++++ - def test_newline_empty(self): - return super().test_newline_empty() - @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ def test_newline_none(self): return super().test_newline_none() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: OSError not raised by seek - def test_relative_seek(self): - return super().test_relative_seek() - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? def test_newlines_property(self): return super().test_newlines_property() From 7c097b5e759c94daa6eb236385d66211c6278b61 Mon Sep 17 00:00:00 2001 From: Kwak Byoung Min Date: Thu, 23 Jul 2026 14:22:07 +0900 Subject: [PATCH 3/3] Keep StringIO newline handling consistent Constraint: StringIO must apply its newline mode consistently when constructing, writing, and reading text.\nRejected: a readline-only fix | it leaves CR and CRLF modes internally inconsistent.\nConfidence: high\nScope-risk: moderate\nDirective: Keep StringIO buffer contents valid WTF-8 before using unchecked views.\nTested: prek run --all-files; cargo clippy -p rustpython-vm -- -D warnings; test_memoryio; test_shlex; test_email; workspace excluding rustpython-capi; full workspace manually verified by contributor\nNot-tested: local rustpython-capi workspace test crashes with a pre-existing macOS SIGSEGV\nAssisted-by: Codex:gpt-5.6-sol --- Lib/test/test_memoryio.py | 34 ---------- Lib/test/test_shlex.py | 4 -- crates/vm/src/stdlib/_io.rs | 128 +++++++++++++++--------------------- 3 files changed, 54 insertions(+), 112 deletions(-) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 2d7881faec9..1683a71fc88 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -1000,14 +1000,6 @@ def __str__(self): memio2.write(MyStr("world")) self.assertEqual(memio2.getvalue(), "hello world") - @unittest.expectedFailure # TODO: RUSTPYTHON; + - def test_issue5265(self): - return super().test_issue5265() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ - def test_newline_none(self): - return super().test_newline_none() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by writable def test_flags(self): return super().test_flags() @@ -1016,15 +1008,6 @@ def test_flags(self): def test_newlines_property(self): return super().test_newlines_property() - @unittest.expectedFailure # TODO: RUSTPYTHON; d - def test_newline_cr(self): - return super().test_newline_cr() - - @unittest.expectedFailure # TODO: RUSTPYTHON; d - def test_newline_crlf(self): - return super().test_newline_crlf() - - class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation @@ -1034,26 +1017,9 @@ def __new__(cls, *args, **kwargs): def __init__(self, *args, **kwargs): pass - @unittest.expectedFailure # TODO: RUSTPYTHON; + - def test_issue5265(self): - return super().test_issue5265() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ? ^^^^^ - def test_newline_none(self): - return super().test_newline_none() - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? def test_newlines_property(self): return super().test_newlines_property() - @unittest.expectedFailure # TODO: RUSTPYTHON; d - def test_newline_cr(self): - return super().test_newline_cr() - - @unittest.expectedFailure # TODO: RUSTPYTHON; d - def test_newline_crlf(self): - return super().test_newline_crlf() - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 7c41432b82f..2a355abdeeb 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -167,12 +167,10 @@ def testSplitNone(self): with self.assertRaises(ValueError): shlex.split(None) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testSplitPosix(self): """Test data splitting with posix parser""" self.splitTest(self.posix_data, comments=True) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testCompat(self): """Test compatibility interface""" for i in range(len(self.data)): @@ -313,7 +311,6 @@ def testEmptyStringHandling(self): s = shlex.shlex("'')abc", punctuation_chars=True) self.assertEqual(list(s), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testUnicodeHandling(self): """Test punctuation_chars and whitespace_split handle unicode.""" ss = "\u2119\u01b4\u2602\u210c\u00f8\u1f24" @@ -356,7 +353,6 @@ def testJoin(self): joined = shlex.join(split_command) self.assertEqual(joined, command) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Error Retrieving Value def testJoinRoundtrip(self): all_data = self.data + self.posix_data for command, *split_command in all_data: diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index 3e1adad242a..33656af79b0 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -2371,10 +2371,10 @@ mod _io { match memchr::memchr(b'\r', remaining) { Some(p) => match remaining.get(p + 1) { Some(&ch_after_cr) => { - let pos_after = p + 2; if ch_after_cr == b'\n' { - break Ok(searched + pos_after); + break Ok(searched + p + 2); } + let pos_after = p + 1; searched += pos_after; remaining = &remaining[pos_after..]; continue; @@ -4414,15 +4414,15 @@ mod _io { Self::Args { object, newline }: Self::Args, _vm: &VirtualMachine, ) -> PyResult<()> { - let raw_bytes = object - .flatten() - .map_or_else(Vec::new, |v| v.as_bytes().to_vec()); - *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); let newline = match newline { OptionalArg::Missing => Newlines::Lf, OptionalArg::Present(None) => Newlines::Universal, OptionalArg::Present(Some(newline)) => newline, }; + let raw_bytes = object.flatten().map_or_else(Vec::new, |v| { + Self::translate_newlines(v.as_wtf8(), newline).into_bytes() + }); + *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); zelf.newline.store(newline); Ok(()) } @@ -4437,52 +4437,44 @@ mod _io { } } - fn char_offset_to_byte( - bytes: &[u8], - char_offset: usize, - vm: &VirtualMachine, - ) -> PyResult { - let text = Wtf8::from_bytes(bytes) - .ok_or_else(|| vm.new_value_error("Error Retrieving Value"))?; - let char_len = text.code_points().count(); - if char_offset >= char_len { - return Ok(bytes.len() + (char_offset - char_len)); + fn translate_newlines(data: &Wtf8, newline: Newlines) -> Wtf8Buf { + match newline { + Newlines::Universal => data + .replace("\r\n".as_ref(), "\n".as_ref()) + .replace("\r".as_ref(), "\n".as_ref()), + Newlines::Cr => data.replace("\n".as_ref(), "\r".as_ref()), + Newlines::Crlf => data.replace("\n".as_ref(), "\r\n".as_ref()), + Newlines::Passthrough | Newlines::Lf => data.to_owned(), } - Ok(crate::common::str::codepoint_range_end(text, char_offset) - .expect("char_offset is within the string")) } - fn byte_offset_to_char( - bytes: &[u8], - byte_offset: usize, - vm: &VirtualMachine, - ) -> PyResult { + fn text(bytes: &[u8]) -> &Wtf8 { + // SAFETY: StringIO is populated only from PyStr values, which are valid WTF-8. + unsafe { Wtf8::from_bytes_unchecked(bytes) } + } + + fn char_offset_to_byte(bytes: &[u8], char_offset: usize) -> usize { + let text = Self::text(bytes); + crate::common::str::codepoint_range_end(text, char_offset) + .unwrap_or_else(|| bytes.len() + (char_offset - text.code_points().count())) + } + + fn byte_offset_to_char(bytes: &[u8], byte_offset: usize) -> usize { let content_len = bytes.len(); let in_content = byte_offset.min(content_len); - let text = Wtf8::from_bytes(&bytes[..in_content]) - .ok_or_else(|| vm.new_value_error("Error Retrieving Value"))?; - Ok(text.code_points().count() + byte_offset.saturating_sub(content_len)) + Self::text(&bytes[..in_content]).code_points().count() + + byte_offset.saturating_sub(content_len) } - fn read_size( - buffer: &BufferedIO, - size: Option, - newline: Option, - vm: &VirtualMachine, - ) -> PyResult { + fn read_size(buffer: &BufferedIO, size: Option, newline: Option) -> usize { let position = buffer.tell() as usize; let bytes = buffer.cursor.get_ref().get(position..).unwrap_or_default(); - let text = Wtf8::from_bytes(bytes) - .ok_or_else(|| vm.new_value_error("Error Retrieving Value"))?; let size_end = size - .and_then(|size| crate::common::str::codepoint_range_end(text, size)) + .and_then(|size| crate::common::str::codepoint_range_end(Self::text(bytes), size)) .unwrap_or(bytes.len()); - Ok(match newline { - Some(newline) => match newline.find_newline(text) { - Ok(line_end) | Err(line_end) => line_end.min(size_end), - }, - None => size_end, - }) + newline + .and_then(|newline| newline.find_newline(Self::text(&bytes[..size_end])).ok()) + .unwrap_or(size_end) } } @@ -4516,9 +4508,9 @@ mod _io { // write string to underlying vector #[pymethod] fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = data.as_bytes(); + let bytes = Self::translate_newlines(data.as_wtf8(), self.newline.load()).into_bytes(); self.buffer(vm)? - .write(bytes) + .write(&bytes) .ok_or_else(|| vm.new_type_error("Error Writing String"))?; Ok(data.char_len() as u64) } @@ -4540,23 +4532,22 @@ mod _io { ) -> PyResult { let offset: isize = ArgSize::try_from_object(vm, offset)?.into(); let how = how.unwrap_or(0); + let mut buffer = self.buffer(vm)?; let char_offset = match how { 0 if offset >= 0 => offset as usize, 0 => return Err(vm.new_value_error(format!("negative seek position {offset}"))), - 1 if offset == 0 => { - let buffer = self.buffer(vm)?; - Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)? + 1 | 2 if offset != 0 => { + let kind = if how == 1 { "cur" } else { "end" }; + return Err(vm.new_os_error(format!("can't do nonzero {kind}-relative seeks"))); } - 1 => return Err(vm.new_os_error("can't do nonzero cur-relative seeks")), - 2 if offset == 0 => { - let buffer = self.buffer(vm)?; - Self::byte_offset_to_char( - buffer.cursor.get_ref(), - buffer.cursor.get_ref().len(), - vm, - )? + 1 | 2 => { + let byte_offset = if how == 1 { + buffer.tell() as usize + } else { + buffer.cursor.get_ref().len() + }; + Self::byte_offset_to_char(buffer.cursor.get_ref(), byte_offset) } - 2 => return Err(vm.new_os_error("can't do nonzero end-relative seeks")), _ => { return Err( vm.new_value_error(format!("invalid whence ({how}, should be 0, 1 or 2)")) @@ -4564,8 +4555,7 @@ mod _io { } }; - let mut buffer = self.buffer(vm)?; - let byte_offset = Self::char_offset_to_byte(buffer.cursor.get_ref(), char_offset, vm)?; + let byte_offset = Self::char_offset_to_byte(buffer.cursor.get_ref(), char_offset); buffer .seek(SeekFrom::Start(byte_offset as u64)) .map_err(|err| os_err(vm, err))?; @@ -4578,7 +4568,7 @@ mod _io { #[pymethod] fn read(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { let mut buffer = self.buffer(vm)?; - let size = Self::read_size(&buffer, size.to_usize(), None, vm)?; + let size = Self::read_size(&buffer, size.to_usize(), None); let data = buffer.read(Some(size)).unwrap_or_default(); let value = Wtf8Buf::from_bytes(data) @@ -4589,20 +4579,13 @@ mod _io { #[pymethod] fn tell(&self, vm: &VirtualMachine) -> PyResult { let buffer = self.buffer(vm)?; - Ok( - Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)? - as u64, - ) + Ok(Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize) as u64) } #[pymethod] fn readline(&self, size: OptionalSize, vm: &VirtualMachine) -> PyResult { let mut buffer = self.buffer(vm)?; - let newline = match self.newline.load() { - Newlines::Passthrough => Newlines::Passthrough, - _ => Newlines::Lf, - }; - let size = Self::read_size(&buffer, size.to_usize(), Some(newline), vm)?; + let size = Self::read_size(&buffer, size.to_usize(), Some(self.newline.load())); let input = buffer.read(Some(size)).unwrap_or_default(); Wtf8Buf::from_bytes(input).map_err(|_| vm.new_value_error("Error Retrieving Value")) } @@ -4612,11 +4595,9 @@ mod _io { let mut buffer = self.buffer(vm)?; let pos = match pos.try_usize(vm)? { Some(pos) => pos, - None => { - Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)? - } + None => Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize), }; - let byte_pos = Self::char_offset_to_byte(buffer.cursor.get_ref(), pos, vm)?; + let byte_pos = Self::char_offset_to_byte(buffer.cursor.get_ref(), pos); buffer.truncate(Some(byte_pos)); Ok(pos) } @@ -4631,8 +4612,7 @@ mod _io { let buffer = zelf.buffer(vm)?; let content = Wtf8Buf::from_bytes(buffer.getvalue()) .map_err(|_| vm.new_value_error("Error Retrieving Value"))?; - let pos = - Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize, vm)?; + let pos = Self::byte_offset_to_char(buffer.cursor.get_ref(), buffer.tell() as usize); drop(buffer); // Get __dict__ if it exists and is non-empty @@ -4683,7 +4663,7 @@ mod _io { let raw_bytes = content.as_bytes().to_vec(); let mut buffer = zelf.buffer.write(); *buffer = BufferedIO::new(Cursor::new(raw_bytes)); - let byte_pos = Self::char_offset_to_byte(buffer.cursor.get_ref(), pos as usize, vm)?; + let byte_pos = Self::char_offset_to_byte(buffer.cursor.get_ref(), pos as usize); buffer .seek(SeekFrom::Start(byte_pos as u64)) .map_err(|err| os_err(vm, err))?;