From ed66cdfeecb384b004a91e3df21da8091b16bdcc Mon Sep 17 00:00:00 2001 From: Kwak Byoung Min Date: Mon, 17 Aug 2026 13:45:45 +0900 Subject: [PATCH] Report observed StringIO newline types Constraint: Match CPython StringIO newline reporting without changing configured newline translation. Rejected: Separate newline-tracking representation | reuse the existing SeenNewline bitflags. Confidence: high Scope-risk: narrow Directive: Keep observed newline state separate from the configured newline mode. Tested: prek run --all-files; test_memoryio; cargo clippy -p rustpython-vm --lib -- -D warnings; workspace tests excluding the macOS C-API baseline SIGSEGV. Not-tested: Full macOS workspace suite is blocked by the existing rustpython-capi SIGSEGV; the Linux suite is running. Assisted-by: Codex:gpt-5.6-sol --- Lib/test/test_memoryio.py | 8 ---- crates/vm/src/stdlib/_io.rs | 90 ++++++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 1683a71fc88..2dd4133e84d 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -1004,10 +1004,6 @@ def __str__(self): def test_flags(self): return super().test_flags() - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? - def test_newlines_property(self): - return super().test_newlines_property() - class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation @@ -1017,9 +1013,5 @@ def __new__(cls, *args, **kwargs): def __init__(self, *args, **kwargs): pass - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? - def test_newlines_property(self): - return super().test_newlines_property() - if __name__ == '__main__': unittest.main() diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index ab1be4297ec..d3d7eb7d6a4 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -4187,6 +4187,37 @@ mod _io { } } + impl SeenNewline { + fn observe(&mut self, text: &Wtf8) { + let bytes = text.as_bytes(); + let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes); + while !self.is_all() { + let Some(i) = matches.next() else { break }; + match bytes[i] { + b'\n' => self.insert(Self::LF), + _ if bytes.get(i + 1) == Some(&b'\n') => { + matches.next(); + self.insert(Self::CRLF); + } + _ => self.insert(Self::CR), + } + } + } + + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + match self.bits() { + 1 => "\n".to_pyobject(vm), + 2 => "\r".to_pyobject(vm), + 3 => ("\r", "\n").to_pyobject(vm), + 4 => "\r\n".to_pyobject(vm), + 5 => ("\n", "\r\n").to_pyobject(vm), + 6 => ("\r", "\r\n").to_pyobject(vm), + 7 => ("\r", "\n", "\r\n").to_pyobject(vm), + _ => vm.ctx.none(), + } + } + } + impl DefaultConstructor for IncrementalNewlineDecoder {} #[derive(FromArgs)] @@ -4278,16 +4309,7 @@ mod _io { #[pygetset] fn newlines(&self, vm: &VirtualMachine) -> PyResult { let data = self.lock(vm)?; - Ok(match data.seennl.bits() { - 1 => "\n".to_pyobject(vm), - 2 => "\r".to_pyobject(vm), - 3 => ("\r", "\n").to_pyobject(vm), - 4 => "\r\n".to_pyobject(vm), - 5 => ("\n", "\r\n").to_pyobject(vm), - 6 => ("\r", "\r\n").to_pyobject(vm), - 7 => ("\r", "\n", "\r\n").to_pyobject(vm), - _ => vm.ctx.none(), - }) + Ok(data.seennl.to_pyobject(vm)) } } @@ -4334,20 +4356,7 @@ mod _io { self.seennl.insert(SeenNewline::LF); } } else if !self.translate { - let output = output.as_bytes(); - let mut matches = memchr::memchr2_iter(b'\r', b'\n', output); - while !self.seennl.is_all() { - let Some(i) = matches.next() else { break }; - match output[i] { - b'\n' => self.seennl.insert(SeenNewline::LF), - // if c isn't \n, it can only be \r - _ if output.get(i + 1) == Some(&b'\n') => { - matches.next(); - self.seennl.insert(SeenNewline::CRLF); - } - _ => self.seennl.insert(SeenNewline::CR), - } - } + self.seennl.observe(&output); } else { let bytes = output.as_bytes(); let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes); @@ -4390,6 +4399,7 @@ mod _io { _base: _TextIOBase, buffer: PyRwLock, newline: AtomicCell, + seennl: AtomicCell, closed: AtomicCell, } @@ -4410,6 +4420,7 @@ mod _io { _base: Default::default(), buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), newline: AtomicCell::new(Newlines::Lf), + seennl: AtomicCell::new(SeenNewline::empty()), closed: AtomicCell::new(false), }) } @@ -4428,11 +4439,15 @@ mod _io { OptionalArg::Present(None) => Newlines::Universal, OptionalArg::Present(Some(newline)) => newline, }; - let raw_bytes = object.flatten().map_or_else(Vec::new, |v| { + let object = object.flatten(); + let raw_bytes = object.as_ref().map_or_else(Vec::new, |v| { Self::translate_newlines(v.as_wtf8(), newline).into_bytes() }); *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); zelf.newline.store(newline); + if let Some(object) = object { + zelf.observe_newlines(object.as_wtf8(), newline); + } Ok(()) } } @@ -4457,6 +4472,14 @@ mod _io { } } + fn observe_newlines(&self, data: &Wtf8, newline: Newlines) { + if matches!(newline, Newlines::Universal | Newlines::Passthrough) { + let mut seennl = self.seennl.load(); + seennl.observe(data); + self.seennl.store(seennl); + } + } + fn text(bytes: &[u8]) -> &Wtf8 { // SAFETY: StringIO is populated only from PyStr values, which are valid WTF-8. unsafe { Wtf8::from_bytes_unchecked(bytes) } @@ -4509,6 +4532,11 @@ mod _io { self.closed.load() } + #[pygetset] + fn newlines(&self, vm: &VirtualMachine) -> PyObjectRef { + self.seennl.load().to_pyobject(vm) + } + #[pymethod] fn close(&self) { self.closed.store(true); @@ -4517,8 +4545,11 @@ mod _io { // write string to underlying vector #[pymethod] fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = Self::translate_newlines(data.as_wtf8(), self.newline.load()).into_bytes(); - self.buffer(vm)? + let newline = self.newline.load(); + let bytes = Self::translate_newlines(data.as_wtf8(), newline).into_bytes(); + let mut buffer = self.buffer(vm)?; + self.observe_newlines(data.as_wtf8(), newline); + buffer .write(&bytes) .ok_or_else(|| vm.new_type_error("Error Writing String"))?; Ok(data.char_len() as u64) @@ -4678,6 +4709,11 @@ mod _io { .map_err(|err| os_err(vm, err))?; drop(buffer); zelf.newline.store(newline); + let mut seennl = SeenNewline::empty(); + if matches!(newline, Newlines::Universal | Newlines::Passthrough) { + seennl.observe(content.as_wtf8()); + } + zelf.seennl.store(seennl); // Set __dict__ if provided if !vm.is_none(dict) {