Skip to content

Commit ed66cdf

Browse files
doma17doma17
committed
Report observed StringIO newline types
Constraint: Match CPython StringIO newline reporting without changing configured newline translation. Rejected: Separate newline-tracking representation | reuse the existing SeenNewline bitflags. Confidence: high Scope-risk: narrow Directive: Keep observed newline state separate from the configured newline mode. Tested: prek run --all-files; test_memoryio; cargo clippy -p rustpython-vm --lib -- -D warnings; workspace tests excluding the macOS C-API baseline SIGSEGV. Not-tested: Full macOS workspace suite is blocked by the existing rustpython-capi SIGSEGV; the Linux suite is running. Assisted-by: Codex:gpt-5.6-sol
1 parent 525ba8c commit ed66cdf

2 files changed

Lines changed: 63 additions & 35 deletions

File tree

‎Lib/test/test_memoryio.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1004,10 +1004,6 @@ def __str__(self):
10041004
def test_flags(self):
10051005
return super().test_flags()
10061006

1007-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'?
1008-
def test_newlines_property(self):
1009-
return super().test_newlines_property()
1010-
10111007
class CStringIOPickleTest(PyStringIOPickleTest):
10121008
UnsupportedOperation = io.UnsupportedOperation
10131009

@@ -1017,9 +1013,5 @@ def __new__(cls, *args, **kwargs):
10171013
def __init__(self, *args, **kwargs):
10181014
pass
10191015

1020-
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'?
1021-
def test_newlines_property(self):
1022-
return super().test_newlines_property()
1023-
10241016
if __name__ == '__main__':
10251017
unittest.main()

‎crates/vm/src/stdlib/_io.rs‎

Lines changed: 63 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4187,6 +4187,37 @@ mod _io {
41874187
}
41884188
}
41894189

4190+
impl SeenNewline {
4191+
fn observe(&mut self, text: &Wtf8) {
4192+
let bytes = text.as_bytes();
4193+
let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes);
4194+
while !self.is_all() {
4195+
let Some(i) = matches.next() else { break };
4196+
match bytes[i] {
4197+
b'\n' => self.insert(Self::LF),
4198+
_ if bytes.get(i + 1) == Some(&b'\n') => {
4199+
matches.next();
4200+
self.insert(Self::CRLF);
4201+
}
4202+
_ => self.insert(Self::CR),
4203+
}
4204+
}
4205+
}
4206+
4207+
fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef {
4208+
match self.bits() {
4209+
1 => "\n".to_pyobject(vm),
4210+
2 => "\r".to_pyobject(vm),
4211+
3 => ("\r", "\n").to_pyobject(vm),
4212+
4 => "\r\n".to_pyobject(vm),
4213+
5 => ("\n", "\r\n").to_pyobject(vm),
4214+
6 => ("\r", "\r\n").to_pyobject(vm),
4215+
7 => ("\r", "\n", "\r\n").to_pyobject(vm),
4216+
_ => vm.ctx.none(),
4217+
}
4218+
}
4219+
}
4220+
41904221
impl DefaultConstructor for IncrementalNewlineDecoder {}
41914222

41924223
#[derive(FromArgs)]
@@ -4278,16 +4309,7 @@ mod _io {
42784309
#[pygetset]
42794310
fn newlines(&self, vm: &VirtualMachine) -> PyResult {
42804311
let data = self.lock(vm)?;
4281-
Ok(match data.seennl.bits() {
4282-
1 => "\n".to_pyobject(vm),
4283-
2 => "\r".to_pyobject(vm),
4284-
3 => ("\r", "\n").to_pyobject(vm),
4285-
4 => "\r\n".to_pyobject(vm),
4286-
5 => ("\n", "\r\n").to_pyobject(vm),
4287-
6 => ("\r", "\r\n").to_pyobject(vm),
4288-
7 => ("\r", "\n", "\r\n").to_pyobject(vm),
4289-
_ => vm.ctx.none(),
4290-
})
4312+
Ok(data.seennl.to_pyobject(vm))
42914313
}
42924314
}
42934315

@@ -4334,20 +4356,7 @@ mod _io {
43344356
self.seennl.insert(SeenNewline::LF);
43354357
}
43364358
} else if !self.translate {
4337-
let output = output.as_bytes();
4338-
let mut matches = memchr::memchr2_iter(b'\r', b'\n', output);
4339-
while !self.seennl.is_all() {
4340-
let Some(i) = matches.next() else { break };
4341-
match output[i] {
4342-
b'\n' => self.seennl.insert(SeenNewline::LF),
4343-
// if c isn't \n, it can only be \r
4344-
_ if output.get(i + 1) == Some(&b'\n') => {
4345-
matches.next();
4346-
self.seennl.insert(SeenNewline::CRLF);
4347-
}
4348-
_ => self.seennl.insert(SeenNewline::CR),
4349-
}
4350-
}
4359+
self.seennl.observe(&output);
43514360
} else {
43524361
let bytes = output.as_bytes();
43534362
let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes);
@@ -4390,6 +4399,7 @@ mod _io {
43904399
_base: _TextIOBase,
43914400
buffer: PyRwLock<BufferedIO>,
43924401
newline: AtomicCell<Newlines>,
4402+
seennl: AtomicCell<SeenNewline>,
43934403
closed: AtomicCell<bool>,
43944404
}
43954405

@@ -4410,6 +4420,7 @@ mod _io {
44104420
_base: Default::default(),
44114421
buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))),
44124422
newline: AtomicCell::new(Newlines::Lf),
4423+
seennl: AtomicCell::new(SeenNewline::empty()),
44134424
closed: AtomicCell::new(false),
44144425
})
44154426
}
@@ -4428,11 +4439,15 @@ mod _io {
44284439
OptionalArg::Present(None) => Newlines::Universal,
44294440
OptionalArg::Present(Some(newline)) => newline,
44304441
};
4431-
let raw_bytes = object.flatten().map_or_else(Vec::new, |v| {
4442+
let object = object.flatten();
4443+
let raw_bytes = object.as_ref().map_or_else(Vec::new, |v| {
44324444
Self::translate_newlines(v.as_wtf8(), newline).into_bytes()
44334445
});
44344446
*zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes));
44354447
zelf.newline.store(newline);
4448+
if let Some(object) = object {
4449+
zelf.observe_newlines(object.as_wtf8(), newline);
4450+
}
44364451
Ok(())
44374452
}
44384453
}
@@ -4457,6 +4472,14 @@ mod _io {
44574472
}
44584473
}
44594474

4475+
fn observe_newlines(&self, data: &Wtf8, newline: Newlines) {
4476+
if matches!(newline, Newlines::Universal | Newlines::Passthrough) {
4477+
let mut seennl = self.seennl.load();
4478+
seennl.observe(data);
4479+
self.seennl.store(seennl);
4480+
}
4481+
}
4482+
44604483
fn text(bytes: &[u8]) -> &Wtf8 {
44614484
// SAFETY: StringIO is populated only from PyStr values, which are valid WTF-8.
44624485
unsafe { Wtf8::from_bytes_unchecked(bytes) }
@@ -4509,6 +4532,11 @@ mod _io {
45094532
self.closed.load()
45104533
}
45114534

4535+
#[pygetset]
4536+
fn newlines(&self, vm: &VirtualMachine) -> PyObjectRef {
4537+
self.seennl.load().to_pyobject(vm)
4538+
}
4539+
45124540
#[pymethod]
45134541
fn close(&self) {
45144542
self.closed.store(true);
@@ -4517,8 +4545,11 @@ mod _io {
45174545
// write string to underlying vector
45184546
#[pymethod]
45194547
fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult<u64> {
4520-
let bytes = Self::translate_newlines(data.as_wtf8(), self.newline.load()).into_bytes();
4521-
self.buffer(vm)?
4548+
let newline = self.newline.load();
4549+
let bytes = Self::translate_newlines(data.as_wtf8(), newline).into_bytes();
4550+
let mut buffer = self.buffer(vm)?;
4551+
self.observe_newlines(data.as_wtf8(), newline);
4552+
buffer
45224553
.write(&bytes)
45234554
.ok_or_else(|| vm.new_type_error("Error Writing String"))?;
45244555
Ok(data.char_len() as u64)
@@ -4678,6 +4709,11 @@ mod _io {
46784709
.map_err(|err| os_err(vm, err))?;
46794710
drop(buffer);
46804711
zelf.newline.store(newline);
4712+
let mut seennl = SeenNewline::empty();
4713+
if matches!(newline, Newlines::Universal | Newlines::Passthrough) {
4714+
seennl.observe(content.as_wtf8());
4715+
}
4716+
zelf.seennl.store(seennl);
46814717

46824718
// Set __dict__ if provided
46834719
if !vm.is_none(dict) {

0 commit comments

Comments
 (0)