From 1df14b35284aab5b2b91ee203c9e57b61c3d2bf8 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 17 Oct 2026 06:04:02 +0900 Subject: [PATCH 1/5] Align weak-reference collection with Python 3.15 finalization Trace and retain owned callback edges independently of weak referent links. Clear callback-bearing weakrefs before finalizers, suppress callbacks from unreachable weakrefs, and close remaining weakref ingress after resurrection checks while the world is stopped. Publish and unlink intrusive nodes safely. Refresh weakref and canonical weakref/GC/finalization tests through update_lib from pinned CPython 3.15.0. Retire four obsolete guards only after fresh body passes, retaining applicable RustPython limitations. Assisted-by: OpenAI Codex:model version unavailable --- Lib/test/test_finalization.py | 46 ++++--- Lib/test/test_gc.py | 112 +++++++++++++++-- Lib/test/test_weakref.py | 33 ++++- Lib/weakref.py | 6 +- crates/vm/src/gc_state.rs | 40 ++++-- crates/vm/src/object/core.rs | 224 ++++++++++++++++++---------------- 6 files changed, 316 insertions(+), 145 deletions(-) diff --git a/Lib/test/test_finalization.py b/Lib/test/test_finalization.py index 42871f8a09b..9dd68cf8d57 100644 --- a/Lib/test/test_finalization.py +++ b/Lib/test/test_finalization.py @@ -174,7 +174,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -188,12 +188,12 @@ def test_simple_resurrect(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) @support.cpython_only def test_non_gc(self): @@ -265,7 +265,7 @@ def test_simple(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) @@ -276,19 +276,24 @@ def test_simple_resurrect(self): s = SelfCycleResurrector() ids = [id(s)] wr = weakref.ref(s) + wrc = weakref.ref(s, lambda x: None) del s gc.collect() self.assert_del_calls(ids) self.assert_survivors(ids) - # XXX is this desirable? - self.assertIs(wr(), None) + # This used to be None because weakrefs were cleared before + # calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + # A weakref with a callback is still cleared before calling + # finalizers. + self.assertIsNone(wrc()) # When trying to destroy the object a second time, __del__ # isn't called anymore (and the object isn't resurrected). self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_simple_suicide(self): # Test the GC is able to deal with an object that kills its last @@ -301,11 +306,11 @@ def test_simple_suicide(self): gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) class ChainedBase: @@ -378,18 +383,27 @@ def check_non_resurrecting_chain(self, classes): def check_resurrecting_chain(self, classes): N = len(classes) + def dummy_callback(ref): + pass with SimpleBase.test(): nodes = self.build_chain(classes) N = len(nodes) ids = [id(s) for s in nodes] survivor_ids = [id(s) for s in nodes if isinstance(s, SimpleResurrector)] wrs = [weakref.ref(s) for s in nodes] + wrcs = [weakref.ref(s, dummy_callback) for s in nodes] del nodes gc.collect() self.assert_del_calls(ids) self.assert_survivors(survivor_ids) - # XXX desirable? - self.assertEqual([wr() for wr in wrs], [None] * N) + for wr in wrs: + # These values used to be None because weakrefs were cleared + # before calling finalizers. Now they are cleared after. + self.assertIsNotNone(wr()) + for wr in wrcs: + # Weakrefs with callbacks are still cleared before calling + # finalizers. + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) @@ -491,7 +505,7 @@ def test_legacy(self): self.assert_del_calls(ids) self.assert_tp_del_calls(ids) self.assert_survivors([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids) @@ -507,13 +521,13 @@ def test_legacy_resurrect(self): self.assert_tp_del_calls(ids) self.assert_survivors(ids) # weakrefs are cleared before tp_del is called. - self.assertIs(wr(), None) + self.assertIsNone(wr()) self.clear_survivors() gc.collect() self.assert_del_calls(ids) self.assert_tp_del_calls(ids * 2) self.assert_survivors(ids) - self.assertIs(wr(), None) + self.assertIsNone(wr()) def test_legacy_self_cycle(self): # Self-cycles with legacy finalizers end up in gc.garbage. @@ -527,11 +541,11 @@ def test_legacy_self_cycle(self): self.assert_tp_del_calls([]) self.assert_survivors([]) self.assert_garbage(ids) - self.assertIsNot(wr(), None) + self.assertIsNotNone(wr()) # Break the cycle to allow collection gc.garbage[0].ref = None self.assert_garbage([]) - self.assertIs(wr(), None) + self.assertIsNone(wr()) if __name__ == "__main__": diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 3ed2aabafcb..c477006c9cc 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -236,7 +236,6 @@ def test_function(self): # is 3 because it includes f's code object. self.assertIn(gc.collect(), (2, 3)) - @unittest.expectedFailure # TODO: RUSTPYTHON; - weakref clear ordering differs from 3.15+ def test_function_tp_clear_leaves_consistent_state(self): # https://github.com/python/cpython/issues/91636 code = """if 1: @@ -310,6 +309,26 @@ def func(): self.assertRegex(stdout, rb"""\A\s*func=None""") self.assertFalse(stderr) + def test_datetime_weakref_cycle(self): + # https://github.com/python/cpython/issues/132413 + # If the weakref used by the datetime extension gets cleared by the GC (due to being + # in an unreachable cycle) then datetime functions would crash (get_module_state() + # was returning a NULL pointer). This bug is fixed by clearing weakrefs without + # callbacks *after* running finalizers. + code = """if 1: + import _datetime + class C: + def __del__(self): + print('__del__ called') + _datetime.timedelta(days=1) # crash? + + l = [C()] + l.append(l) + """ + rc, stdout, stderr = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(stdout.strip(), b'__del__ called') + @refcount_test def test_frame(self): def f(): @@ -667,9 +686,8 @@ def callback(ignored): gc.collect() self.assertEqual(len(ouch), 2) # else the callbacks didn't run for x in ouch: - # If the callback resurrected one of these guys, the instance - # would be damaged, with an empty __dict__. - self.assertEqual(x, None) + # The weakref should be cleared before executing the callback. + self.assertIsNone(x) def test_bug21435(self): # This is a poor test - its only virtue is that it happened to @@ -831,17 +849,20 @@ def __del__(self): rc, out, err = assert_python_ok(TESTFN) self.assertEqual(out.strip(), b'__del__ called') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_stats(self): stats = gc.get_stats() self.assertEqual(len(stats), 3) for st in stats: self.assertIsInstance(st, dict) - self.assertEqual(set(st), - {"collected", "collections", "uncollectable"}) + self.assertEqual( + set(st), + {"collected", "collections", "uncollectable", "candidates", "duration"} + ) self.assertGreaterEqual(st["collected"], 0) self.assertGreaterEqual(st["collections"], 0) self.assertGreaterEqual(st["uncollectable"], 0) + self.assertGreaterEqual(st["candidates"], 0) + self.assertGreaterEqual(st["duration"], 0) # Check that collection counts are incremented correctly if gc.isenabled(): self.addCleanup(gc.enable) @@ -852,11 +873,25 @@ def test_get_stats(self): self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) self.assertEqual(new[1]["collections"], old[1]["collections"]) self.assertEqual(new[2]["collections"], old[2]["collections"]) + self.assertGreater(new[0]["duration"], old[0]["duration"]) + self.assertEqual(new[1]["duration"], old[1]["duration"]) + self.assertEqual(new[2]["duration"], old[2]["duration"]) + for stat in ["collected", "uncollectable", "candidates"]: + self.assertGreaterEqual(new[0][stat], old[0][stat]) + self.assertEqual(new[1][stat], old[1][stat]) + self.assertEqual(new[2][stat], old[2][stat]) gc.collect(2) - new = gc.get_stats() - self.assertEqual(new[0]["collections"], old[0]["collections"] + 1) + old, new = new, gc.get_stats() + self.assertEqual(new[0]["collections"], old[0]["collections"]) self.assertEqual(new[1]["collections"], old[1]["collections"]) self.assertEqual(new[2]["collections"], old[2]["collections"] + 1) + self.assertEqual(new[0]["duration"], old[0]["duration"]) + self.assertEqual(new[1]["duration"], old[1]["duration"]) + self.assertGreater(new[2]["duration"], old[2]["duration"]) + for stat in ["collected", "uncollectable", "candidates"]: + self.assertEqual(new[0][stat], old[0][stat]) + self.assertEqual(new[1][stat], old[1][stat]) + self.assertGreaterEqual(new[2][stat], old[2][stat]) def test_freeze(self): gc.freeze() @@ -1204,6 +1239,37 @@ def test_something(self): """) assert_python_ok("-c", source) + def test_do_not_cleanup_type_subclasses_before_finalization(self): + # See https://github.com/python/cpython/issues/135552 + # If we cleanup weakrefs for tp_subclasses before calling + # the finalizer (__del__) then the line `fail = BaseNode.next.next` + # should fail because we are trying to access a subclass + # attribute. But subclass type cache was not properly invalidated. + code = """ + class BaseNode: + def __del__(self): + BaseNode.next = BaseNode.next.next + fail = BaseNode.next.next + + class Node(BaseNode): + pass + + BaseNode.next = Node() + BaseNode.next.next = Node() + """ + # this test checks garbage collection while interp + # finalization + assert_python_ok("-c", textwrap.dedent(code)) + + code_inside_function = textwrap.dedent(F""" + def test(): + {textwrap.indent(code, ' ')} + + test() + """) + # this test checks regular garbage collection + assert_python_ok("-c", code_inside_function) + @unittest.expectedFailure # TODO: RUSTPYTHON; long_lived_total does not count untracked tuples @unittest.skipUnless(Py_GIL_DISABLED, "requires free-threaded GC") @@ -1308,9 +1374,11 @@ def test_collect(self): # Check that we got the right info dict for all callbacks for v in self.visit: info = v[2] - self.assertTrue("generation" in info) - self.assertTrue("collected" in info) - self.assertTrue("uncollectable" in info) + self.assertIn("generation", info) + self.assertIn("collected", info) + self.assertIn("uncollectable", info) + self.assertIn("candidates", info) + self.assertIn("duration", info) def test_collect_generation(self): self.preclean() @@ -1498,6 +1566,7 @@ def callback(ignored): self.assertEqual(x, None) @gc_threshold(1000, 0, 0) + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820d(self): # Corresponds to temp2d.py in the bug report. This is very much like # test_bug1055820c, but uses a __del__ method instead of a weakref @@ -1595,6 +1664,25 @@ def test_indirect_calls_with_gc_disabled(self): finally: gc.enable() + def test_get_count_nonnegative(self): + xs = [[] for _ in range(1500)] + gc.collect() + del xs[:1000] + self.assertGreaterEqual(gc.get_count()[0], 0) + + @gc_threshold(1000, 0, 0) + def test_get_count_does_not_prevent_collection(self): + junk = [] + gc.collect() + detector = GC_Detector() + for _ in range(10000): + junk.append([]) + gc.get_count() + if detector.gc_happened: + break + else: + self.fail("gc didn't happen after 10000 iterations") + # Ensure that setting *threshold0* to zero disables collection. @gc_threshold(0) def test_threshold_zero(self): diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 5b5b50effc1..631611ead1f 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -751,7 +751,6 @@ class D: del c1, c2, C, D gc.collect() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_callback_in_cycle_resurrection(self): import gc @@ -798,7 +797,6 @@ def C_went_away(ignore): gc.collect() self.assertEqual(alist, []) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_callbacks_on_callback(self): import gc @@ -1046,6 +1044,32 @@ def callback(obj): stderr = res.err.decode("ascii", "backslashreplace") self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + def test_clearing_weakrefs_in_gc(self): + # This test checks that when finalizers are called: + # 1. weakrefs with callbacks have been cleared + # 2. weakrefs without callbacks have not been cleared + errors = [] + def test(): + class Class: + def __init__(self): + self._self = self + self.wr1 = weakref.ref(Class, lambda x: None) + self.wr2 = weakref.ref(Class) + + def __del__(self): + # we can't use assert* here, because gc will swallow + # exceptions + if self.wr1() is not None: + errors.append("weakref with callback as cleared") + if self.wr2() is not Class: + errors.append("weakref without callback was cleared") + + Class() + + test() + gc.collect() + self.assertEqual(errors, []) + class SubclassableWeakrefTestCase(TestBase): @@ -1791,6 +1815,11 @@ def test_weak_valued_union_operators(self): def test_weak_keyed_dict_update(self): self.check_update(weakref.WeakKeyDictionary, {C(): 1, C(): 2, C(): 3}) + d = weakref.WeakKeyDictionary() + msg = ("Keyword arguments are not supported: " + "cannot create weak reference to 'str' object") + with self.assertRaisesRegex(TypeError, msg): + d.update(k='v') def test_weak_keyed_delitem(self): d = weakref.WeakKeyDictionary() diff --git a/Lib/weakref.py b/Lib/weakref.py index 94e4278143c..af7244553c9 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -408,14 +408,16 @@ def setdefault(self, key, default=None): return self.data.setdefault(ref(key, self._remove),default) def update(self, dict=None, /, **kwargs): + if kwargs: + msg = ("Keyword arguments are not supported: " + "cannot create weak reference to 'str' object") + raise TypeError(msg) d = self.data if dict is not None: if not hasattr(dict, "items"): dict = type({})(dict) for key, value in dict.items(): d[ref(key, self._remove)] = value - if len(kwargs): - self.update(kwargs) def __ior__(self, other): self.update(other) diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index d78d9c1b932..38d31aec713 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -921,11 +921,15 @@ impl GcState { }) .collect(); - // 6c: Clear existing weakrefs BEFORE calling __del__ + // 6c: Clear callback-bearing weakrefs BEFORE calling __del__. Keep + // callback-free weakrefs usable throughout finalization. + let unreachable_set: GcSet = unreachable.iter().copied().collect(); + let is_unreachable = + |obj: &PyObject| unreachable_set.contains(&GcPtr(core::ptr::NonNull::from(obj))); let mut all_callbacks: Vec<(crate::PyRef, crate::PyObjectRef)> = Vec::new(); for obj_ref in &unreachable_refs { - let callbacks = obj_ref.gc_clear_weakrefs_collect_callbacks(); + let callbacks = obj_ref.gc_clear_weakrefs_collect_callbacks(&is_unreachable); all_callbacks.extend(callbacks); } for (wr, cb) in all_callbacks { @@ -945,7 +949,6 @@ impl GcState { // Detect resurrection let mut resurrected_set: GcSet = GcSet::default(); - let unreachable_set: GcSet = unreachable.iter().copied().collect(); for obj in &unreachable_refs { let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); @@ -1039,6 +1042,14 @@ impl GcState { // skips tp_clear for saved objects). let save_all = debug.contains(GcDebugFlags::SAVEALL); + // Callback-free weakrefs remain usable during finalizers, so a + // thread can still acquire an external reference at this point. + // Stop before taking generation/list locks, and keep the final + // liveness check and weakref closure in the same stopped phase. + // No callbacks, finalizers or tp_clear may run under this guard. + #[cfg(feature = "threading")] + let mut clear_stw = CollectStopTheWorld::new(); + // Untrack dead objects BEFORE clearing them, mirroring the // untrack-then-clear ordering of the refcount dealloc path. // A cleared object (e.g. a frame husk with iframe == None) must @@ -1056,11 +1067,10 @@ impl GcState { // One strong reference held by the `truly_dead` vec itself. expected_counts.insert(GcPtr(NonNull::from(obj)), 1); } - // With the objects out of the generation lists, no new external - // reference can appear. Count the references coming from within - // the dead set; any surplus in strong_count means another thread - // grabbed a reference before untracking (late resurrection) and - // the object must not be cleared. + // With the world stopped and these objects off the generation + // lists, count references from within the dead set. Any surplus + // means an external reference appeared before this stopped phase + // (late resurrection), and the object must not be cleared. let mut referents: GcMap>> = GcMap::default(); for obj_ref in &truly_dead { let referent_ptrs = unsafe { obj_ref.gc_get_referent_ptrs() }; @@ -1106,6 +1116,20 @@ impl GcState { unsafe { self.track_object(ptr.0, owner) }; } } + // Clear every weakref in or pointing to objects that will be + // cleared, including weakrefs created by finalizers. Do not run + // or drop callbacks here: all objects must remain intact until + // this pass finishes. Resurrected objects retain their weakrefs. + for obj_ref in &truly_dead { + if !late_resurrected.contains(&GcPtr(NonNull::from(obj_ref.as_ref()))) { + obj_ref.gc_clear_weakrefs(); + } + } + // Every remaining dead object is now unreachable through both + // generation lists and weakrefs. Resume before tp_clear can release + // edges or invoke arbitrary Python through deallocation. + #[cfg(feature = "threading")] + clear_stw.restart(); rustpython_common::refcount::with_deferred_drops(|| { if !save_all { for obj_ref in &truly_dead { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index fa2e7ae648a..651496b4ba6 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -24,7 +24,6 @@ use alloc::fmt; use core::{ any::TypeId, borrow::Borrow, - cell::UnsafeCell, marker::PhantomData, mem::ManuallyDrop, num::NonZeroUsize, @@ -740,8 +739,8 @@ impl WeakRefList { // hashes to the same stripe, which would deadlock on the spinlock. let weak_payload = PyWeak { pointers: Pointers::new(), - wr_object: Radium::new(obj as *const PyObject as *mut PyObject), - callback: UnsafeCell::new(callback), + wr_object: Radium::new(ptr::null_mut()), + callback: PyRwLock::new(callback), hash: Radium::new(crate::common::hash::SENTINEL), }; let weak = PyRef::new_ref(weak_payload, cls, dict); @@ -760,7 +759,7 @@ impl WeakRefList { None }; if let Some(existing) = existing { - // Nullify wr_object so drop_inner won't unlink an + // Nullify wr_object so clear_ref won't unlink an // un-inserted node (which would corrupt the list head). weak.wr_object.store(ptr::null_mut(), Ordering::Relaxed); return existing; @@ -784,6 +783,10 @@ impl WeakRefList { if is_generic { self.generic.store(node_ptr.as_ptr(), Ordering::Relaxed); } + // Only linked nodes may advertise a referent: an allocation unwind + // must never try to unlink an unpublished node. + weak.wr_object + .store(obj as *const PyObject as *mut PyObject, Ordering::Release); weak } @@ -828,7 +831,7 @@ impl WeakRefList { match NonNull::new(candidate_ptr) { Some(candidate) => { let node = unsafe { candidate.as_ref() }; - let has_callback = unsafe { (&*node.payload.callback.get()).is_some() }; + let has_callback = node.payload.callback.read().is_some(); let node_cls = node.class(); // PyWeakref_CheckProxy: the basic-proxy slot is reserved for // the canonical proxy type; subclasses and callback-less ref @@ -849,97 +852,93 @@ impl WeakRefList { /// Called when the owner object is being dropped. // PyObject_ClearWeakRefs fn clear(&self, obj: &PyObject) { - let obj_addr = obj as *const PyObject as usize; - let _lock = weakref_lock::lock(obj_addr); - - // Clear generic cache + // Owners precede the guard so unwinding also drops them after unlock. + let mut callbacks: Vec<(PyRef, Option)> = Vec::new(); + let _lock = weakref_lock::lock(obj as *const PyObject as usize); self.generic.store(ptr::null_mut(), Ordering::Relaxed); - - // Walk the list, collecting weakrefs with callbacks - let mut callbacks: Vec<(PyRef, PyObjectRef)> = Vec::new(); let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); while let Some(node) = current { let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; - let wr = unsafe { node.as_ref() }; - - // Mark weakref as dead + // Reserve before acquiring a Python owner; push cannot then unwind. + callbacks.reserve(1); + if wr.ref_count.safe_inc() { + let owner = unsafe { PyRef::::from_raw(wr as *const Py) }; + callbacks.push((owner, None)); + // Keep the owner in the outer batch before accessing a lock. + let entry = callbacks.last_mut().unwrap(); + entry.1 = wr.payload.callback.write().take(); + } + unsafe { unlink_weakref(self, node) }; + // This must be the last access if safe_inc failed: a concurrent + // deallocator may skip the stripe after observing the null pointer. wr.payload .wr_object - .store(ptr::null_mut(), Ordering::Relaxed); - - // Unlink from list - unsafe { - let mut ptrs = WeakLink::pointers(node); - ptrs.as_mut().set_prev(None); - ptrs.as_mut().set_next(None); + .store(ptr::null_mut(), Ordering::Release); + current = next; + } + drop(_lock); + for (wr, callback) in callbacks { + if let Some(cb) = callback { + crate::vm::thread::with_vm(&cb, |vm| { + let _ = cb.call((wr.clone(),), vm); + }); } + } + } - // Collect callback only if we can still acquire a strong ref. - if wr.ref_count.safe_inc() { - let wr_ref = unsafe { PyRef::::from_raw(wr as *const Py) }; - let cb = unsafe { wr.payload.callback.get().replace(None) }; - if let Some(cb) = cb { - callbacks.push((wr_ref, cb)); + /// Clear callback-bearing weakrefs before finalizers. Retain callback + /// ownership on each weakref until its later tp_clear or deallocation. + fn clear_for_gc_collect_callbacks( + &self, + obj: &PyObject, + is_unreachable: &impl Fn(&PyObject) -> bool, + ) -> Vec<(PyRef, PyObjectRef)> { + let mut owners: Vec> = Vec::new(); + let mut callbacks = Vec::new(); + let _lock = weakref_lock::lock(obj as *const PyObject as usize); + let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); + while let Some(node) = current { + let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; + let wr = unsafe { node.as_ref() }; + if wr.payload.callback.read().is_some() { + owners.reserve(1); + callbacks.reserve(1); + if !is_unreachable(wr.as_object()) && wr.ref_count.safe_inc() { + let owner = unsafe { PyRef::::from_raw(wr as *const Py) }; + owners.push(owner); + let callback = wr.payload.callback.read().clone(); + if let Some(callback) = callback { + callbacks.push((owners.last().unwrap().clone(), callback)); + } } + unsafe { unlink_weakref(self, node) }; + // No node access after publishing null without a strong pin. + wr.payload + .wr_object + .store(ptr::null_mut(), Ordering::Release); } - current = next; } - self.head.store(ptr::null_mut(), Ordering::Relaxed); - - // Invoke callbacks outside the lock drop(_lock); - for (wr, cb) in callbacks { - crate::vm::thread::with_vm(&cb, |vm| { - let _ = cb.call((wr.clone(),), vm); - }); - } + callbacks } - /// Clear all weakrefs but DON'T call callbacks. Instead, return them for later invocation. - /// Used by GC to ensure ALL weakrefs are cleared BEFORE any callbacks are invoked. - /// handle_weakrefs() clears all weakrefs first, then invokes callbacks. - fn clear_for_gc_collect_callbacks(&self, obj: &PyObject) -> Vec<(PyRef, PyObjectRef)> { - let obj_addr = obj as *const PyObject as usize; - let _lock = weakref_lock::lock(obj_addr); - - // Clear generic cache + /// Unlink remaining weakrefs without invoking or releasing callbacks. + fn clear_for_gc(&self, obj: &PyObject) { + let _lock = weakref_lock::lock(obj as *const PyObject as usize); self.generic.store(ptr::null_mut(), Ordering::Relaxed); - - let mut callbacks = Vec::new(); let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); while let Some(node) = current { let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; - let wr = unsafe { node.as_ref() }; - - // Mark weakref as dead + unsafe { unlink_weakref(self, node) }; + // Publishing null is the final access to this unpinned node. wr.payload .wr_object - .store(ptr::null_mut(), Ordering::Relaxed); - - // Unlink from list - unsafe { - let mut ptrs = WeakLink::pointers(node); - ptrs.as_mut().set_prev(None); - ptrs.as_mut().set_next(None); - } - - // Collect callback without invoking only if we can keep weakref alive. - if wr.ref_count.safe_inc() { - let wr_ref = unsafe { PyRef::::from_raw(wr as *const Py) }; - let cb = unsafe { wr.payload.callback.get().replace(None) }; - if let Some(cb) = cb { - callbacks.push((wr_ref, cb)); - } - } - + .store(ptr::null_mut(), Ordering::Release); current = next; } - self.head.store(ptr::null_mut(), Ordering::Relaxed); - - callbacks } fn count(&self, obj: &PyObject) -> usize { @@ -956,11 +955,12 @@ impl WeakRefList { } fn get_weak_references(&self, obj: &PyObject) -> Vec> { - let _lock = weakref_lock::lock(obj as *const PyObject as usize); let mut v = Vec::new(); + let _lock = weakref_lock::lock(obj as *const PyObject as usize); let mut current = NonNull::new(self.head.load(Ordering::Relaxed)); while let Some(node) = current { let wr = unsafe { node.as_ref() }; + v.reserve(1); if wr.ref_count.safe_inc() { v.push(unsafe { PyRef::::from_raw(wr as *const Py) }); } @@ -1000,15 +1000,16 @@ unsafe impl Link for WeakLink { } // PyWeakReference: each weakref holds a direct pointer to its referent. -#[pyclass(name = "ReferenceType", module = "weakref")] +#[pyclass(name = "ReferenceType", module = "weakref", traverse = "manual")] #[derive(Debug)] pub struct PyWeak { pointers: Pointers>, /// Direct pointer to the referent object, null when dead. /// Equivalent to wr_object in PyWeakReference. wr_object: PyAtomic<*mut PyObject>, - /// Protected by stripe lock (keyed on wr_object address). - callback: UnsafeCell>, + /// Owned callback edge, protected independently of the referent list. + /// GC can retain this edge after unlinking the weakref. + callback: PyRwLock>, pub(crate) hash: PyAtomic, } @@ -1048,31 +1049,13 @@ impl PyWeak { self.wr_object.load(Ordering::Acquire).is_null() } - /// Get the callback associated with this weak reference. - /// Returns `None` if there is no callback or if the referent has been - /// collected (at which point the callback was already consumed). + /// Get the owned callback, which GC can retain after clearing the referent. pub(crate) fn get_callback(&self) -> Option { - let obj_ptr = self.wr_object.load(Ordering::Acquire); - if obj_ptr.is_null() { - // Dead weakref: callback was consumed during clear - return None; - } - - let _lock = weakref_lock::lock(obj_ptr as usize); - - // Double-check under lock (clear may have run between our check and lock) - let obj_ptr = self.wr_object.load(Ordering::Relaxed); - if obj_ptr.is_null() { - return None; - } - - // Safety: we hold the stripe lock that protects the callback field - let callback = unsafe { &*self.callback.get() }; - callback.clone() + self.callback.read().clone() } - /// weakref_dealloc: remove from list if still linked. - fn drop_inner(&self) { + /// _PyWeakref_ClearRef: unlink without releasing the callback. + fn clear_ref(&self) { let obj_ptr = self.wr_object.load(Ordering::Acquire); if obj_ptr.is_null() { return; // Already cleared by WeakRefList::clear() @@ -1106,12 +1089,31 @@ impl PyWeak { } } +// SAFETY: the callback is the only owned Python edge; wr_object is borrowed. +unsafe impl Traverse for PyWeak { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + // A stopped thread may hold this lock. Missing a busy edge can + // postpone collection, but must not deadlock the collector. + if let Some(callback) = self.callback.try_read() { + callback.traverse(tracer_fn); + } + } + + fn clear(&mut self, out: &mut Vec) { + self.clear_ref(); + let callback = self.callback.write().take(); + if let Some(callback) = callback { + out.push(callback); + } + } +} + impl Drop for PyWeak { #[inline(always)] fn drop(&mut self) { // we do NOT have actual exclusive access! let me: &Self = self; - me.drop_inner(); + me.clear_ref(); } } @@ -2133,7 +2135,7 @@ impl PyObject { } // Clear weak refs AFTER __del__. - // Note: This differs from GC behavior which clears weakrefs before finalizers, + // GC clears callback-bearing weakrefs before finalizers and the rest later, // but for direct deallocation (drop_slow_inner), we need to allow the finalizer // to run without triggering use-after-free from WeakRefList operations. if let Some(wrl) = self.weak_ref_list() { @@ -2260,18 +2262,30 @@ impl PyObject { } } - /// Clear weakrefs but collect callbacks instead of calling them. - /// This is used by GC to ensure ALL weakrefs are cleared BEFORE any callbacks run. - /// Returns collected callbacks as (PyRef, callback) pairs. - // = handle_weakrefs - pub fn gc_clear_weakrefs_collect_callbacks(&self) -> Vec<(PyRef, PyObjectRef)> { + /// Clear callback-bearing weakrefs before finalizers and return callbacks + /// belonging to reachable weakrefs for invocation outside the list lock. + pub fn gc_clear_weakrefs_collect_callbacks( + &self, + is_unreachable: &impl Fn(&Self) -> bool, + ) -> Vec<(PyRef, PyObjectRef)> { if let Some(wrl) = self.weak_ref_list() { - wrl.clear_for_gc_collect_callbacks(self) + wrl.clear_for_gc_collect_callbacks(self, is_unreachable) } else { vec![] } } + /// Clear weakrefs both in and pointing into the final unreachable set. + /// Callback edges remain owned until tp_clear or deallocation. + pub(crate) fn gc_clear_weakrefs(&self) { + if let Some(weakref) = self.downcast_ref::() { + weakref.clear_ref(); + } + if let Some(wrl) = self.weak_ref_list() { + wrl.clear_for_gc(self); + } + } + /// Get raw pointers to referents without incrementing reference counts. /// This is used during GC to avoid reference count manipulation. /// tp_traverse visits objects without incref From 429c7718e703b139fadc1b12f417c6bb9d76f351 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 17 Oct 2026 06:47:44 +0900 Subject: [PATCH 2/5] Refresh canonical IO tests for weakref finalization Migrate the complete CPython 3.15.0 test_io package at d220409 with the fail-closed legacy-map updater, preserving all ten existing guards. Retire the seven consolidated standalone suites and update io/file dependency paths with mapping regression tests. The canonical GC tests ignore ResourceWarning without retaining its source, so callback-free weakrefs correctly reflect surviving owners. This closes the stale test dependency exposed by the lifecycle change. Assisted-by: OpenAI Codex:model version unavailable --- Lib/test/test_bufio.py | 73 - Lib/test/test_io.py | 5125 ------------------- Lib/test/test_io/__init__.py | 28 + Lib/test/test_io/__main__.py | 4 + Lib/test/test_io/test_bufferedio.py | 1570 ++++++ Lib/test/{ => test_io}/test_file.py | 0 Lib/test/{ => test_io}/test_fileio.py | 0 Lib/test/test_io/test_general.py | 1432 ++++++ Lib/test/{ => test_io}/test_largefile.py | 6 +- Lib/test/{ => test_io}/test_memoryio.py | 0 Lib/test/test_io/test_signals.py | 280 + Lib/test/test_io/test_textio.py | 1704 ++++++ Lib/test/{ => test_io}/test_univnewlines.py | 0 Lib/test/test_io/utils.py | 318 ++ scripts/update_lib/deps.py | 8 +- scripts/update_lib/tests/test_deps.py | 12 + 16 files changed, 5352 insertions(+), 5208 deletions(-) delete mode 100644 Lib/test/test_bufio.py delete mode 100644 Lib/test/test_io.py create mode 100644 Lib/test/test_io/__init__.py create mode 100644 Lib/test/test_io/__main__.py create mode 100644 Lib/test/test_io/test_bufferedio.py rename Lib/test/{ => test_io}/test_file.py (100%) rename Lib/test/{ => test_io}/test_fileio.py (100%) create mode 100644 Lib/test/test_io/test_general.py rename Lib/test/{ => test_io}/test_largefile.py (98%) rename Lib/test/{ => test_io}/test_memoryio.py (100%) create mode 100644 Lib/test/test_io/test_signals.py create mode 100644 Lib/test/test_io/test_textio.py rename Lib/test/{ => test_io}/test_univnewlines.py (100%) create mode 100644 Lib/test/test_io/utils.py diff --git a/Lib/test/test_bufio.py b/Lib/test/test_bufio.py deleted file mode 100644 index cb9cb4d0bc7..00000000000 --- a/Lib/test/test_bufio.py +++ /dev/null @@ -1,73 +0,0 @@ -import unittest -from test.support import os_helper - -import io # C implementation. -import _pyio as pyio # Python implementation. - -# Simple test to ensure that optimizations in the IO library deliver the -# expected results. For best testing, run this under a debug-build Python too -# (to exercise asserts in the C code). - -lengths = list(range(1, 257)) + [512, 1000, 1024, 2048, 4096, 8192, 10000, - 16384, 32768, 65536, 1000000] - -class BufferSizeTest: - def try_one(self, s): - # Write s + "\n" + s to file, then open it and ensure that successive - # .readline()s deliver what we wrote. - - # Ensure we can open TESTFN for writing. - os_helper.unlink(os_helper.TESTFN) - - # Since C doesn't guarantee we can write/read arbitrary bytes in text - # files, use binary mode. - f = self.open(os_helper.TESTFN, "wb") - try: - # write once with \n and once without - f.write(s) - f.write(b"\n") - f.write(s) - f.close() - f = self.open(os_helper.TESTFN, "rb") - line = f.readline() - self.assertEqual(line, s + b"\n") - line = f.readline() - self.assertEqual(line, s) - line = f.readline() - self.assertFalse(line) # Must be at EOF - f.close() - finally: - os_helper.unlink(os_helper.TESTFN) - - def drive_one(self, pattern): - for length in lengths: - # Repeat string 'pattern' as often as needed to reach total length - # 'length'. Then call try_one with that string, a string one larger - # than that, and a string one smaller than that. Try this with all - # small sizes and various powers of 2, so we exercise all likely - # stdio buffer sizes, and "off by one" errors on both sides. - q, r = divmod(length, len(pattern)) - teststring = pattern * q + pattern[:r] - self.assertEqual(len(teststring), length) - self.try_one(teststring) - self.try_one(teststring + b"x") - self.try_one(teststring[:-1]) - - def test_primepat(self): - # A pattern with prime length, to avoid simple relationships with - # stdio buffer sizes. - self.drive_one(b"1234567890\00\01\02\03\04\05\06") - - def test_nullpat(self): - self.drive_one(b'\0' * 1000) - - -class CBufferSizeTest(BufferSizeTest, unittest.TestCase): - open = io.open - -class PyBufferSizeTest(BufferSizeTest, unittest.TestCase): - open = staticmethod(pyio.open) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py deleted file mode 100644 index 3c569c3c69e..00000000000 --- a/Lib/test/test_io.py +++ /dev/null @@ -1,5125 +0,0 @@ -"""Unit tests for the io module.""" - -# Tests of io are scattered over the test suite: -# * test_bufio - tests file buffering -# * test_memoryio - tests BytesIO and StringIO -# * test_fileio - tests FileIO -# * test_file - tests the file interface -# * test_io - tests everything else in the io module -# * test_univnewlines - tests universal newline support -# * test_largefile - tests operations on a file greater than 2**32 bytes -# (only enabled with -ulargefile) - -################################################################################ -# ATTENTION TEST WRITERS!!! -################################################################################ -# When writing tests for io, it's important to test both the C and Python -# implementations. This is usually done by writing a base test that refers to -# the type it is testing as an attribute. Then it provides custom subclasses to -# test both implementations. This file has lots of examples. -################################################################################ - -import abc -import array -import errno -import locale -import os -import pickle -import random -import signal -import sys -import textwrap -import threading -import time -import unittest -import warnings -import weakref -from collections import deque, UserList -from itertools import cycle, count -from test import support -from test.support.script_helper import ( - assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import ( - import_helper, is_apple, os_helper, threading_helper, warnings_helper, -) -from test.support.os_helper import FakePath - -import codecs -import io # C implementation of io -import _pyio as pyio # Python implementation of io - -try: - import ctypes -except ImportError: - def byteslike(*pos, **kw): - return array.array("b", bytes(*pos, **kw)) -else: - def byteslike(*pos, **kw): - """Create a bytes-like object having no string or sequence methods""" - data = bytes(*pos, **kw) - obj = EmptyStruct() - ctypes.resize(obj, len(data)) - memoryview(obj).cast("B")[:] = data - return obj - class EmptyStruct(ctypes.Structure): - pass - - -def _default_chunk_size(): - """Get the default TextIOWrapper chunk size""" - with open(__file__, "r", encoding="latin-1") as f: - return f._CHUNK_SIZE - -requires_alarm = unittest.skipUnless( - hasattr(signal, "alarm"), "test requires signal.alarm()" -) - - -class BadIndex: - def __index__(self): - 1/0 - -class MockRawIOWithoutRead: - """A RawIO implementation without read(), so as to exercise the default - RawIO.read() which calls readinto().""" - - def __init__(self, read_stack=()): - self._read_stack = list(read_stack) - self._write_stack = [] - self._reads = 0 - self._extraneous_reads = 0 - - def write(self, b): - self._write_stack.append(bytes(b)) - return len(b) - - def writable(self): - return True - - def fileno(self): - return 42 - - def readable(self): - return True - - def seekable(self): - return True - - def seek(self, pos, whence): - return 0 # wrong but we gotta return something - - def tell(self): - return 0 # same comment as above - - def readinto(self, buf): - self._reads += 1 - max_len = len(buf) - try: - data = self._read_stack[0] - except IndexError: - self._extraneous_reads += 1 - return 0 - if data is None: - del self._read_stack[0] - return None - n = len(data) - if len(data) <= max_len: - del self._read_stack[0] - buf[:n] = data - return n - else: - buf[:] = data[:max_len] - self._read_stack[0] = data[max_len:] - return max_len - - def truncate(self, pos=None): - return pos - -class CMockRawIOWithoutRead(MockRawIOWithoutRead, io.RawIOBase): - pass - -class PyMockRawIOWithoutRead(MockRawIOWithoutRead, pyio.RawIOBase): - pass - - -class MockRawIO(MockRawIOWithoutRead): - - def read(self, n=None): - self._reads += 1 - try: - return self._read_stack.pop(0) - except: - self._extraneous_reads += 1 - return b"" - -class CMockRawIO(MockRawIO, io.RawIOBase): - pass - -class PyMockRawIO(MockRawIO, pyio.RawIOBase): - pass - - -class MisbehavedRawIO(MockRawIO): - def write(self, b): - return super().write(b) * 2 - - def read(self, n=None): - return super().read(n) * 2 - - def seek(self, pos, whence): - return -123 - - def tell(self): - return -456 - - def readinto(self, buf): - super().readinto(buf) - return len(buf) * 5 - -class CMisbehavedRawIO(MisbehavedRawIO, io.RawIOBase): - pass - -class PyMisbehavedRawIO(MisbehavedRawIO, pyio.RawIOBase): - pass - - -class SlowFlushRawIO(MockRawIO): - def __init__(self): - super().__init__() - self.in_flush = threading.Event() - - def flush(self): - self.in_flush.set() - time.sleep(0.25) - -class CSlowFlushRawIO(SlowFlushRawIO, io.RawIOBase): - pass - -class PySlowFlushRawIO(SlowFlushRawIO, pyio.RawIOBase): - pass - - -class CloseFailureIO(MockRawIO): - closed = 0 - - def close(self): - if not self.closed: - self.closed = 1 - raise OSError - -class CCloseFailureIO(CloseFailureIO, io.RawIOBase): - pass - -class PyCloseFailureIO(CloseFailureIO, pyio.RawIOBase): - pass - - -class MockFileIO: - - def __init__(self, data): - self.read_history = [] - super().__init__(data) - - def read(self, n=None): - res = super().read(n) - self.read_history.append(None if res is None else len(res)) - return res - - def readinto(self, b): - res = super().readinto(b) - self.read_history.append(res) - return res - -class CMockFileIO(MockFileIO, io.BytesIO): - pass - -class PyMockFileIO(MockFileIO, pyio.BytesIO): - pass - - -class MockUnseekableIO: - def seekable(self): - return False - - def seek(self, *args): - raise self.UnsupportedOperation("not seekable") - - def tell(self, *args): - raise self.UnsupportedOperation("not seekable") - - def truncate(self, *args): - raise self.UnsupportedOperation("not seekable") - -class CMockUnseekableIO(MockUnseekableIO, io.BytesIO): - UnsupportedOperation = io.UnsupportedOperation - -class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): - UnsupportedOperation = pyio.UnsupportedOperation - - -class MockCharPseudoDevFileIO(MockFileIO): - # GH-95782 - # ftruncate() does not work on these special files (and CPython then raises - # appropriate exceptions), so truncate() does not have to be accounted for - # here. - def __init__(self, data): - super().__init__(data) - - def seek(self, *args): - return 0 - - def tell(self, *args): - return 0 - -class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): - pass - -class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): - pass - - -class MockNonBlockWriterIO: - - def __init__(self): - self._write_stack = [] - self._blocker_char = None - - def pop_written(self): - s = b"".join(self._write_stack) - self._write_stack[:] = [] - return s - - def block_on(self, char): - """Block when a given char is encountered.""" - self._blocker_char = char - - def readable(self): - return True - - def seekable(self): - return True - - def seek(self, pos, whence=0): - # naive implementation, enough for tests - return 0 - - def writable(self): - return True - - def write(self, b): - b = bytes(b) - n = -1 - if self._blocker_char: - try: - n = b.index(self._blocker_char) - except ValueError: - pass - else: - if n > 0: - # write data up to the first blocker - self._write_stack.append(b[:n]) - return n - else: - # cancel blocker and indicate would block - self._blocker_char = None - return None - self._write_stack.append(b) - return len(b) - -class CMockNonBlockWriterIO(MockNonBlockWriterIO, io.RawIOBase): - BlockingIOError = io.BlockingIOError - -class PyMockNonBlockWriterIO(MockNonBlockWriterIO, pyio.RawIOBase): - BlockingIOError = pyio.BlockingIOError - - -class IOTest(unittest.TestCase): - - def setUp(self): - os_helper.unlink(os_helper.TESTFN) - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def write_ops(self, f): - self.assertEqual(f.write(b"blah."), 5) - f.truncate(0) - self.assertEqual(f.tell(), 5) - f.seek(0) - - self.assertEqual(f.write(b"blah."), 5) - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.write(b"Hello."), 6) - self.assertEqual(f.tell(), 6) - self.assertEqual(f.seek(-1, 1), 5) - self.assertEqual(f.tell(), 5) - buffer = bytearray(b" world\n\n\n") - self.assertEqual(f.write(buffer), 9) - buffer[:] = b"*" * 9 # Overwrite our copy of the data - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.write(b"h"), 1) - self.assertEqual(f.seek(-1, 2), 13) - self.assertEqual(f.tell(), 13) - - self.assertEqual(f.truncate(12), 12) - self.assertEqual(f.tell(), 13) - self.assertRaises(TypeError, f.seek, 0.0) - - def read_ops(self, f, buffered=False): - data = f.read(5) - self.assertEqual(data, b"hello") - data = byteslike(data) - self.assertEqual(f.readinto(data), 5) - self.assertEqual(bytes(data), b" worl") - data = bytearray(5) - self.assertEqual(f.readinto(data), 2) - self.assertEqual(len(data), 5) - self.assertEqual(data[:2], b"d\n") - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.read(20), b"hello world\n") - self.assertEqual(f.read(1), b"") - self.assertEqual(f.readinto(byteslike(b"x")), 0) - self.assertEqual(f.seek(-6, 2), 6) - self.assertEqual(f.read(5), b"world") - self.assertEqual(f.read(0), b"") - self.assertEqual(f.readinto(byteslike()), 0) - self.assertEqual(f.seek(-6, 1), 5) - self.assertEqual(f.read(5), b" worl") - self.assertEqual(f.tell(), 10) - self.assertRaises(TypeError, f.seek, 0.0) - if buffered: - f.seek(0) - self.assertEqual(f.read(), b"hello world\n") - f.seek(6) - self.assertEqual(f.read(), b"world\n") - self.assertEqual(f.read(), b"") - f.seek(0) - data = byteslike(5) - self.assertEqual(f.readinto1(data), 5) - self.assertEqual(bytes(data), b"hello") - - LARGE = 2**31 - - def large_file_ops(self, f): - assert f.readable() - assert f.writable() - try: - self.assertEqual(f.seek(self.LARGE), self.LARGE) - except (OverflowError, ValueError): - self.skipTest("no largefile support") - self.assertEqual(f.tell(), self.LARGE) - self.assertEqual(f.write(b"xxx"), 3) - self.assertEqual(f.tell(), self.LARGE + 3) - self.assertEqual(f.seek(-1, 1), self.LARGE + 2) - self.assertEqual(f.truncate(), self.LARGE + 2) - self.assertEqual(f.tell(), self.LARGE + 2) - self.assertEqual(f.seek(0, 2), self.LARGE + 2) - self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1) - self.assertEqual(f.tell(), self.LARGE + 2) - self.assertEqual(f.seek(0, 2), self.LARGE + 1) - self.assertEqual(f.seek(-1, 2), self.LARGE) - self.assertEqual(f.read(2), b"x") - - def test_invalid_operations(self): - # Try writing on a file opened in read mode and vice-versa. - exc = self.UnsupportedOperation - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as fp: - self.assertRaises(exc, fp.read) - self.assertRaises(exc, fp.readline) - with self.open(os_helper.TESTFN, "wb") as fp: - self.assertRaises(exc, fp.read) - self.assertRaises(exc, fp.readline) - with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: - self.assertRaises(exc, fp.read) - self.assertRaises(exc, fp.readline) - with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: - self.assertRaises(exc, fp.write, b"blah") - self.assertRaises(exc, fp.writelines, [b"blah\n"]) - with self.open(os_helper.TESTFN, "rb") as fp: - self.assertRaises(exc, fp.write, b"blah") - self.assertRaises(exc, fp.writelines, [b"blah\n"]) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as fp: - self.assertRaises(exc, fp.write, "blah") - self.assertRaises(exc, fp.writelines, ["blah\n"]) - # Non-zero seeking from current or end pos - self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR) - self.assertRaises(exc, fp.seek, -1, self.SEEK_END) - - @support.cpython_only - def test_startup_optimization(self): - # gh-132952: Test that `io` is not imported at startup and that the - # __module__ of UnsupportedOperation is set to "io". - assert_python_ok("-S", "-c", textwrap.dedent( - """ - import sys - assert "io" not in sys.modules - try: - sys.stdin.truncate() - except Exception as e: - typ = type(e) - assert typ.__module__ == "io", (typ, typ.__module__) - assert typ.__name__ == "UnsupportedOperation", (typ, typ.__name__) - else: - raise AssertionError("Expected UnsupportedOperation") - """ - )) - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_optional_abilities(self): - # Test for OSError when optional APIs are not supported - # The purpose of this test is to try fileno(), reading, writing and - # seeking operations with various objects that indicate they do not - # support these operations. - - def pipe_reader(): - [r, w] = os.pipe() - os.close(w) # So that read() is harmless - return self.FileIO(r, "r") - - def pipe_writer(): - [r, w] = os.pipe() - self.addCleanup(os.close, r) - # Guarantee that we can write into the pipe without blocking - thread = threading.Thread(target=os.read, args=(r, 100)) - thread.start() - self.addCleanup(thread.join) - return self.FileIO(w, "w") - - def buffered_reader(): - return self.BufferedReader(self.MockUnseekableIO()) - - def buffered_writer(): - return self.BufferedWriter(self.MockUnseekableIO()) - - def buffered_random(): - return self.BufferedRandom(self.BytesIO()) - - def buffered_rw_pair(): - return self.BufferedRWPair(self.MockUnseekableIO(), - self.MockUnseekableIO()) - - def text_reader(): - class UnseekableReader(self.MockUnseekableIO): - writable = self.BufferedIOBase.writable - write = self.BufferedIOBase.write - return self.TextIOWrapper(UnseekableReader(), "ascii") - - def text_writer(): - class UnseekableWriter(self.MockUnseekableIO): - readable = self.BufferedIOBase.readable - read = self.BufferedIOBase.read - return self.TextIOWrapper(UnseekableWriter(), "ascii") - - tests = ( - (pipe_reader, "fr"), (pipe_writer, "fw"), - (buffered_reader, "r"), (buffered_writer, "w"), - (buffered_random, "rws"), (buffered_rw_pair, "rw"), - (text_reader, "r"), (text_writer, "w"), - (self.BytesIO, "rws"), (self.StringIO, "rws"), - ) - - def do_test(test, obj, abilities): - readable = "r" in abilities - self.assertEqual(obj.readable(), readable) - writable = "w" in abilities - self.assertEqual(obj.writable(), writable) - - if isinstance(obj, self.TextIOBase): - data = "3" - elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): - data = b"3" - else: - self.fail("Unknown base class") - - if "f" in abilities: - obj.fileno() - else: - self.assertRaises(OSError, obj.fileno) - - if readable: - obj.read(1) - obj.read() - else: - self.assertRaises(OSError, obj.read, 1) - self.assertRaises(OSError, obj.read) - - if writable: - obj.write(data) - else: - self.assertRaises(OSError, obj.write, data) - - if sys.platform.startswith("win") and test in ( - pipe_reader, pipe_writer): - # Pipes seem to appear as seekable on Windows - return - seekable = "s" in abilities - self.assertEqual(obj.seekable(), seekable) - - if seekable: - obj.tell() - obj.seek(0) - else: - self.assertRaises(OSError, obj.tell) - self.assertRaises(OSError, obj.seek, 0) - - if writable and seekable: - obj.truncate() - obj.truncate(0) - else: - self.assertRaises(OSError, obj.truncate) - self.assertRaises(OSError, obj.truncate, 0) - - for [test, abilities] in tests: - with self.subTest(test): - if test == pipe_writer and not threading_helper.can_start_thread: - self.skipTest("Need threads") - with test() as obj: - do_test(test, obj, abilities) - - - def test_open_handles_NUL_chars(self): - fn_with_NUL = 'foo\0bar' - self.assertRaises(ValueError, self.open, fn_with_NUL, 'w', encoding="utf-8") - - bytes_fn = bytes(fn_with_NUL, 'ascii') - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - self.assertRaises(ValueError, self.open, bytes_fn, 'w', encoding="utf-8") - - def test_raw_file_io(self): - with self.open(os_helper.TESTFN, "wb", buffering=0) as f: - self.assertEqual(f.readable(), False) - self.assertEqual(f.writable(), True) - self.assertEqual(f.seekable(), True) - self.write_ops(f) - with self.open(os_helper.TESTFN, "rb", buffering=0) as f: - self.assertEqual(f.readable(), True) - self.assertEqual(f.writable(), False) - self.assertEqual(f.seekable(), True) - self.read_ops(f) - - def test_buffered_file_io(self): - with self.open(os_helper.TESTFN, "wb") as f: - self.assertEqual(f.readable(), False) - self.assertEqual(f.writable(), True) - self.assertEqual(f.seekable(), True) - self.write_ops(f) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.readable(), True) - self.assertEqual(f.writable(), False) - self.assertEqual(f.seekable(), True) - self.read_ops(f, True) - - def test_readline(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"abc\ndef\nxyzzy\nfoo\x00bar\nanother line") - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.readline(), b"abc\n") - self.assertEqual(f.readline(10), b"def\n") - self.assertEqual(f.readline(2), b"xy") - self.assertEqual(f.readline(4), b"zzy\n") - self.assertEqual(f.readline(), b"foo\x00bar\n") - self.assertEqual(f.readline(None), b"another line") - self.assertRaises(TypeError, f.readline, 5.3) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - self.assertRaises(TypeError, f.readline, 5.3) - - def test_readline_nonsizeable(self): - # Issue #30061 - # Crash when readline() returns an object without __len__ - class R(self.IOBase): - def readline(self): - return None - self.assertRaises((TypeError, StopIteration), next, R()) - - def test_next_nonsizeable(self): - # Issue #30061 - # Crash when __next__() returns an object without __len__ - class R(self.IOBase): - def __next__(self): - return None - self.assertRaises(TypeError, R().readlines, 1) - - def test_raw_bytes_io(self): - f = self.BytesIO() - self.write_ops(f) - data = f.getvalue() - self.assertEqual(data, b"hello world\n") - f = self.BytesIO(data) - self.read_ops(f, True) - - def test_large_file_ops(self): - # On Windows and Apple platforms this test consumes large resources; It - # takes a long time to build the >2 GiB file and takes >2 GiB of disk - # space therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or is_apple: - support.requires( - 'largefile', - 'test requires %s bytes and a long time to run' % self.LARGE) - with self.open(os_helper.TESTFN, "w+b", 0) as f: - self.large_file_ops(f) - with self.open(os_helper.TESTFN, "w+b") as f: - self.large_file_ops(f) - - def test_with_open(self): - for bufsize in (0, 100): - with self.open(os_helper.TESTFN, "wb", bufsize) as f: - f.write(b"xxx") - self.assertEqual(f.closed, True) - try: - with self.open(os_helper.TESTFN, "wb", bufsize) as f: - 1/0 - except ZeroDivisionError: - self.assertEqual(f.closed, True) - else: - self.fail("1/0 didn't raise an exception") - - # issue 5008 - def test_append_mode_tell(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"xxx") - with self.open(os_helper.TESTFN, "ab", buffering=0) as f: - self.assertEqual(f.tell(), 3) - with self.open(os_helper.TESTFN, "ab") as f: - self.assertEqual(f.tell(), 3) - with self.open(os_helper.TESTFN, "a", encoding="utf-8") as f: - self.assertGreater(f.tell(), 0) - - def test_destructor(self): - record = [] - class MyFileIO(self.FileIO): - def __del__(self): - record.append(1) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(2) - super().close() - def flush(self): - record.append(3) - super().flush() - with warnings_helper.check_warnings(('', ResourceWarning)): - f = MyFileIO(os_helper.TESTFN, "wb") - f.write(b"xxx") - del f - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"xxx") - - def _check_base_destructor(self, base): - record = [] - class MyIO(base): - def __init__(self): - # This exercises the availability of attributes on object - # destruction. - # (in the C version, close() is called by the tp_dealloc - # function, not by __del__) - self.on_del = 1 - self.on_close = 2 - self.on_flush = 3 - def __del__(self): - record.append(self.on_del) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(self.on_close) - super().close() - def flush(self): - record.append(self.on_flush) - super().flush() - f = MyIO() - del f - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - - def test_IOBase_destructor(self): - self._check_base_destructor(self.IOBase) - - def test_RawIOBase_destructor(self): - self._check_base_destructor(self.RawIOBase) - - def test_BufferedIOBase_destructor(self): - self._check_base_destructor(self.BufferedIOBase) - - def test_TextIOBase_destructor(self): - self._check_base_destructor(self.TextIOBase) - - def test_close_flushes(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"xxx") - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"xxx") - - def test_array_writes(self): - a = array.array('i', range(10)) - n = len(a.tobytes()) - def check(f): - with f: - self.assertEqual(f.write(a), n) - f.writelines((a,)) - check(self.BytesIO()) - check(self.FileIO(os_helper.TESTFN, "w")) - check(self.BufferedWriter(self.MockRawIO())) - check(self.BufferedRandom(self.MockRawIO())) - check(self.BufferedRWPair(self.MockRawIO(), self.MockRawIO())) - - def test_closefd(self): - self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'w', - encoding="utf-8", closefd=False) - - def test_read_closed(self): - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - f.write("egg\n") - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) - self.assertEqual(file.read(), "egg\n") - file.seek(0) - file.close() - self.assertRaises(ValueError, file.read) - with self.open(os_helper.TESTFN, "rb") as f: - file = self.open(f.fileno(), "rb", closefd=False) - self.assertEqual(file.read()[:3], b"egg") - file.close() - self.assertRaises(ValueError, file.readinto, bytearray(1)) - - def test_no_closefd_with_filename(self): - # can't use closefd in combination with a file name - self.assertRaises(ValueError, self.open, os_helper.TESTFN, "r", - encoding="utf-8", closefd=False) - - def test_closefd_attr(self): - with self.open(os_helper.TESTFN, "wb") as f: - f.write(b"egg\n") - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - self.assertEqual(f.buffer.raw.closefd, True) - file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) - self.assertEqual(file.buffer.raw.closefd, False) - - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") - def test_garbage_collection(self): - # FileIO objects are collected, and collecting them flushes - # all data to disk. - with warnings_helper.check_warnings(('', ResourceWarning)): - f = self.FileIO(os_helper.TESTFN, "wb") - f.write(b"abcxxx") - f.f = f - wr = weakref.ref(f) - del f - support.gc_collect() - self.assertIsNone(wr(), wr) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"abcxxx") - - def test_unbounded_file(self): - # Issue #1174606: reading from an unbounded stream such as /dev/zero. - zero = "/dev/zero" - if not os.path.exists(zero): - self.skipTest("{0} does not exist".format(zero)) - if sys.maxsize > 0x7FFFFFFF: - self.skipTest("test can only run in a 32-bit address space") - if support.real_max_memuse < support._2G: - self.skipTest("test requires at least 2 GiB of memory") - with self.open(zero, "rb", buffering=0) as f: - self.assertRaises(OverflowError, f.read) - with self.open(zero, "rb") as f: - self.assertRaises(OverflowError, f.read) - with self.open(zero, "r") as f: - self.assertRaises(OverflowError, f.read) - - def check_flush_error_on_close(self, *args, **kwargs): - # Test that the file is closed despite failed flush - # and that flush() is called before file closed. - f = self.open(*args, **kwargs) - closed = [] - def bad_flush(): - closed[:] = [f.closed] - raise OSError() - f.flush = bad_flush - self.assertRaises(OSError, f.close) # exception not swallowed - self.assertTrue(f.closed) - self.assertTrue(closed) # flush() called - self.assertFalse(closed[0]) # flush() called before file closed - f.flush = lambda: None # break reference loop - - def test_flush_error_on_close(self): - # raw file - # Issue #5700: io.FileIO calls flush() after file closed - self.check_flush_error_on_close(os_helper.TESTFN, 'wb', buffering=0) - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb', buffering=0) - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb', buffering=0, closefd=False) - os.close(fd) - # buffered io - self.check_flush_error_on_close(os_helper.TESTFN, 'wb') - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb') - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'wb', closefd=False) - os.close(fd) - # text io - self.check_flush_error_on_close(os_helper.TESTFN, 'w', encoding="utf-8") - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'w', encoding="utf-8") - fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) - self.check_flush_error_on_close(fd, 'w', encoding="utf-8", closefd=False) - os.close(fd) - - def test_multi_close(self): - f = self.open(os_helper.TESTFN, "wb", buffering=0) - f.close() - f.close() - f.close() - self.assertRaises(ValueError, f.flush) - - def test_RawIOBase_read(self): - # Exercise the default limited RawIOBase.read(n) implementation (which - # calls readinto() internally). - rawio = self.MockRawIOWithoutRead((b"abc", b"d", None, b"efg", None)) - self.assertEqual(rawio.read(2), b"ab") - self.assertEqual(rawio.read(2), b"c") - self.assertEqual(rawio.read(2), b"d") - self.assertEqual(rawio.read(2), None) - self.assertEqual(rawio.read(2), b"ef") - self.assertEqual(rawio.read(2), b"g") - self.assertEqual(rawio.read(2), None) - self.assertEqual(rawio.read(2), b"") - - def test_RawIOBase_read_bounds_checking(self): - # Make sure a `.readinto` call which returns a value outside - # (0, len(buffer)) raises. - class Misbehaved(self.RawIOBase): - def __init__(self, readinto_return) -> None: - self._readinto_return = readinto_return - def readinto(self, b): - return self._readinto_return - - with self.assertRaises(ValueError) as cm: - Misbehaved(2).read(1) - self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1") - for bad_size in (2147483647, sys.maxsize, -1, -1000): - with self.assertRaises(ValueError): - Misbehaved(bad_size).read() - - def test_types_have_dict(self): - test = ( - self.IOBase(), - self.RawIOBase(), - self.TextIOBase(), - self.StringIO(), - self.BytesIO() - ) - for obj in test: - self.assertHasAttr(obj, "__dict__") - - def test_opener(self): - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - f.write("egg\n") - fd = os.open(os_helper.TESTFN, os.O_RDONLY) - def opener(path, flags): - return fd - with self.open("non-existent", "r", encoding="utf-8", opener=opener) as f: - self.assertEqual(f.read(), "egg\n") - - def test_bad_opener_negative_1(self): - # Issue #27066. - def badopener(fname, flags): - return -1 - with self.assertRaises(ValueError) as cm: - self.open('non-existent', 'r', opener=badopener) - self.assertEqual(str(cm.exception), 'opener returned -1') - - def test_bad_opener_other_negative(self): - # Issue #27066. - def badopener(fname, flags): - return -2 - with self.assertRaises(ValueError) as cm: - self.open('non-existent', 'r', opener=badopener) - self.assertEqual(str(cm.exception), 'opener returned -2') - - def test_opener_invalid_fd(self): - # Check that OSError is raised with error code EBADF if the - # opener returns an invalid file descriptor (see gh-82212). - fd = os_helper.make_bad_fd() - with self.assertRaises(OSError) as cm: - self.open('foo', opener=lambda name, flags: fd) - self.assertEqual(cm.exception.errno, errno.EBADF) - - def test_fileio_closefd(self): - # Issue #4841 - with self.open(__file__, 'rb') as f1, \ - self.open(__file__, 'rb') as f2: - fileio = self.FileIO(f1.fileno(), closefd=False) - # .__init__() must not close f1 - fileio.__init__(f2.fileno(), closefd=False) - f1.readline() - # .close() must not close f2 - fileio.close() - f2.readline() - - def test_nonbuffered_textio(self): - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError): - self.open(os_helper.TESTFN, 'w', encoding="utf-8", buffering=0) - - def test_invalid_newline(self): - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError): - self.open(os_helper.TESTFN, 'w', encoding="utf-8", newline='invalid') - - def test_buffered_readinto_mixin(self): - # Test the implementation provided by BufferedIOBase - class Stream(self.BufferedIOBase): - def read(self, size): - return b"12345" - read1 = read - stream = Stream() - for method in ("readinto", "readinto1"): - with self.subTest(method): - buffer = byteslike(5) - self.assertEqual(getattr(stream, method)(buffer), 5) - self.assertEqual(bytes(buffer), b"12345") - - def test_fspath_support(self): - def check_path_succeeds(path): - with self.open(path, "w", encoding="utf-8") as f: - f.write("egg\n") - - with self.open(path, "r", encoding="utf-8") as f: - self.assertEqual(f.read(), "egg\n") - - check_path_succeeds(FakePath(os_helper.TESTFN)) - check_path_succeeds(FakePath(os.fsencode(os_helper.TESTFN))) - - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - bad_path = FakePath(f.fileno()) - with self.assertRaises(TypeError): - self.open(bad_path, 'w', encoding="utf-8") - - bad_path = FakePath(None) - with self.assertRaises(TypeError): - self.open(bad_path, 'w', encoding="utf-8") - - bad_path = FakePath(FloatingPointError) - with self.assertRaises(FloatingPointError): - self.open(bad_path, 'w', encoding="utf-8") - - # ensure that refcounting is correct with some error conditions - with self.assertRaisesRegex(ValueError, 'read/write/append mode'): - self.open(FakePath(os_helper.TESTFN), 'rwxa', encoding="utf-8") - - def test_RawIOBase_readall(self): - # Exercise the default unlimited RawIOBase.read() and readall() - # implementations. - rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) - self.assertEqual(rawio.read(), b"abcdefg") - rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) - self.assertEqual(rawio.readall(), b"abcdefg") - - def test_BufferedIOBase_readinto(self): - # Exercise the default BufferedIOBase.readinto() and readinto1() - # implementations (which call read() or read1() internally). - class Reader(self.BufferedIOBase): - def __init__(self, avail): - self.avail = avail - def read(self, size): - result = self.avail[:size] - self.avail = self.avail[size:] - return result - def read1(self, size): - """Returns no more than 5 bytes at once""" - return self.read(min(size, 5)) - tests = ( - # (test method, total data available, read buffer size, expected - # read size) - ("readinto", 10, 5, 5), - ("readinto", 10, 6, 6), # More than read1() can return - ("readinto", 5, 6, 5), # Buffer larger than total available - ("readinto", 6, 7, 6), - ("readinto", 10, 0, 0), # Empty buffer - ("readinto1", 10, 5, 5), # Result limited to single read1() call - ("readinto1", 10, 6, 5), # Buffer larger than read1() can return - ("readinto1", 5, 6, 5), # Buffer larger than total available - ("readinto1", 6, 7, 5), - ("readinto1", 10, 0, 0), # Empty buffer - ) - UNUSED_BYTE = 0x81 - for test in tests: - with self.subTest(test): - method, avail, request, result = test - reader = Reader(bytes(range(avail))) - buffer = bytearray((UNUSED_BYTE,) * request) - method = getattr(reader, method) - self.assertEqual(method(buffer), result) - self.assertEqual(len(buffer), request) - self.assertSequenceEqual(buffer[:result], range(result)) - unused = (UNUSED_BYTE,) * (request - result) - self.assertSequenceEqual(buffer[result:], unused) - self.assertEqual(len(reader.avail), avail - result) - - def test_close_assert(self): - class R(self.IOBase): - def __setattr__(self, name, value): - pass - def flush(self): - raise OSError() - f = R() - # This would cause an assertion failure. - self.assertRaises(OSError, f.close) - - # Silence destructor error - R.flush = lambda self: None - - @threading_helper.requires_working_threading() - def test_write_readline_races(self): - # gh-134908: Concurrent iteration over a file caused races - thread_count = 2 - write_count = 100 - read_count = 100 - - def writer(file, barrier): - barrier.wait() - for _ in range(write_count): - file.write("x") - - def reader(file, barrier): - barrier.wait() - for _ in range(read_count): - for line in file: - self.assertEqual(line, "") - - with self.open(os_helper.TESTFN, "w+") as f: - barrier = threading.Barrier(thread_count + 1) - reader = threading.Thread(target=reader, args=(f, barrier)) - writers = [threading.Thread(target=writer, args=(f, barrier)) - for _ in range(thread_count)] - with threading_helper.catch_threading_exception() as cm: - with threading_helper.start_threads(writers + [reader]): - pass - self.assertIsNone(cm.exc_type) - - self.assertEqual(os.stat(os_helper.TESTFN).st_size, - write_count * thread_count) - - -class CIOTest(IOTest): - - def test_IOBase_finalize(self): - # Issue #12149: segmentation fault on _PyIOBase_finalize when both a - # class which inherits IOBase and an object of this class are caught - # in a reference cycle and close() is already in the method cache. - class MyIO(self.IOBase): - def close(self): - pass - - # create an instance to populate the method cache - MyIO() - obj = MyIO() - obj.obj = obj - wr = weakref.ref(obj) - del MyIO - del obj - support.gc_collect() - self.assertIsNone(wr(), wr) - -@support.cpython_only -class TestIOCTypes(unittest.TestCase): - def setUp(self): - _io = import_helper.import_module("_io") - self.types = [ - _io.BufferedRWPair, - _io.BufferedRandom, - _io.BufferedReader, - _io.BufferedWriter, - _io.BytesIO, - _io.FileIO, - _io.IncrementalNewlineDecoder, - _io.StringIO, - _io.TextIOWrapper, - _io._BufferedIOBase, - _io._BytesIOBuffer, - _io._IOBase, - _io._RawIOBase, - _io._TextIOBase, - ] - if sys.platform == "win32": - self.types.append(_io._WindowsConsoleIO) - self._io = _io - - def test_immutable_types(self): - for tp in self.types: - with self.subTest(tp=tp): - with self.assertRaisesRegex(TypeError, "immutable"): - tp.foo = "bar" - - def test_class_hierarchy(self): - def check_subs(types, base): - for tp in types: - with self.subTest(tp=tp, base=base): - self.assertIsSubclass(tp, base) - - def recursive_check(d): - for k, v in d.items(): - if isinstance(v, dict): - recursive_check(v) - elif isinstance(v, set): - check_subs(v, k) - else: - self.fail("corrupt test dataset") - - _io = self._io - hierarchy = { - _io._IOBase: { - _io._BufferedIOBase: { - _io.BufferedRWPair, - _io.BufferedRandom, - _io.BufferedReader, - _io.BufferedWriter, - _io.BytesIO, - }, - _io._RawIOBase: { - _io.FileIO, - }, - _io._TextIOBase: { - _io.StringIO, - _io.TextIOWrapper, - }, - }, - } - if sys.platform == "win32": - hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO) - - recursive_check(hierarchy) - - def test_subclassing(self): - _io = self._io - dataset = {k: True for k in self.types} - dataset[_io._BytesIOBuffer] = False - - for tp, is_basetype in dataset.items(): - with self.subTest(tp=tp, is_basetype=is_basetype): - name = f"{tp.__name__}_subclass" - bases = (tp,) - if is_basetype: - _ = type(name, bases, {}) - else: - msg = "not an acceptable base type" - with self.assertRaisesRegex(TypeError, msg): - _ = type(name, bases, {}) - - def test_disallow_instantiation(self): - _io = self._io - support.check_disallow_instantiation(self, _io._BytesIOBuffer) - - def test_stringio_setstate(self): - # gh-127182: Calling __setstate__() with invalid arguments must not crash - obj = self._io.StringIO() - with self.assertRaisesRegex( - TypeError, - 'initial_value must be str or None, not int', - ): - obj.__setstate__((1, '', 0, {})) - - obj.__setstate__((None, '', 0, {})) # should not crash - self.assertEqual(obj.getvalue(), '') - - obj.__setstate__(('', '', 0, {})) - self.assertEqual(obj.getvalue(), '') - -class PyIOTest(IOTest): - pass - - -@support.cpython_only -class APIMismatchTest(unittest.TestCase): - - def test_RawIOBase_io_in_pyio_match(self): - """Test that pyio RawIOBase class has all c RawIOBase methods""" - mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__', '__static_attributes__')) - self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') - - def test_RawIOBase_pyio_in_io_match(self): - """Test that c RawIOBase class has all pyio RawIOBase methods""" - mismatch = support.detect_api_mismatch(io.RawIOBase, pyio.RawIOBase) - self.assertEqual(mismatch, set(), msg='C RawIOBase does not have all Python RawIOBase methods') - - -class CommonBufferedTests: - # Tests common to BufferedReader, BufferedWriter and BufferedRandom - - def test_detach(self): - raw = self.MockRawIO() - buf = self.tp(raw) - self.assertIs(buf.detach(), raw) - self.assertRaises(ValueError, buf.detach) - - repr(buf) # Should still work - - def test_fileno(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - - self.assertEqual(42, bufio.fileno()) - - def test_invalid_args(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - # Invalid whence - self.assertRaises(ValueError, bufio.seek, 0, -1) - self.assertRaises(ValueError, bufio.seek, 0, 9) - - def test_override_destructor(self): - tp = self.tp - record = [] - class MyBufferedIO(tp): - def __del__(self): - record.append(1) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(2) - super().close() - def flush(self): - record.append(3) - super().flush() - rawio = self.MockRawIO() - bufio = MyBufferedIO(rawio) - del bufio - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - - def test_context_manager(self): - # Test usability as a context manager - rawio = self.MockRawIO() - bufio = self.tp(rawio) - def _with(): - with bufio: - pass - _with() - # bufio should now be closed, and using it a second time should raise - # a ValueError. - self.assertRaises(ValueError, _with) - - def test_error_through_destructor(self): - # Test that the exception state is not modified by a destructor, - # even if close() fails. - rawio = self.CloseFailureIO() - with support.catch_unraisable_exception() as cm: - with self.assertRaises(AttributeError): - self.tp(rawio).xyzzy - - self.assertEqual(cm.unraisable.exc_type, OSError) - - def test_repr(self): - raw = self.MockRawIO() - b = self.tp(raw) - clsname = r"(%s\.)?%s" % (self.tp.__module__, self.tp.__qualname__) - self.assertRegex(repr(b), "<%s>" % clsname) - raw.name = "dummy" - self.assertRegex(repr(b), "<%s name='dummy'>" % clsname) - raw.name = b"dummy" - self.assertRegex(repr(b), "<%s name=b'dummy'>" % clsname) - - def test_recursive_repr(self): - # Issue #25455 - raw = self.MockRawIO() - b = self.tp(raw) - with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): - with self.assertRaises(RuntimeError): - repr(b) # Should not crash - - def test_flush_error_on_close(self): - # Test that buffered file is closed despite failed flush - # and that flush() is called before file closed. - raw = self.MockRawIO() - closed = [] - def bad_flush(): - closed[:] = [b.closed, raw.closed] - raise OSError() - raw.flush = bad_flush - b = self.tp(raw) - self.assertRaises(OSError, b.close) # exception not swallowed - self.assertTrue(b.closed) - self.assertTrue(raw.closed) - self.assertTrue(closed) # flush() called - self.assertFalse(closed[0]) # flush() called before file closed - self.assertFalse(closed[1]) - raw.flush = lambda: None # break reference loop - - def test_close_error_on_close(self): - raw = self.MockRawIO() - def bad_flush(): - raise OSError('flush') - def bad_close(): - raise OSError('close') - raw.close = bad_close - b = self.tp(raw) - b.flush = bad_flush - with self.assertRaises(OSError) as err: # exception not swallowed - b.close() - self.assertEqual(err.exception.args, ('close',)) - self.assertIsInstance(err.exception.__context__, OSError) - self.assertEqual(err.exception.__context__.args, ('flush',)) - self.assertFalse(b.closed) - - # Silence destructor error - raw.close = lambda: None - b.flush = lambda: None - - def test_nonnormalized_close_error_on_close(self): - # Issue #21677 - raw = self.MockRawIO() - def bad_flush(): - raise non_existing_flush - def bad_close(): - raise non_existing_close - raw.close = bad_close - b = self.tp(raw) - b.flush = bad_flush - with self.assertRaises(NameError) as err: # exception not swallowed - b.close() - self.assertIn('non_existing_close', str(err.exception)) - self.assertIsInstance(err.exception.__context__, NameError) - self.assertIn('non_existing_flush', str(err.exception.__context__)) - self.assertFalse(b.closed) - - # Silence destructor error - b.flush = lambda: None - raw.close = lambda: None - - def test_multi_close(self): - raw = self.MockRawIO() - b = self.tp(raw) - b.close() - b.close() - b.close() - self.assertRaises(ValueError, b.flush) - - def test_unseekable(self): - bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) - self.assertRaises(self.UnsupportedOperation, bufio.tell) - self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) - - def test_readonly_attributes(self): - raw = self.MockRawIO() - buf = self.tp(raw) - x = self.MockRawIO() - with self.assertRaises(AttributeError): - buf.raw = x - - def test_pickling_subclass(self): - global MyBufferedIO - class MyBufferedIO(self.tp): - def __init__(self, raw, tag): - super().__init__(raw) - self.tag = tag - def __getstate__(self): - return self.tag, self.raw.getvalue() - def __setstate__(slf, state): - tag, value = state - slf.__init__(self.BytesIO(value), tag) - - raw = self.BytesIO(b'data') - buf = MyBufferedIO(raw, tag='ham') - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(protocol=proto): - pickled = pickle.dumps(buf, proto) - newbuf = pickle.loads(pickled) - self.assertEqual(newbuf.raw.getvalue(), b'data') - self.assertEqual(newbuf.tag, 'ham') - del MyBufferedIO - - -class SizeofTest: - - @support.cpython_only - def test_sizeof(self): - bufsize1 = 4096 - bufsize2 = 8192 - rawio = self.MockRawIO() - bufio = self.tp(rawio, buffer_size=bufsize1) - size = sys.getsizeof(bufio) - bufsize1 - rawio = self.MockRawIO() - bufio = self.tp(rawio, buffer_size=bufsize2) - self.assertEqual(sys.getsizeof(bufio), size + bufsize2) - - @support.cpython_only - def test_buffer_freeing(self) : - bufsize = 4096 - rawio = self.MockRawIO() - bufio = self.tp(rawio, buffer_size=bufsize) - size = sys.getsizeof(bufio) - bufsize - bufio.close() - self.assertEqual(sys.getsizeof(bufio), size) - -class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): - read_mode = "rb" - - def test_constructor(self): - rawio = self.MockRawIO([b"abc"]) - bufio = self.tp(rawio) - bufio.__init__(rawio) - bufio.__init__(rawio, buffer_size=1024) - bufio.__init__(rawio, buffer_size=16) - self.assertEqual(b"abc", bufio.read()) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - rawio = self.MockRawIO([b"abc"]) - bufio.__init__(rawio) - self.assertEqual(b"abc", bufio.read()) - - def test_uninitialized(self): - bufio = self.tp.__new__(self.tp) - del bufio - bufio = self.tp.__new__(self.tp) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - bufio.read, 0) - bufio.__init__(self.MockRawIO()) - self.assertEqual(bufio.read(0), b'') - - def test_read(self): - for arg in (None, 7): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertEqual(b"abcdefg", bufio.read(arg)) - # Invalid args - self.assertRaises(ValueError, bufio.read, -2) - - def test_read1(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertEqual(b"a", bufio.read(1)) - self.assertEqual(b"b", bufio.read1(1)) - self.assertEqual(rawio._reads, 1) - self.assertEqual(b"", bufio.read1(0)) - self.assertEqual(b"c", bufio.read1(100)) - self.assertEqual(rawio._reads, 1) - self.assertEqual(b"d", bufio.read1(100)) - self.assertEqual(rawio._reads, 2) - self.assertEqual(b"efg", bufio.read1(100)) - self.assertEqual(rawio._reads, 3) - self.assertEqual(b"", bufio.read1(100)) - self.assertEqual(rawio._reads, 4) - - def test_read1_arbitrary(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertEqual(b"a", bufio.read(1)) - self.assertEqual(b"bc", bufio.read1()) - self.assertEqual(b"d", bufio.read1()) - self.assertEqual(b"efg", bufio.read1(-1)) - self.assertEqual(rawio._reads, 3) - self.assertEqual(b"", bufio.read1()) - self.assertEqual(rawio._reads, 4) - - def test_readinto(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - b = bytearray(2) - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"ab") - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"cd") - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"ef") - self.assertEqual(bufio.readinto(b), 1) - self.assertEqual(b, b"gf") - self.assertEqual(bufio.readinto(b), 0) - self.assertEqual(b, b"gf") - rawio = self.MockRawIO((b"abc", None)) - bufio = self.tp(rawio) - self.assertEqual(bufio.readinto(b), 2) - self.assertEqual(b, b"ab") - self.assertEqual(bufio.readinto(b), 1) - self.assertEqual(b, b"cb") - - def test_readinto1(self): - buffer_size = 10 - rawio = self.MockRawIO((b"abc", b"de", b"fgh", b"jkl")) - bufio = self.tp(rawio, buffer_size=buffer_size) - b = bytearray(2) - self.assertEqual(bufio.peek(3), b'abc') - self.assertEqual(rawio._reads, 1) - self.assertEqual(bufio.readinto1(b), 2) - self.assertEqual(b, b"ab") - self.assertEqual(rawio._reads, 1) - self.assertEqual(bufio.readinto1(b), 1) - self.assertEqual(b[:1], b"c") - self.assertEqual(rawio._reads, 1) - self.assertEqual(bufio.readinto1(b), 2) - self.assertEqual(b, b"de") - self.assertEqual(rawio._reads, 2) - b = bytearray(2*buffer_size) - self.assertEqual(bufio.peek(3), b'fgh') - self.assertEqual(rawio._reads, 3) - self.assertEqual(bufio.readinto1(b), 6) - self.assertEqual(b[:6], b"fghjkl") - self.assertEqual(rawio._reads, 4) - - def test_readinto_array(self): - buffer_size = 60 - data = b"a" * 26 - rawio = self.MockRawIO((data,)) - bufio = self.tp(rawio, buffer_size=buffer_size) - - # Create an array with element size > 1 byte - b = array.array('i', b'x' * 32) - assert len(b) != 16 - - # Read into it. We should get as many *bytes* as we can fit into b - # (which is more than the number of elements) - n = bufio.readinto(b) - self.assertGreater(n, len(b)) - - # Check that old contents of b are preserved - bm = memoryview(b).cast('B') - self.assertLess(n, len(bm)) - self.assertEqual(bm[:n], data[:n]) - self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) - - def test_readinto1_array(self): - buffer_size = 60 - data = b"a" * 26 - rawio = self.MockRawIO((data,)) - bufio = self.tp(rawio, buffer_size=buffer_size) - - # Create an array with element size > 1 byte - b = array.array('i', b'x' * 32) - assert len(b) != 16 - - # Read into it. We should get as many *bytes* as we can fit into b - # (which is more than the number of elements) - n = bufio.readinto1(b) - self.assertGreater(n, len(b)) - - # Check that old contents of b are preserved - bm = memoryview(b).cast('B') - self.assertLess(n, len(bm)) - self.assertEqual(bm[:n], data[:n]) - self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) - - def test_readlines(self): - def bufio(): - rawio = self.MockRawIO((b"abc\n", b"d\n", b"ef")) - return self.tp(rawio) - self.assertEqual(bufio().readlines(), [b"abc\n", b"d\n", b"ef"]) - self.assertEqual(bufio().readlines(5), [b"abc\n", b"d\n"]) - self.assertEqual(bufio().readlines(None), [b"abc\n", b"d\n", b"ef"]) - - def test_buffering(self): - data = b"abcdefghi" - dlen = len(data) - - tests = [ - [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ], - [ 100, [ 3, 3, 3], [ dlen ] ], - [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ], - ] - - for bufsize, buf_read_sizes, raw_read_sizes in tests: - rawio = self.MockFileIO(data) - bufio = self.tp(rawio, buffer_size=bufsize) - pos = 0 - for nbytes in buf_read_sizes: - self.assertEqual(bufio.read(nbytes), data[pos:pos+nbytes]) - pos += nbytes - # this is mildly implementation-dependent - self.assertEqual(rawio.read_history, raw_read_sizes) - - def test_read_non_blocking(self): - # Inject some None's in there to simulate EWOULDBLOCK - rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None)) - bufio = self.tp(rawio) - self.assertEqual(b"abcd", bufio.read(6)) - self.assertEqual(b"e", bufio.read(1)) - self.assertEqual(b"fg", bufio.read()) - self.assertEqual(b"", bufio.peek(1)) - self.assertIsNone(bufio.read()) - self.assertEqual(b"", bufio.read()) - - rawio = self.MockRawIO((b"a", None, None)) - self.assertEqual(b"a", rawio.readall()) - self.assertIsNone(rawio.readall()) - - def test_read_past_eof(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - - self.assertEqual(b"abcdefg", bufio.read(9000)) - - def test_read_all(self): - rawio = self.MockRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - - self.assertEqual(b"abcdefg", bufio.read()) - - @threading_helper.requires_working_threading() - @support.requires_resource('cpu') - def test_threads(self): - try: - # Write out many bytes with exactly the same number of 0's, - # 1's... 255's. This will help us check that concurrent reading - # doesn't duplicate or forget contents. - N = 1000 - l = list(range(256)) * N - random.shuffle(l) - s = bytes(bytearray(l)) - with self.open(os_helper.TESTFN, "wb") as f: - f.write(s) - with self.open(os_helper.TESTFN, self.read_mode, buffering=0) as raw: - bufio = self.tp(raw, 8) - errors = [] - results = [] - def f(): - try: - # Intra-buffer read then buffer-flushing read - for n in cycle([1, 19]): - s = bufio.read(n) - if not s: - break - # list.append() is atomic - results.append(s) - except Exception as e: - errors.append(e) - raise - threads = [threading.Thread(target=f) for x in range(20)] - with threading_helper.start_threads(threads): - time.sleep(0.02) # yield - self.assertFalse(errors, - "the following exceptions were caught: %r" % errors) - s = b''.join(results) - for i in range(256): - c = bytes(bytearray([i])) - self.assertEqual(s.count(c), N) - finally: - os_helper.unlink(os_helper.TESTFN) - - def test_unseekable(self): - bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) - self.assertRaises(self.UnsupportedOperation, bufio.tell) - self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) - bufio.read(1) - self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) - self.assertRaises(self.UnsupportedOperation, bufio.tell) - - def test_misbehaved_io(self): - rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - self.assertRaises(OSError, bufio.seek, 0) - self.assertRaises(OSError, bufio.tell) - - # Silence destructor error - bufio.close = lambda: None - - def test_no_extraneous_read(self): - # Issue #9550; when the raw IO object has satisfied the read request, - # we should not issue any additional reads, otherwise it may block - # (e.g. socket). - bufsize = 16 - for n in (2, bufsize - 1, bufsize, bufsize + 1, bufsize * 2): - rawio = self.MockRawIO([b"x" * n]) - bufio = self.tp(rawio, bufsize) - self.assertEqual(bufio.read(n), b"x" * n) - # Simple case: one raw read is enough to satisfy the request. - self.assertEqual(rawio._extraneous_reads, 0, - "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) - # A more complex case where two raw reads are needed to satisfy - # the request. - rawio = self.MockRawIO([b"x" * (n - 1), b"x"]) - bufio = self.tp(rawio, bufsize) - self.assertEqual(bufio.read(n), b"x" * n) - self.assertEqual(rawio._extraneous_reads, 0, - "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) - - def test_read_on_closed(self): - # Issue #23796 - b = self.BufferedReader(self.BytesIO(b"12")) - b.read(1) - b.close() - with self.subTest('peek'): - self.assertRaises(ValueError, b.peek) - with self.subTest('read1'): - self.assertRaises(ValueError, b.read1, 1) - with self.subTest('read'): - self.assertRaises(ValueError, b.read) - with self.subTest('readinto'): - self.assertRaises(ValueError, b.readinto, bytearray()) - with self.subTest('readinto1'): - self.assertRaises(ValueError, b.readinto1, bytearray()) - with self.subTest('flush'): - self.assertRaises(ValueError, b.flush) - with self.subTest('truncate'): - self.assertRaises(ValueError, b.truncate) - with self.subTest('seek'): - self.assertRaises(ValueError, b.seek, 0) - - def test_truncate_on_read_only(self): - rawio = self.MockFileIO(b"abc") - bufio = self.tp(rawio) - self.assertFalse(bufio.writable()) - self.assertRaises(self.UnsupportedOperation, bufio.truncate) - self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) - - def test_tell_character_device_file(self): - # GH-95782 - # For the (former) bug in BufferedIO to manifest, the wrapped IO obj - # must be able to produce at least 2 bytes. - raw = self.MockCharPseudoDevFileIO(b"12") - buf = self.tp(raw) - self.assertEqual(buf.tell(), 0) - self.assertEqual(buf.read(1), b"1") - self.assertEqual(buf.tell(), 0) - - def test_seek_character_device_file(self): - raw = self.MockCharPseudoDevFileIO(b"12") - buf = self.tp(raw) - self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) - self.assertEqual(buf.seek(1, io.SEEK_SET), 0) - self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) - self.assertEqual(buf.read(1), b"1") - - # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, - # which means that the next seek() could return a negative offset if it - # does not sanity-check: - self.assertEqual(buf.tell(), 0) - self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) - - -class CBufferedReaderTest(BufferedReaderTest, SizeofTest): - tp = io.BufferedReader - - def test_initialization(self): - rawio = self.MockRawIO([b"abc"]) - bufio = self.tp(rawio) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.read) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.read) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - self.assertRaises(ValueError, bufio.read) - - def test_misbehaved_io_read(self): - rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) - bufio = self.tp(rawio) - # _pyio.BufferedReader seems to implement reading different, so that - # checking this is not so easy. - self.assertRaises(OSError, bufio.read, 10) - - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") - def test_garbage_collection(self): - # C BufferedReader objects are collected. - # The Python version has __del__, so it ends into gc.garbage instead - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): - rawio = self.FileIO(os_helper.TESTFN, "w+b") - f = self.tp(rawio) - f.f = f - wr = weakref.ref(f) - del f - support.gc_collect() - self.assertIsNone(wr(), wr) - - def test_args_error(self): - # Issue #17275 - with self.assertRaisesRegex(TypeError, "BufferedReader"): - self.tp(self.BytesIO(), 1024, 1024, 1024) - - def test_bad_readinto_value(self): - rawio = self.tp(self.BytesIO(b"12")) - rawio.readinto = lambda buf: -1 - bufio = self.tp(rawio) - with self.assertRaises(OSError) as cm: - bufio.readline() - self.assertIsNone(cm.exception.__cause__) - - def test_bad_readinto_type(self): - rawio = self.tp(self.BytesIO(b"12")) - rawio.readinto = lambda buf: b'' - bufio = self.tp(rawio) - with self.assertRaises(OSError) as cm: - bufio.readline() - self.assertIsInstance(cm.exception.__cause__, TypeError) - - -class PyBufferedReaderTest(BufferedReaderTest): - tp = pyio.BufferedReader - - -class BufferedWriterTest(unittest.TestCase, CommonBufferedTests): - write_mode = "wb" - - def test_constructor(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - bufio.__init__(rawio) - bufio.__init__(rawio, buffer_size=1024) - bufio.__init__(rawio, buffer_size=16) - self.assertEqual(3, bufio.write(b"abc")) - bufio.flush() - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - bufio.__init__(rawio) - self.assertEqual(3, bufio.write(b"ghi")) - bufio.flush() - self.assertEqual(b"".join(rawio._write_stack), b"abcghi") - - def test_uninitialized(self): - bufio = self.tp.__new__(self.tp) - del bufio - bufio = self.tp.__new__(self.tp) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - bufio.write, b'') - bufio.__init__(self.MockRawIO()) - self.assertEqual(bufio.write(b''), 0) - - def test_detach_flush(self): - raw = self.MockRawIO() - buf = self.tp(raw) - buf.write(b"howdy!") - self.assertFalse(raw._write_stack) - buf.detach() - self.assertEqual(raw._write_stack, [b"howdy!"]) - - def test_write(self): - # Write to the buffered IO but don't overflow the buffer. - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.write(b"abc") - self.assertFalse(writer._write_stack) - buffer = bytearray(b"def") - bufio.write(buffer) - buffer[:] = b"***" # Overwrite our copy of the data - bufio.flush() - self.assertEqual(b"".join(writer._write_stack), b"abcdef") - - def test_write_overflow(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - contents = b"abcdefghijklmnop" - for n in range(0, len(contents), 3): - bufio.write(contents[n:n+3]) - flushed = b"".join(writer._write_stack) - # At least (total - 8) bytes were implicitly flushed, perhaps more - # depending on the implementation. - self.assertStartsWith(flushed, contents[:-8]) - - def check_writes(self, intermediate_func): - # Lots of writes, test the flushed output is as expected. - contents = bytes(range(256)) * 1000 - n = 0 - writer = self.MockRawIO() - bufio = self.tp(writer, 13) - # Generator of write sizes: repeat each N 15 times then proceed to N+1 - def gen_sizes(): - for size in count(1): - for i in range(15): - yield size - sizes = gen_sizes() - while n < len(contents): - size = min(next(sizes), len(contents) - n) - self.assertEqual(bufio.write(contents[n:n+size]), size) - intermediate_func(bufio) - n += size - bufio.flush() - self.assertEqual(contents, b"".join(writer._write_stack)) - - def test_writes(self): - self.check_writes(lambda bufio: None) - - def test_writes_and_flushes(self): - self.check_writes(lambda bufio: bufio.flush()) - - def test_writes_and_seeks(self): - def _seekabs(bufio): - pos = bufio.tell() - bufio.seek(pos + 1, 0) - bufio.seek(pos - 1, 0) - bufio.seek(pos, 0) - self.check_writes(_seekabs) - def _seekrel(bufio): - pos = bufio.seek(0, 1) - bufio.seek(+1, 1) - bufio.seek(-1, 1) - bufio.seek(pos, 0) - self.check_writes(_seekrel) - - def test_writes_and_truncates(self): - self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) - - def test_write_non_blocking(self): - raw = self.MockNonBlockWriterIO() - bufio = self.tp(raw, 8) - - self.assertEqual(bufio.write(b"abcd"), 4) - self.assertEqual(bufio.write(b"efghi"), 5) - # 1 byte will be written, the rest will be buffered - raw.block_on(b"k") - self.assertEqual(bufio.write(b"jklmn"), 5) - - # 8 bytes will be written, 8 will be buffered and the rest will be lost - raw.block_on(b"0") - try: - bufio.write(b"opqrwxyz0123456789") - except self.BlockingIOError as e: - written = e.characters_written - else: - self.fail("BlockingIOError should have been raised") - self.assertEqual(written, 16) - self.assertEqual(raw.pop_written(), - b"abcdefghijklmnopqrwxyz") - - self.assertEqual(bufio.write(b"ABCDEFGHI"), 9) - s = raw.pop_written() - # Previously buffered bytes were flushed - self.assertStartsWith(s, b"01234567A") - - def test_write_and_rewind(self): - raw = self.BytesIO() - bufio = self.tp(raw, 4) - self.assertEqual(bufio.write(b"abcdef"), 6) - self.assertEqual(bufio.tell(), 6) - bufio.seek(0, 0) - self.assertEqual(bufio.write(b"XY"), 2) - bufio.seek(6, 0) - self.assertEqual(raw.getvalue(), b"XYcdef") - self.assertEqual(bufio.write(b"123456"), 6) - bufio.flush() - self.assertEqual(raw.getvalue(), b"XYcdef123456") - - def test_flush(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.write(b"abc") - bufio.flush() - self.assertEqual(b"abc", writer._write_stack[0]) - - def test_writelines(self): - l = [b'ab', b'cd', b'ef'] - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.writelines(l) - bufio.flush() - self.assertEqual(b''.join(writer._write_stack), b'abcdef') - - def test_writelines_userlist(self): - l = UserList([b'ab', b'cd', b'ef']) - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.writelines(l) - bufio.flush() - self.assertEqual(b''.join(writer._write_stack), b'abcdef') - - def test_writelines_error(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - self.assertRaises(TypeError, bufio.writelines, [1, 2, 3]) - self.assertRaises(TypeError, bufio.writelines, None) - self.assertRaises(TypeError, bufio.writelines, 'abc') - - def test_destructor(self): - writer = self.MockRawIO() - bufio = self.tp(writer, 8) - bufio.write(b"abc") - del bufio - support.gc_collect() - self.assertEqual(b"abc", writer._write_stack[0]) - - def test_truncate(self): - # Truncate implicitly flushes the buffer. - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: - bufio = self.tp(raw, 8) - bufio.write(b"abcdef") - self.assertEqual(bufio.truncate(3), 3) - self.assertEqual(bufio.tell(), 6) - with self.open(os_helper.TESTFN, "rb", buffering=0) as f: - self.assertEqual(f.read(), b"abc") - - def test_truncate_after_write(self): - # Ensure that truncate preserves the file position after - # writes longer than the buffer size. - # Issue: https://bugs.python.org/issue32228 - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with self.open(os_helper.TESTFN, "wb") as f: - # Fill with some buffer - f.write(b'\x00' * 10000) - buffer_sizes = [8192, 4096, 200] - for buffer_size in buffer_sizes: - with self.open(os_helper.TESTFN, "r+b", buffering=buffer_size) as f: - f.write(b'\x00' * (buffer_size + 1)) - # After write write_pos and write_end are set to 0 - f.read(1) - # read operation makes sure that pos != raw_pos - f.truncate() - self.assertEqual(f.tell(), buffer_size + 2) - - @threading_helper.requires_working_threading() - @support.requires_resource('cpu') - def test_threads(self): - try: - # Write out many bytes from many threads and test they were - # all flushed. - N = 1000 - contents = bytes(range(256)) * N - sizes = cycle([1, 19]) - n = 0 - queue = deque() - while n < len(contents): - size = next(sizes) - queue.append(contents[n:n+size]) - n += size - del contents - # We use a real file object because it allows us to - # exercise situations where the GIL is released before - # writing the buffer to the raw streams. This is in addition - # to concurrency issues due to switching threads in the middle - # of Python code. - with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: - bufio = self.tp(raw, 8) - errors = [] - def f(): - try: - while True: - try: - s = queue.popleft() - except IndexError: - return - bufio.write(s) - except Exception as e: - errors.append(e) - raise - threads = [threading.Thread(target=f) for x in range(20)] - with threading_helper.start_threads(threads): - time.sleep(0.02) # yield - self.assertFalse(errors, - "the following exceptions were caught: %r" % errors) - bufio.close() - with self.open(os_helper.TESTFN, "rb") as f: - s = f.read() - for i in range(256): - self.assertEqual(s.count(bytes([i])), N) - finally: - os_helper.unlink(os_helper.TESTFN) - - def test_misbehaved_io(self): - rawio = self.MisbehavedRawIO() - bufio = self.tp(rawio, 5) - self.assertRaises(OSError, bufio.seek, 0) - self.assertRaises(OSError, bufio.tell) - self.assertRaises(OSError, bufio.write, b"abcdef") - - # Silence destructor error - bufio.close = lambda: None - - def test_max_buffer_size_removal(self): - with self.assertRaises(TypeError): - self.tp(self.MockRawIO(), 8, 12) - - def test_write_error_on_close(self): - raw = self.MockRawIO() - def bad_write(b): - raise OSError() - raw.write = bad_write - b = self.tp(raw) - b.write(b'spam') - self.assertRaises(OSError, b.close) # exception not swallowed - self.assertTrue(b.closed) - - @threading_helper.requires_working_threading() - def test_slow_close_from_thread(self): - # Issue #31976 - rawio = self.SlowFlushRawIO() - bufio = self.tp(rawio, 8) - t = threading.Thread(target=bufio.close) - t.start() - rawio.in_flush.wait() - self.assertRaises(ValueError, bufio.write, b'spam') - self.assertTrue(bufio.closed) - t.join() - - - -class CBufferedWriterTest(BufferedWriterTest, SizeofTest): - tp = io.BufferedWriter - - def test_initialization(self): - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) - self.assertRaises(ValueError, bufio.write, b"def") - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) - self.assertRaises(ValueError, bufio.write, b"def") - self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) - self.assertRaises(ValueError, bufio.write, b"def") - - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") - def test_garbage_collection(self): - # C BufferedWriter objects are collected, and collecting them flushes - # all data to disk. - # The Python version has __del__, so it ends into gc.garbage instead - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with warnings_helper.check_warnings(('', ResourceWarning)): - rawio = self.FileIO(os_helper.TESTFN, "w+b") - f = self.tp(rawio) - f.write(b"123xxx") - f.x = f - wr = weakref.ref(f) - del f - support.gc_collect() - self.assertIsNone(wr(), wr) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"123xxx") - - def test_args_error(self): - # Issue #17275 - with self.assertRaisesRegex(TypeError, "BufferedWriter"): - self.tp(self.BytesIO(), 1024, 1024, 1024) - - -class PyBufferedWriterTest(BufferedWriterTest): - tp = pyio.BufferedWriter - -class BufferedRWPairTest(unittest.TestCase): - - def test_constructor(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertFalse(pair.closed) - - def test_uninitialized(self): - pair = self.tp.__new__(self.tp) - del pair - pair = self.tp.__new__(self.tp) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - pair.read, 0) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - pair.write, b'') - pair.__init__(self.MockRawIO(), self.MockRawIO()) - self.assertEqual(pair.read(0), b'') - self.assertEqual(pair.write(b''), 0) - - def test_detach(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertRaises(self.UnsupportedOperation, pair.detach) - - def test_constructor_max_buffer_size_removal(self): - with self.assertRaises(TypeError): - self.tp(self.MockRawIO(), self.MockRawIO(), 8, 12) - - def test_constructor_with_not_readable(self): - class NotReadable(MockRawIO): - def readable(self): - return False - - self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO()) - - def test_constructor_with_not_writeable(self): - class NotWriteable(MockRawIO): - def writable(self): - return False - - self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable()) - - def test_read(self): - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - self.assertEqual(pair.read(3), b"abc") - self.assertEqual(pair.read(1), b"d") - self.assertEqual(pair.read(), b"ef") - pair = self.tp(self.BytesIO(b"abc"), self.MockRawIO()) - self.assertEqual(pair.read(None), b"abc") - - def test_readlines(self): - pair = lambda: self.tp(self.BytesIO(b"abc\ndef\nh"), self.MockRawIO()) - self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) - self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) - self.assertEqual(pair().readlines(5), [b"abc\n", b"def\n"]) - - def test_read1(self): - # .read1() is delegated to the underlying reader object, so this test - # can be shallow. - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - self.assertEqual(pair.read1(3), b"abc") - self.assertEqual(pair.read1(), b"def") - - def test_readinto(self): - for method in ("readinto", "readinto1"): - with self.subTest(method): - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - data = byteslike(b'\0' * 5) - self.assertEqual(getattr(pair, method)(data), 5) - self.assertEqual(bytes(data), b"abcde") - - def test_write(self): - w = self.MockRawIO() - pair = self.tp(self.MockRawIO(), w) - - pair.write(b"abc") - pair.flush() - buffer = bytearray(b"def") - pair.write(buffer) - buffer[:] = b"***" # Overwrite our copy of the data - pair.flush() - self.assertEqual(w._write_stack, [b"abc", b"def"]) - - def test_peek(self): - pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - - self.assertStartsWith(pair.peek(3), b"abc") - self.assertEqual(pair.read(3), b"abc") - - def test_readable(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertTrue(pair.readable()) - - def test_writeable(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertTrue(pair.writable()) - - def test_seekable(self): - # BufferedRWPairs are never seekable, even if their readers and writers - # are. - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertFalse(pair.seekable()) - - # .flush() is delegated to the underlying writer object and has been - # tested in the test_write method. - - def test_close_and_closed(self): - pair = self.tp(self.MockRawIO(), self.MockRawIO()) - self.assertFalse(pair.closed) - pair.close() - self.assertTrue(pair.closed) - - def test_reader_close_error_on_close(self): - def reader_close(): - reader_non_existing - reader = self.MockRawIO() - reader.close = reader_close - writer = self.MockRawIO() - pair = self.tp(reader, writer) - with self.assertRaises(NameError) as err: - pair.close() - self.assertIn('reader_non_existing', str(err.exception)) - self.assertTrue(pair.closed) - self.assertFalse(reader.closed) - self.assertTrue(writer.closed) - - # Silence destructor error - reader.close = lambda: None - - def test_writer_close_error_on_close(self): - def writer_close(): - writer_non_existing - reader = self.MockRawIO() - writer = self.MockRawIO() - writer.close = writer_close - pair = self.tp(reader, writer) - with self.assertRaises(NameError) as err: - pair.close() - self.assertIn('writer_non_existing', str(err.exception)) - self.assertFalse(pair.closed) - self.assertTrue(reader.closed) - self.assertFalse(writer.closed) - - # Silence destructor error - writer.close = lambda: None - writer = None - - # Ignore BufferedWriter (of the BufferedRWPair) unraisable exception - with support.catch_unraisable_exception(): - # Ignore BufferedRWPair unraisable exception - with support.catch_unraisable_exception(): - pair = None - support.gc_collect() - support.gc_collect() - - def test_reader_writer_close_error_on_close(self): - def reader_close(): - reader_non_existing - def writer_close(): - writer_non_existing - reader = self.MockRawIO() - reader.close = reader_close - writer = self.MockRawIO() - writer.close = writer_close - pair = self.tp(reader, writer) - with self.assertRaises(NameError) as err: - pair.close() - self.assertIn('reader_non_existing', str(err.exception)) - self.assertIsInstance(err.exception.__context__, NameError) - self.assertIn('writer_non_existing', str(err.exception.__context__)) - self.assertFalse(pair.closed) - self.assertFalse(reader.closed) - self.assertFalse(writer.closed) - - # Silence destructor error - reader.close = lambda: None - writer.close = lambda: None - - def test_isatty(self): - class SelectableIsAtty(MockRawIO): - def __init__(self, isatty): - MockRawIO.__init__(self) - self._isatty = isatty - - def isatty(self): - return self._isatty - - pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(False)) - self.assertFalse(pair.isatty()) - - pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(False)) - self.assertTrue(pair.isatty()) - - pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(True)) - self.assertTrue(pair.isatty()) - - pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(True)) - self.assertTrue(pair.isatty()) - - def test_weakref_clearing(self): - brw = self.tp(self.MockRawIO(), self.MockRawIO()) - ref = weakref.ref(brw) - brw = None - ref = None # Shouldn't segfault. - -class CBufferedRWPairTest(BufferedRWPairTest): - tp = io.BufferedRWPair - -class PyBufferedRWPairTest(BufferedRWPairTest): - tp = pyio.BufferedRWPair - - -class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): - read_mode = "rb+" - write_mode = "wb+" - - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - BufferedWriterTest.test_constructor(self) - - def test_uninitialized(self): - BufferedReaderTest.test_uninitialized(self) - BufferedWriterTest.test_uninitialized(self) - - def test_read_and_write(self): - raw = self.MockRawIO((b"asdf", b"ghjk")) - rw = self.tp(raw, 8) - - self.assertEqual(b"as", rw.read(2)) - rw.write(b"ddd") - rw.write(b"eee") - self.assertFalse(raw._write_stack) # Buffer writes - self.assertEqual(b"ghjk", rw.read()) - self.assertEqual(b"dddeee", raw._write_stack[0]) - - def test_seek_and_tell(self): - raw = self.BytesIO(b"asdfghjkl") - rw = self.tp(raw) - - self.assertEqual(b"as", rw.read(2)) - self.assertEqual(2, rw.tell()) - rw.seek(0, 0) - self.assertEqual(b"asdf", rw.read(4)) - - rw.write(b"123f") - rw.seek(0, 0) - self.assertEqual(b"asdf123fl", rw.read()) - self.assertEqual(9, rw.tell()) - rw.seek(-4, 2) - self.assertEqual(5, rw.tell()) - rw.seek(2, 1) - self.assertEqual(7, rw.tell()) - self.assertEqual(b"fl", rw.read(11)) - rw.flush() - self.assertEqual(b"asdf123fl", raw.getvalue()) - - self.assertRaises(TypeError, rw.seek, 0.0) - - def check_flush_and_read(self, read_func): - raw = self.BytesIO(b"abcdefghi") - bufio = self.tp(raw) - - self.assertEqual(b"ab", read_func(bufio, 2)) - bufio.write(b"12") - self.assertEqual(b"ef", read_func(bufio, 2)) - self.assertEqual(6, bufio.tell()) - bufio.flush() - self.assertEqual(6, bufio.tell()) - self.assertEqual(b"ghi", read_func(bufio)) - raw.seek(0, 0) - raw.write(b"XYZ") - # flush() resets the read buffer - bufio.flush() - bufio.seek(0, 0) - self.assertEqual(b"XYZ", read_func(bufio, 3)) - - def test_flush_and_read(self): - self.check_flush_and_read(lambda bufio, *args: bufio.read(*args)) - - def test_flush_and_readinto(self): - def _readinto(bufio, n=-1): - b = bytearray(n if n >= 0 else 9999) - n = bufio.readinto(b) - return bytes(b[:n]) - self.check_flush_and_read(_readinto) - - def test_flush_and_peek(self): - def _peek(bufio, n=-1): - # This relies on the fact that the buffer can contain the whole - # raw stream, otherwise peek() can return less. - b = bufio.peek(n) - if n != -1: - b = b[:n] - bufio.seek(len(b), 1) - return b - self.check_flush_and_read(_peek) - - def test_flush_and_write(self): - raw = self.BytesIO(b"abcdefghi") - bufio = self.tp(raw) - - bufio.write(b"123") - bufio.flush() - bufio.write(b"45") - bufio.flush() - bufio.seek(0, 0) - self.assertEqual(b"12345fghi", raw.getvalue()) - self.assertEqual(b"12345fghi", bufio.read()) - - def test_threads(self): - BufferedReaderTest.test_threads(self) - BufferedWriterTest.test_threads(self) - - def test_writes_and_peek(self): - def _peek(bufio): - bufio.peek(1) - self.check_writes(_peek) - def _peek(bufio): - pos = bufio.tell() - bufio.seek(-1, 1) - bufio.peek(1) - bufio.seek(pos, 0) - self.check_writes(_peek) - - def test_writes_and_reads(self): - def _read(bufio): - bufio.seek(-1, 1) - bufio.read(1) - self.check_writes(_read) - - def test_writes_and_read1s(self): - def _read1(bufio): - bufio.seek(-1, 1) - bufio.read1(1) - self.check_writes(_read1) - - def test_writes_and_readintos(self): - def _read(bufio): - bufio.seek(-1, 1) - bufio.readinto(bytearray(1)) - self.check_writes(_read) - - def test_write_after_readahead(self): - # Issue #6629: writing after the buffer was filled by readahead should - # first rewind the raw stream. - for overwrite_size in [1, 5]: - raw = self.BytesIO(b"A" * 10) - bufio = self.tp(raw, 4) - # Trigger readahead - self.assertEqual(bufio.read(1), b"A") - self.assertEqual(bufio.tell(), 1) - # Overwriting should rewind the raw stream if it needs so - bufio.write(b"B" * overwrite_size) - self.assertEqual(bufio.tell(), overwrite_size + 1) - # If the write size was smaller than the buffer size, flush() and - # check that rewind happens. - bufio.flush() - self.assertEqual(bufio.tell(), overwrite_size + 1) - s = raw.getvalue() - self.assertEqual(s, - b"A" + b"B" * overwrite_size + b"A" * (9 - overwrite_size)) - - def test_write_rewind_write(self): - # Various combinations of reading / writing / seeking backwards / writing again - def mutate(bufio, pos1, pos2): - assert pos2 >= pos1 - # Fill the buffer - bufio.seek(pos1) - bufio.read(pos2 - pos1) - bufio.write(b'\x02') - # This writes earlier than the previous write, but still inside - # the buffer. - bufio.seek(pos1) - bufio.write(b'\x01') - - b = b"\x80\x81\x82\x83\x84" - for i in range(0, len(b)): - for j in range(i, len(b)): - raw = self.BytesIO(b) - bufio = self.tp(raw, 100) - mutate(bufio, i, j) - bufio.flush() - expected = bytearray(b) - expected[j] = 2 - expected[i] = 1 - self.assertEqual(raw.getvalue(), expected, - "failed result for i=%d, j=%d" % (i, j)) - - def test_truncate_after_read_or_write(self): - raw = self.BytesIO(b"A" * 10) - bufio = self.tp(raw, 100) - self.assertEqual(bufio.read(2), b"AA") # the read buffer gets filled - self.assertEqual(bufio.truncate(), 2) - self.assertEqual(bufio.write(b"BB"), 2) # the write buffer increases - self.assertEqual(bufio.truncate(), 4) - - def test_misbehaved_io(self): - BufferedReaderTest.test_misbehaved_io(self) - BufferedWriterTest.test_misbehaved_io(self) - - def test_interleaved_read_write(self): - # Test for issue #12213 - with self.BytesIO(b'abcdefgh') as raw: - with self.tp(raw, 100) as f: - f.write(b"1") - self.assertEqual(f.read(1), b'b') - f.write(b'2') - self.assertEqual(f.read1(1), b'd') - f.write(b'3') - buf = bytearray(1) - f.readinto(buf) - self.assertEqual(buf, b'f') - f.write(b'4') - self.assertEqual(f.peek(1), b'h') - f.flush() - self.assertEqual(raw.getvalue(), b'1b2d3f4h') - - with self.BytesIO(b'abc') as raw: - with self.tp(raw, 100) as f: - self.assertEqual(f.read(1), b'a') - f.write(b"2") - self.assertEqual(f.read(1), b'c') - f.flush() - self.assertEqual(raw.getvalue(), b'a2c') - - def test_read1_after_write(self): - with self.BytesIO(b'abcdef') as raw: - with self.tp(raw, 3) as f: - f.write(b"1") - self.assertEqual(f.read1(1), b'b') - f.flush() - self.assertEqual(raw.getvalue(), b'1bcdef') - with self.BytesIO(b'abcdef') as raw: - with self.tp(raw, 3) as f: - f.write(b"1") - self.assertEqual(f.read1(), b'bcd') - f.flush() - self.assertEqual(raw.getvalue(), b'1bcdef') - with self.BytesIO(b'abcdef') as raw: - with self.tp(raw, 3) as f: - f.write(b"1") - # XXX: read(100) returns different numbers of bytes - # in Python and C implementations. - self.assertEqual(f.read1(100)[:3], b'bcd') - f.flush() - self.assertEqual(raw.getvalue(), b'1bcdef') - - def test_interleaved_readline_write(self): - with self.BytesIO(b'ab\ncdef\ng\n') as raw: - with self.tp(raw) as f: - f.write(b'1') - self.assertEqual(f.readline(), b'b\n') - f.write(b'2') - self.assertEqual(f.readline(), b'def\n') - f.write(b'3') - self.assertEqual(f.readline(), b'\n') - f.flush() - self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n') - - # You can't construct a BufferedRandom over a non-seekable stream. - test_unseekable = None - - # writable() returns True, so there's no point to test it over - # a writable stream. - test_truncate_on_read_only = None - - -class CBufferedRandomTest(BufferedRandomTest, SizeofTest): - tp = io.BufferedRandom - - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") - def test_garbage_collection(self): - CBufferedReaderTest.test_garbage_collection(self) - CBufferedWriterTest.test_garbage_collection(self) - - def test_args_error(self): - # Issue #17275 - with self.assertRaisesRegex(TypeError, "BufferedRandom"): - self.tp(self.BytesIO(), 1024, 1024, 1024) - - -class PyBufferedRandomTest(BufferedRandomTest): - tp = pyio.BufferedRandom - - -# To fully exercise seek/tell, the StatefulIncrementalDecoder has these -# properties: -# - A single output character can correspond to many bytes of input. -# - The number of input bytes to complete the character can be -# undetermined until the last input byte is received. -# - The number of input bytes can vary depending on previous input. -# - A single input byte can correspond to many characters of output. -# - The number of output characters can be undetermined until the -# last input byte is received. -# - The number of output characters can vary depending on previous input. - -class StatefulIncrementalDecoder(codecs.IncrementalDecoder): - """ - For testing seek/tell behavior with a stateful, buffering decoder. - - Input is a sequence of words. Words may be fixed-length (length set - by input) or variable-length (period-terminated). In variable-length - mode, extra periods are ignored. Possible words are: - - 'i' followed by a number sets the input length, I (maximum 99). - When I is set to 0, words are space-terminated. - - 'o' followed by a number sets the output length, O (maximum 99). - - Any other word is converted into a word followed by a period on - the output. The output word consists of the input word truncated - or padded out with hyphens to make its length equal to O. If O - is 0, the word is output verbatim without truncating or padding. - I and O are initially set to 1. When I changes, any buffered input is - re-scanned according to the new I. EOF also terminates the last word. - """ - - def __init__(self, errors='strict'): - codecs.IncrementalDecoder.__init__(self, errors) - self.reset() - - def __repr__(self): - return '' % id(self) - - def reset(self): - self.i = 1 - self.o = 1 - self.buffer = bytearray() - - def getstate(self): - i, o = self.i ^ 1, self.o ^ 1 # so that flags = 0 after reset() - return bytes(self.buffer), i*100 + o - - def setstate(self, state): - buffer, io = state - self.buffer = bytearray(buffer) - i, o = divmod(io, 100) - self.i, self.o = i ^ 1, o ^ 1 - - def decode(self, input, final=False): - output = '' - for b in input: - if self.i == 0: # variable-length, terminated with period - if b == ord('.'): - if self.buffer: - output += self.process_word() - else: - self.buffer.append(b) - else: # fixed-length, terminate after self.i bytes - self.buffer.append(b) - if len(self.buffer) == self.i: - output += self.process_word() - if final and self.buffer: # EOF terminates the last word - output += self.process_word() - return output - - def process_word(self): - output = '' - if self.buffer[0] == ord('i'): - self.i = min(99, int(self.buffer[1:] or 0)) # set input length - elif self.buffer[0] == ord('o'): - self.o = min(99, int(self.buffer[1:] or 0)) # set output length - else: - output = self.buffer.decode('ascii') - if len(output) < self.o: - output += '-'*self.o # pad out with hyphens - if self.o: - output = output[:self.o] # truncate to output length - output += '.' - self.buffer = bytearray() - return output - - codecEnabled = False - - -# bpo-41919: This method is separated from StatefulIncrementalDecoder to avoid a resource leak -# when registering codecs and cleanup functions. -def lookupTestDecoder(name): - if StatefulIncrementalDecoder.codecEnabled and name == 'test_decoder': - latin1 = codecs.lookup('latin-1') - return codecs.CodecInfo( - name='test_decoder', encode=latin1.encode, decode=None, - incrementalencoder=None, - streamreader=None, streamwriter=None, - incrementaldecoder=StatefulIncrementalDecoder) - - -class StatefulIncrementalDecoderTest(unittest.TestCase): - """ - Make sure the StatefulIncrementalDecoder actually works. - """ - - test_cases = [ - # I=1, O=1 (fixed-length input == fixed-length output) - (b'abcd', False, 'a.b.c.d.'), - # I=0, O=0 (variable-length input, variable-length output) - (b'oiabcd', True, 'abcd.'), - # I=0, O=0 (should ignore extra periods) - (b'oi...abcd...', True, 'abcd.'), - # I=0, O=6 (variable-length input, fixed-length output) - (b'i.o6.x.xyz.toolongtofit.', False, 'x-----.xyz---.toolon.'), - # I=2, O=6 (fixed-length input < fixed-length output) - (b'i.i2.o6xyz', True, 'xy----.z-----.'), - # I=6, O=3 (fixed-length input > fixed-length output) - (b'i.o3.i6.abcdefghijklmnop', True, 'abc.ghi.mno.'), - # I=0, then 3; O=29, then 15 (with longer output) - (b'i.o29.a.b.cde.o15.abcdefghijabcdefghij.i3.a.b.c.d.ei00k.l.m', True, - 'a----------------------------.' + - 'b----------------------------.' + - 'cde--------------------------.' + - 'abcdefghijabcde.' + - 'a.b------------.' + - '.c.------------.' + - 'd.e------------.' + - 'k--------------.' + - 'l--------------.' + - 'm--------------.') - ] - - def test_decoder(self): - # Try a few one-shot test cases. - for input, eof, output in self.test_cases: - d = StatefulIncrementalDecoder() - self.assertEqual(d.decode(input, eof), output) - - # Also test an unfinished decode, followed by forcing EOF. - d = StatefulIncrementalDecoder() - self.assertEqual(d.decode(b'oiabcd'), '') - self.assertEqual(d.decode(b'', 1), 'abcd.') - -class TextIOWrapperTest(unittest.TestCase): - - def setUp(self): - self.testdata = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n" - self.normalized = b"AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii") - os_helper.unlink(os_helper.TESTFN) - codecs.register(lookupTestDecoder) - self.addCleanup(codecs.unregister, lookupTestDecoder) - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def test_constructor(self): - r = self.BytesIO(b"\xc3\xa9\n\n") - b = self.BufferedReader(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8") - t.__init__(b, encoding="latin-1", newline="\r\n") - self.assertEqual(t.encoding, "latin-1") - self.assertEqual(t.line_buffering, False) - t.__init__(b, encoding="utf-8", line_buffering=True) - self.assertEqual(t.encoding, "utf-8") - self.assertEqual(t.line_buffering, True) - self.assertEqual("\xe9\n", t.readline()) - invalid_type = TypeError if self.is_C else ValueError - with self.assertRaises(invalid_type): - t.__init__(b, encoding=42) - with self.assertRaises(UnicodeEncodeError): - t.__init__(b, encoding='\udcfe') - with self.assertRaises(ValueError): - t.__init__(b, encoding='utf-8\0') - with self.assertRaises(invalid_type): - t.__init__(b, encoding="utf-8", errors=42) - if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: - with self.assertRaises(UnicodeEncodeError): - t.__init__(b, encoding="utf-8", errors='\udcfe') - if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", errors='replace\0') - with self.assertRaises(TypeError): - t.__init__(b, encoding="utf-8", newline=42) - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", newline='\udcfe') - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", newline='\n\0') - with self.assertRaises(ValueError): - t.__init__(b, encoding="utf-8", newline='xyzzy') - - def test_uninitialized(self): - t = self.TextIOWrapper.__new__(self.TextIOWrapper) - del t - t = self.TextIOWrapper.__new__(self.TextIOWrapper) - self.assertRaises(Exception, repr, t) - self.assertRaisesRegex((ValueError, AttributeError), - 'uninitialized|has no attribute', - t.read, 0) - t.__init__(self.MockRawIO(), encoding="utf-8") - self.assertEqual(t.read(0), '') - - def test_non_text_encoding_codecs_are_rejected(self): - # Ensure the constructor complains if passed a codec that isn't - # marked as a text encoding - # http://bugs.python.org/issue20404 - r = self.BytesIO() - b = self.BufferedWriter(r) - with self.assertRaisesRegex(LookupError, "is not a text encoding"): - self.TextIOWrapper(b, encoding="hex") - - def test_detach(self): - r = self.BytesIO() - b = self.BufferedWriter(r) - t = self.TextIOWrapper(b, encoding="ascii") - self.assertIs(t.detach(), b) - - t = self.TextIOWrapper(b, encoding="ascii") - t.write("howdy") - self.assertFalse(r.getvalue()) - t.detach() - self.assertEqual(r.getvalue(), b"howdy") - self.assertRaises(ValueError, t.detach) - - # Operations independent of the detached stream should still work - repr(t) - self.assertEqual(t.encoding, "ascii") - self.assertEqual(t.errors, "strict") - self.assertFalse(t.line_buffering) - self.assertFalse(t.write_through) - - def test_repr(self): - raw = self.BytesIO("hello".encode("utf-8")) - b = self.BufferedReader(raw) - t = self.TextIOWrapper(b, encoding="utf-8") - modname = self.TextIOWrapper.__module__ - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper encoding='utf-8'>" % modname) - raw.name = "dummy" - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper name='dummy' encoding='utf-8'>" % modname) - t.mode = "r" - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper name='dummy' mode='r' encoding='utf-8'>" % modname) - raw.name = b"dummy" - self.assertRegex(repr(t), - r"<(%s\.)?TextIOWrapper name=b'dummy' mode='r' encoding='utf-8'>" % modname) - - t.buffer.detach() - repr(t) # Should not raise an exception - - def test_recursive_repr(self): - # Issue #25455 - raw = self.BytesIO() - t = self.TextIOWrapper(raw, encoding="utf-8") - with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): - with self.assertRaises(RuntimeError): - repr(t) # Should not crash - - def test_subclass_repr(self): - class TestSubclass(self.TextIOWrapper): - pass - - f = TestSubclass(self.StringIO()) - self.assertIn(TestSubclass.__name__, repr(f)) - - def test_line_buffering(self): - r = self.BytesIO() - b = self.BufferedWriter(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=True) - t.write("X") - self.assertEqual(r.getvalue(), b"") # No flush happened - t.write("Y\nZ") - self.assertEqual(r.getvalue(), b"XY\nZ") # All got flushed - t.write("A\rB") - self.assertEqual(r.getvalue(), b"XY\nZA\rB") - - def test_reconfigure_line_buffering(self): - r = self.BytesIO() - b = self.BufferedWriter(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=False) - t.write("AB\nC") - self.assertEqual(r.getvalue(), b"") - - t.reconfigure(line_buffering=True) # implicit flush - self.assertEqual(r.getvalue(), b"AB\nC") - t.write("DEF\nG") - self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") - t.write("H") - self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") - t.reconfigure(line_buffering=False) # implicit flush - self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") - t.write("IJ") - self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") - - # Keeping default value - t.reconfigure() - t.reconfigure(line_buffering=None) - self.assertEqual(t.line_buffering, False) - t.reconfigure(line_buffering=True) - t.reconfigure() - t.reconfigure(line_buffering=None) - self.assertEqual(t.line_buffering, True) - - @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") - def test_default_encoding(self): - with os_helper.EnvironmentVarGuard() as env: - # try to get a user preferred encoding different than the current - # locale encoding to check that TextIOWrapper() uses the current - # locale encoding and not the user preferred encoding - env.unset('LC_ALL', 'LANG', 'LC_CTYPE') - - current_locale_encoding = locale.getencoding() - b = self.BytesIO() - with warnings.catch_warnings(): - warnings.simplefilter("ignore", EncodingWarning) - t = self.TextIOWrapper(b) - self.assertEqual(t.encoding, current_locale_encoding) - - def test_encoding(self): - # Check the encoding attribute is always set, and valid - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="utf-8") - self.assertEqual(t.encoding, "utf-8") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", EncodingWarning) - t = self.TextIOWrapper(b) - self.assertIsNotNone(t.encoding) - codecs.lookup(t.encoding) - - def test_encoding_errors_reading(self): - # (1) default - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii") - self.assertRaises(UnicodeError, t.read) - # (2) explicit strict - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii", errors="strict") - self.assertRaises(UnicodeError, t.read) - # (3) ignore - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii", errors="ignore") - self.assertEqual(t.read(), "abc\n\n") - # (4) replace - b = self.BytesIO(b"abc\n\xff\n") - t = self.TextIOWrapper(b, encoding="ascii", errors="replace") - self.assertEqual(t.read(), "abc\n\ufffd\n") - - def test_encoding_errors_writing(self): - # (1) default - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii") - self.assertRaises(UnicodeError, t.write, "\xff") - # (2) explicit strict - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii", errors="strict") - self.assertRaises(UnicodeError, t.write, "\xff") - # (3) ignore - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii", errors="ignore", - newline="\n") - t.write("abc\xffdef\n") - t.flush() - self.assertEqual(b.getvalue(), b"abcdef\n") - # (4) replace - b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="ascii", errors="replace", - newline="\n") - t.write("abc\xffdef\n") - t.flush() - self.assertEqual(b.getvalue(), b"abc?def\n") - - def test_newlines(self): - input_lines = [ "unix\n", "windows\r\n", "os9\r", "last\n", "nonl" ] - - tests = [ - [ None, [ 'unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl' ] ], - [ '', input_lines ], - [ '\n', [ "unix\n", "windows\r\n", "os9\rlast\n", "nonl" ] ], - [ '\r\n', [ "unix\nwindows\r\n", "os9\rlast\nnonl" ] ], - [ '\r', [ "unix\nwindows\r", "\nos9\r", "last\nnonl" ] ], - ] - encodings = ( - 'utf-8', 'latin-1', - 'utf-16', 'utf-16-le', 'utf-16-be', - 'utf-32', 'utf-32-le', 'utf-32-be', - ) - - # Try a range of buffer sizes to test the case where \r is the last - # character in TextIOWrapper._pending_line. - for encoding in encodings: - # XXX: str.encode() should return bytes - data = bytes(''.join(input_lines).encode(encoding)) - for do_reads in (False, True): - for bufsize in range(1, 10): - for newline, exp_lines in tests: - bufio = self.BufferedReader(self.BytesIO(data), bufsize) - textio = self.TextIOWrapper(bufio, newline=newline, - encoding=encoding) - if do_reads: - got_lines = [] - while True: - c2 = textio.read(2) - if c2 == '': - break - self.assertEqual(len(c2), 2) - got_lines.append(c2 + textio.readline()) - else: - got_lines = list(textio) - - for got_line, exp_line in zip(got_lines, exp_lines): - self.assertEqual(got_line, exp_line) - self.assertEqual(len(got_lines), len(exp_lines)) - - def test_newlines_input(self): - testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" - normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - for newline, expected in [ - (None, normalized.decode("ascii").splitlines(keepends=True)), - ("", testdata.decode("ascii").splitlines(keepends=True)), - ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), - ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), - ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), - ]: - buf = self.BytesIO(testdata) - txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) - self.assertEqual(txt.readlines(), expected) - txt.seek(0) - self.assertEqual(txt.read(), "".join(expected)) - - def test_newlines_output(self): - testdict = { - "": b"AAA\nBBB\nCCC\nX\rY\r\nZ", - "\n": b"AAA\nBBB\nCCC\nX\rY\r\nZ", - "\r": b"AAA\rBBB\rCCC\rX\rY\r\rZ", - "\r\n": b"AAA\r\nBBB\r\nCCC\r\nX\rY\r\r\nZ", - } - tests = [(None, testdict[os.linesep])] + sorted(testdict.items()) - for newline, expected in tests: - buf = self.BytesIO() - txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) - txt.write("AAA\nB") - txt.write("BB\nCCC\n") - txt.write("X\rY\r\nZ") - txt.flush() - self.assertEqual(buf.closed, False) - self.assertEqual(buf.getvalue(), expected) - - def test_destructor(self): - l = [] - base = self.BytesIO - class MyBytesIO(base): - def close(self): - l.append(self.getvalue()) - base.close(self) - b = MyBytesIO() - t = self.TextIOWrapper(b, encoding="ascii") - t.write("abc") - del t - support.gc_collect() - self.assertEqual([b"abc"], l) - - def test_override_destructor(self): - record = [] - class MyTextIO(self.TextIOWrapper): - def __del__(self): - record.append(1) - try: - f = super().__del__ - except AttributeError: - pass - else: - f() - def close(self): - record.append(2) - super().close() - def flush(self): - record.append(3) - super().flush() - b = self.BytesIO() - t = MyTextIO(b, encoding="ascii") - del t - support.gc_collect() - self.assertEqual(record, [1, 2, 3]) - - def test_error_through_destructor(self): - # Test that the exception state is not modified by a destructor, - # even if close() fails. - rawio = self.CloseFailureIO() - with support.catch_unraisable_exception() as cm: - with self.assertRaises(AttributeError): - self.TextIOWrapper(rawio, encoding="utf-8").xyzzy - - self.assertEqual(cm.unraisable.exc_type, OSError) - - # Systematic tests of the text I/O API - - def test_basic_io(self): - for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65): - for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le": - f = self.open(os_helper.TESTFN, "w+", encoding=enc) - f._CHUNK_SIZE = chunksize - self.assertEqual(f.write("abc"), 3) - f.close() - f = self.open(os_helper.TESTFN, "r+", encoding=enc) - f._CHUNK_SIZE = chunksize - self.assertEqual(f.tell(), 0) - self.assertEqual(f.read(), "abc") - cookie = f.tell() - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.read(None), "abc") - f.seek(0) - self.assertEqual(f.read(2), "ab") - self.assertEqual(f.read(1), "c") - self.assertEqual(f.read(1), "") - self.assertEqual(f.read(), "") - self.assertEqual(f.tell(), cookie) - self.assertEqual(f.seek(0), 0) - self.assertEqual(f.seek(0, 2), cookie) - self.assertEqual(f.write("def"), 3) - self.assertEqual(f.seek(cookie), cookie) - self.assertEqual(f.read(), "def") - if enc.startswith("utf"): - self.multi_line_test(f, enc) - f.close() - - def multi_line_test(self, f, enc): - f.seek(0) - f.truncate() - sample = "s\xff\u0fff\uffff" - wlines = [] - for size in (0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 62, 63, 64, 65, 1000): - chars = [] - for i in range(size): - chars.append(sample[i % len(sample)]) - line = "".join(chars) + "\n" - wlines.append((f.tell(), line)) - f.write(line) - f.seek(0) - rlines = [] - while True: - pos = f.tell() - line = f.readline() - if not line: - break - rlines.append((pos, line)) - self.assertEqual(rlines, wlines) - - def test_telling(self): - f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") - p0 = f.tell() - f.write("\xff\n") - p1 = f.tell() - f.write("\xff\n") - p2 = f.tell() - f.seek(0) - self.assertEqual(f.tell(), p0) - self.assertEqual(f.readline(), "\xff\n") - self.assertEqual(f.tell(), p1) - self.assertEqual(f.readline(), "\xff\n") - self.assertEqual(f.tell(), p2) - f.seek(0) - for line in f: - self.assertEqual(line, "\xff\n") - self.assertRaises(OSError, f.tell) - self.assertEqual(f.tell(), p2) - f.close() - - def test_seeking(self): - chunk_size = _default_chunk_size() - prefix_size = chunk_size - 2 - u_prefix = "a" * prefix_size - prefix = bytes(u_prefix.encode("utf-8")) - self.assertEqual(len(u_prefix), len(prefix)) - u_suffix = "\u8888\n" - suffix = bytes(u_suffix.encode("utf-8")) - line = prefix + suffix - with self.open(os_helper.TESTFN, "wb") as f: - f.write(line*2) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - s = f.read(prefix_size) - self.assertEqual(s, str(prefix, "ascii")) - self.assertEqual(f.tell(), prefix_size) - self.assertEqual(f.readline(), u_suffix) - - def test_seeking_too(self): - # Regression test for a specific bug - data = b'\xe0\xbf\xbf\n' - with self.open(os_helper.TESTFN, "wb") as f: - f.write(data) - with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: - f._CHUNK_SIZE # Just test that it exists - f._CHUNK_SIZE = 2 - f.readline() - f.tell() - - def test_seek_and_tell(self): - #Test seek/tell using the StatefulIncrementalDecoder. - # Make test faster by doing smaller seeks - CHUNK_SIZE = 128 - - def test_seek_and_tell_with_data(data, min_pos=0): - """Tell/seek to various points within a data stream and ensure - that the decoded data returned by read() is consistent.""" - f = self.open(os_helper.TESTFN, 'wb') - f.write(data) - f.close() - f = self.open(os_helper.TESTFN, encoding='test_decoder') - f._CHUNK_SIZE = CHUNK_SIZE - decoded = f.read() - f.close() - - for i in range(min_pos, len(decoded) + 1): # seek positions - for j in [1, 5, len(decoded) - i]: # read lengths - f = self.open(os_helper.TESTFN, encoding='test_decoder') - self.assertEqual(f.read(i), decoded[:i]) - cookie = f.tell() - self.assertEqual(f.read(j), decoded[i:i + j]) - f.seek(cookie) - self.assertEqual(f.read(), decoded[i:]) - f.close() - - # Enable the test decoder. - StatefulIncrementalDecoder.codecEnabled = 1 - - # Run the tests. - try: - # Try each test case. - for input, _, _ in StatefulIncrementalDecoderTest.test_cases: - test_seek_and_tell_with_data(input) - - # Position each test case so that it crosses a chunk boundary. - for input, _, _ in StatefulIncrementalDecoderTest.test_cases: - offset = CHUNK_SIZE - len(input)//2 - prefix = b'.'*offset - # Don't bother seeking into the prefix (takes too long). - min_pos = offset*2 - test_seek_and_tell_with_data(prefix + input, min_pos) - - # Ensure our test decoder won't interfere with subsequent tests. - finally: - StatefulIncrementalDecoder.codecEnabled = 0 - - def test_multibyte_seek_and_tell(self): - f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") - f.write("AB\n\u3046\u3048\n") - f.close() - - f = self.open(os_helper.TESTFN, "r", encoding="euc_jp") - self.assertEqual(f.readline(), "AB\n") - p0 = f.tell() - self.assertEqual(f.readline(), "\u3046\u3048\n") - p1 = f.tell() - f.seek(p0) - self.assertEqual(f.readline(), "\u3046\u3048\n") - self.assertEqual(f.tell(), p1) - f.close() - - def test_tell_after_readline_with_cr(self): - # Test for gh-141314: TextIOWrapper.tell() assertion failure - # when dealing with standalone carriage returns - data = b'line1\r' - with self.open(os_helper.TESTFN, "wb") as f: - f.write(data) - - with self.open(os_helper.TESTFN, "r") as f: - # Read line that ends with \r - line = f.readline() - self.assertEqual(line, "line1\n") - # This should not cause an assertion failure - pos = f.tell() - # Verify we can seek back to this position - f.seek(pos) - remaining = f.read() - self.assertEqual(remaining, "") - - def test_seek_with_encoder_state(self): - f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") - f.write("\u00e6\u0300") - p0 = f.tell() - f.write("\u00e6") - f.seek(p0) - f.write("\u0300") - f.close() - - f = self.open(os_helper.TESTFN, "r", encoding="euc_jis_2004") - self.assertEqual(f.readline(), "\u00e6\u0300\u0300") - f.close() - - def test_encoded_writes(self): - data = "1234567890" - tests = ("utf-16", - "utf-16-le", - "utf-16-be", - "utf-32", - "utf-32-le", - "utf-32-be") - for encoding in tests: - buf = self.BytesIO() - f = self.TextIOWrapper(buf, encoding=encoding) - # Check if the BOM is written only once (see issue1753). - f.write(data) - f.write(data) - f.seek(0) - self.assertEqual(f.read(), data * 2) - f.seek(0) - self.assertEqual(f.read(), data * 2) - self.assertEqual(buf.getvalue(), (data * 2).encode(encoding)) - - def test_unreadable(self): - class UnReadable(self.BytesIO): - def readable(self): - return False - txt = self.TextIOWrapper(UnReadable(), encoding="utf-8") - self.assertRaises(OSError, txt.read) - - def test_read_one_by_one(self): - txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB"), encoding="utf-8") - reads = "" - while True: - c = txt.read(1) - if not c: - break - reads += c - self.assertEqual(reads, "AA\nBB") - - def test_readlines(self): - txt = self.TextIOWrapper(self.BytesIO(b"AA\nBB\nCC"), encoding="utf-8") - self.assertEqual(txt.readlines(), ["AA\n", "BB\n", "CC"]) - txt.seek(0) - self.assertEqual(txt.readlines(None), ["AA\n", "BB\n", "CC"]) - txt.seek(0) - self.assertEqual(txt.readlines(5), ["AA\n", "BB\n"]) - - # read in amounts equal to TextIOWrapper._CHUNK_SIZE which is 128. - def test_read_by_chunk(self): - # make sure "\r\n" straddles 128 char boundary. - txt = self.TextIOWrapper(self.BytesIO(b"A" * 127 + b"\r\nB"), encoding="utf-8") - reads = "" - while True: - c = txt.read(128) - if not c: - break - reads += c - self.assertEqual(reads, "A"*127+"\nB") - - def test_writelines(self): - l = ['ab', 'cd', 'ef'] - buf = self.BytesIO() - txt = self.TextIOWrapper(buf, encoding="utf-8") - txt.writelines(l) - txt.flush() - self.assertEqual(buf.getvalue(), b'abcdef') - - def test_writelines_userlist(self): - l = UserList(['ab', 'cd', 'ef']) - buf = self.BytesIO() - txt = self.TextIOWrapper(buf, encoding="utf-8") - txt.writelines(l) - txt.flush() - self.assertEqual(buf.getvalue(), b'abcdef') - - def test_writelines_error(self): - txt = self.TextIOWrapper(self.BytesIO(), encoding="utf-8") - self.assertRaises(TypeError, txt.writelines, [1, 2, 3]) - self.assertRaises(TypeError, txt.writelines, None) - self.assertRaises(TypeError, txt.writelines, b'abc') - - def test_issue1395_1(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - - # read one char at a time - reads = "" - while True: - c = txt.read(1) - if not c: - break - reads += c - self.assertEqual(reads, self.normalized) - - def test_issue1395_2(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = "" - while True: - c = txt.read(4) - if not c: - break - reads += c - self.assertEqual(reads, self.normalized) - - def test_issue1395_3(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = txt.read(4) - reads += txt.read(4) - reads += txt.readline() - reads += txt.readline() - reads += txt.readline() - self.assertEqual(reads, self.normalized) - - def test_issue1395_4(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = txt.read(4) - reads += txt.read() - self.assertEqual(reads, self.normalized) - - def test_issue1395_5(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt._CHUNK_SIZE = 4 - - reads = txt.read(4) - pos = txt.tell() - txt.seek(0) - txt.seek(pos) - self.assertEqual(txt.read(4), "BBB\n") - - def test_issue2282(self): - buffer = self.BytesIO(self.testdata) - txt = self.TextIOWrapper(buffer, encoding="ascii") - - self.assertEqual(buffer.seekable(), txt.seekable()) - - def test_append_bom(self): - # The BOM is not written again when appending to a non-empty file - filename = os_helper.TESTFN - for charset in ('utf-8-sig', 'utf-16', 'utf-32'): - with self.open(filename, 'w', encoding=charset) as f: - f.write('aaa') - pos = f.tell() - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'aaa'.encode(charset)) - - with self.open(filename, 'a', encoding=charset) as f: - f.write('xxx') - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - - def test_seek_bom(self): - # Same test, but when seeking manually - filename = os_helper.TESTFN - for charset in ('utf-8-sig', 'utf-16', 'utf-32'): - with self.open(filename, 'w', encoding=charset) as f: - f.write('aaa') - pos = f.tell() - with self.open(filename, 'r+', encoding=charset) as f: - f.seek(pos) - f.write('zzz') - f.seek(0) - f.write('bbb') - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) - - def test_seek_append_bom(self): - # Same test, but first seek to the start and then to the end - filename = os_helper.TESTFN - for charset in ('utf-8-sig', 'utf-16', 'utf-32'): - with self.open(filename, 'w', encoding=charset) as f: - f.write('aaa') - with self.open(filename, 'a', encoding=charset) as f: - f.seek(0) - f.seek(0, self.SEEK_END) - f.write('xxx') - with self.open(filename, 'rb') as f: - self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - - def test_errors_property(self): - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - self.assertEqual(f.errors, "strict") - with self.open(os_helper.TESTFN, "w", encoding="utf-8", errors="replace") as f: - self.assertEqual(f.errors, "replace") - - @support.no_tracing - @threading_helper.requires_working_threading() - def test_threads_write(self): - # Issue6750: concurrent writes could duplicate data - event = threading.Event() - with self.open(os_helper.TESTFN, "w", encoding="utf-8", buffering=1) as f: - def run(n): - text = "Thread%03d\n" % n - event.wait() - f.write(text) - threads = [threading.Thread(target=run, args=(x,)) - for x in range(20)] - with threading_helper.start_threads(threads, event.set): - time.sleep(0.02) - with self.open(os_helper.TESTFN, encoding="utf-8") as f: - content = f.read() - for n in range(20): - self.assertEqual(content.count("Thread%03d\n" % n), 1) - - def test_flush_error_on_close(self): - # Test that text file is closed despite failed flush - # and that flush() is called before file closed. - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - closed = [] - def bad_flush(): - closed[:] = [txt.closed, txt.buffer.closed] - raise OSError() - txt.flush = bad_flush - self.assertRaises(OSError, txt.close) # exception not swallowed - self.assertTrue(txt.closed) - self.assertTrue(txt.buffer.closed) - self.assertTrue(closed) # flush() called - self.assertFalse(closed[0]) # flush() called before file closed - self.assertFalse(closed[1]) - txt.flush = lambda: None # break reference loop - - def test_close_error_on_close(self): - buffer = self.BytesIO(self.testdata) - def bad_flush(): - raise OSError('flush') - def bad_close(): - raise OSError('close') - buffer.close = bad_close - txt = self.TextIOWrapper(buffer, encoding="ascii") - txt.flush = bad_flush - with self.assertRaises(OSError) as err: # exception not swallowed - txt.close() - self.assertEqual(err.exception.args, ('close',)) - self.assertIsInstance(err.exception.__context__, OSError) - self.assertEqual(err.exception.__context__.args, ('flush',)) - self.assertFalse(txt.closed) - - # Silence destructor error - buffer.close = lambda: None - txt.flush = lambda: None - - def test_nonnormalized_close_error_on_close(self): - # Issue #21677 - buffer = self.BytesIO(self.testdata) - def bad_flush(): - raise non_existing_flush - def bad_close(): - raise non_existing_close - buffer.close = bad_close - txt = self.TextIOWrapper(buffer, encoding="ascii") - txt.flush = bad_flush - with self.assertRaises(NameError) as err: # exception not swallowed - txt.close() - self.assertIn('non_existing_close', str(err.exception)) - self.assertIsInstance(err.exception.__context__, NameError) - self.assertIn('non_existing_flush', str(err.exception.__context__)) - self.assertFalse(txt.closed) - - # Silence destructor error - buffer.close = lambda: None - txt.flush = lambda: None - - def test_multi_close(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - txt.close() - txt.close() - txt.close() - self.assertRaises(ValueError, txt.flush) - - def test_unseekable(self): - txt = self.TextIOWrapper(self.MockUnseekableIO(self.testdata), encoding="utf-8") - self.assertRaises(self.UnsupportedOperation, txt.tell) - self.assertRaises(self.UnsupportedOperation, txt.seek, 0) - - def test_readonly_attributes(self): - txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") - buf = self.BytesIO(self.testdata) - with self.assertRaises(AttributeError): - txt.buffer = buf - - def test_rawio(self): - # Issue #12591: TextIOWrapper must work with raw I/O objects, so - # that subprocess.Popen() can have the required unbuffered - # semantics with universal_newlines=True. - raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - # Reads - self.assertEqual(txt.read(4), 'abcd') - self.assertEqual(txt.readline(), 'efghi\n') - self.assertEqual(list(txt), ['jkl\n', 'opq\n']) - - def test_rawio_write_through(self): - # Issue #12591: with write_through=True, writes don't need a flush - raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n', - write_through=True) - txt.write('1') - txt.write('23\n4') - txt.write('5') - self.assertEqual(b''.join(raw._write_stack), b'123\n45') - - def test_bufio_write_through(self): - # Issue #21396: write_through=True doesn't force a flush() - # on the underlying binary buffered object. - flush_called, write_called = [], [] - class BufferedWriter(self.BufferedWriter): - def flush(self, *args, **kwargs): - flush_called.append(True) - return super().flush(*args, **kwargs) - def write(self, *args, **kwargs): - write_called.append(True) - return super().write(*args, **kwargs) - - rawio = self.BytesIO() - data = b"a" - bufio = BufferedWriter(rawio, len(data)*2) - textio = self.TextIOWrapper(bufio, encoding='ascii', - write_through=True) - # write to the buffered io but don't overflow the buffer - text = data.decode('ascii') - textio.write(text) - - # buffer.flush is not called with write_through=True - self.assertFalse(flush_called) - # buffer.write *is* called with write_through=True - self.assertTrue(write_called) - self.assertEqual(rawio.getvalue(), b"") # no flush - - write_called = [] # reset - textio.write(text * 10) # total content is larger than bufio buffer - self.assertTrue(write_called) - self.assertEqual(rawio.getvalue(), data * 11) # all flushed - - def test_reconfigure_write_through(self): - raw = self.MockRawIO([]) - t = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - t.write('1') - t.reconfigure(write_through=True) # implied flush - self.assertEqual(t.write_through, True) - self.assertEqual(b''.join(raw._write_stack), b'1') - t.write('23') - self.assertEqual(b''.join(raw._write_stack), b'123') - t.reconfigure(write_through=False) - self.assertEqual(t.write_through, False) - t.write('45') - t.flush() - self.assertEqual(b''.join(raw._write_stack), b'12345') - # Keeping default value - t.reconfigure() - t.reconfigure(write_through=None) - self.assertEqual(t.write_through, False) - t.reconfigure(write_through=True) - t.reconfigure() - t.reconfigure(write_through=None) - self.assertEqual(t.write_through, True) - - def test_read_nonbytes(self): - # Issue #17106 - # Crash when underlying read() returns non-bytes - t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") - self.assertRaises(TypeError, t.read, 1) - t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") - self.assertRaises(TypeError, t.readline) - t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") - self.assertRaises(TypeError, t.read) - - def test_illegal_encoder(self): - # Issue 31271: Calling write() while the return value of encoder's - # encode() is invalid shouldn't cause an assertion failure. - rot13 = codecs.lookup("rot13") - with support.swap_attr(rot13, '_is_text_encoding', True): - t = self.TextIOWrapper(self.BytesIO(b'foo'), encoding="rot13") - self.assertRaises(TypeError, t.write, 'bar') - - def test_illegal_decoder(self): - # Issue #17106 - # Bypass the early encoding check added in issue 20404 - def _make_illegal_wrapper(): - quopri = codecs.lookup("quopri") - quopri._is_text_encoding = True - try: - t = self.TextIOWrapper(self.BytesIO(b'aaaaaa'), - newline='\n', encoding="quopri") - finally: - quopri._is_text_encoding = False - return t - # Crash when decoder returns non-string - t = _make_illegal_wrapper() - self.assertRaises(TypeError, t.read, 1) - t = _make_illegal_wrapper() - self.assertRaises(TypeError, t.readline) - t = _make_illegal_wrapper() - self.assertRaises(TypeError, t.read) - - # Issue 31243: calling read() while the return value of decoder's - # getstate() is invalid should neither crash the interpreter nor - # raise a SystemError. - def _make_very_illegal_wrapper(getstate_ret_val): - class BadDecoder: - def getstate(self): - return getstate_ret_val - def _get_bad_decoder(dummy): - return BadDecoder() - quopri = codecs.lookup("quopri") - with support.swap_attr(quopri, 'incrementaldecoder', - _get_bad_decoder): - return _make_illegal_wrapper() - t = _make_very_illegal_wrapper(42) - self.assertRaises(TypeError, t.read, 42) - t = _make_very_illegal_wrapper(()) - self.assertRaises(TypeError, t.read, 42) - t = _make_very_illegal_wrapper((1, 2)) - self.assertRaises(TypeError, t.read, 42) - - def _check_create_at_shutdown(self, **kwargs): - # Issue #20037: creating a TextIOWrapper at shutdown - # shouldn't crash the interpreter. - iomod = self.io.__name__ - code = """if 1: - import codecs - import {iomod} as io - - # Avoid looking up codecs at shutdown - codecs.lookup('utf-8') - - class C: - def __del__(self): - io.TextIOWrapper(io.BytesIO(), **{kwargs}) - print("ok") - c = C() - """.format(iomod=iomod, kwargs=kwargs) - return assert_python_ok("-c", code) - - def test_create_at_shutdown_without_encoding(self): - rc, out, err = self._check_create_at_shutdown() - if err: - # Can error out with a RuntimeError if the module state - # isn't found. - self.assertIn(self.shutdown_error, err.decode()) - else: - self.assertEqual("ok", out.decode().strip()) - - def test_create_at_shutdown_with_encoding(self): - rc, out, err = self._check_create_at_shutdown(encoding='utf-8', - errors='strict') - self.assertFalse(err) - self.assertEqual("ok", out.decode().strip()) - - def test_read_byteslike(self): - r = MemviewBytesIO(b'Just some random string\n') - t = self.TextIOWrapper(r, 'utf-8') - - # TextIOwrapper will not read the full string, because - # we truncate it to a multiple of the native int size - # so that we can construct a more complex memoryview. - bytes_val = _to_memoryview(r.getvalue()).tobytes() - - self.assertEqual(t.read(200), bytes_val.decode('utf-8')) - - def test_issue22849(self): - class F(object): - def readable(self): return True - def writable(self): return True - def seekable(self): return True - - for i in range(10): - try: - self.TextIOWrapper(F(), encoding='utf-8') - except Exception: - pass - - F.tell = lambda x: 0 - t = self.TextIOWrapper(F(), encoding='utf-8') - - def test_reconfigure_locale(self): - wrapper = self.TextIOWrapper(self.BytesIO(b"test")) - wrapper.reconfigure(encoding="locale") - - def test_reconfigure_encoding_read(self): - # latin1 -> utf8 - # (latin1 can decode utf-8 encoded string) - data = 'abc\xe9\n'.encode('latin1') + 'd\xe9f\n'.encode('utf8') - raw = self.BytesIO(data) - txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') - self.assertEqual(txt.readline(), 'abc\xe9\n') - with self.assertRaises(self.UnsupportedOperation): - txt.reconfigure(encoding='utf-8') - with self.assertRaises(self.UnsupportedOperation): - txt.reconfigure(newline=None) - - def test_reconfigure_write_fromascii(self): - # ascii has a specific encodefunc in the C implementation, - # but utf-8-sig has not. Make sure that we get rid of the - # cached encodefunc when we switch encoders. - raw = self.BytesIO() - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - txt.write('foo\n') - txt.reconfigure(encoding='utf-8-sig') - txt.write('\xe9\n') - txt.flush() - self.assertEqual(raw.getvalue(), b'foo\n\xc3\xa9\n') - - def test_reconfigure_write(self): - # latin -> utf8 - raw = self.BytesIO() - txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') - txt.write('abc\xe9\n') - txt.reconfigure(encoding='utf-8') - self.assertEqual(raw.getvalue(), b'abc\xe9\n') - txt.write('d\xe9f\n') - txt.flush() - self.assertEqual(raw.getvalue(), b'abc\xe9\nd\xc3\xa9f\n') - - # ascii -> utf-8-sig: ensure that no BOM is written in the middle of - # the file - raw = self.BytesIO() - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - txt.write('abc\n') - txt.reconfigure(encoding='utf-8-sig') - txt.write('d\xe9f\n') - txt.flush() - self.assertEqual(raw.getvalue(), b'abc\nd\xc3\xa9f\n') - - def test_reconfigure_write_non_seekable(self): - raw = self.BytesIO() - raw.seekable = lambda: False - raw.seek = None - txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') - txt.write('abc\n') - txt.reconfigure(encoding='utf-8-sig') - txt.write('d\xe9f\n') - txt.flush() - - # If the raw stream is not seekable, there'll be a BOM - self.assertEqual(raw.getvalue(), b'abc\n\xef\xbb\xbfd\xc3\xa9f\n') - - def test_reconfigure_defaults(self): - txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\n') - txt.reconfigure(encoding=None) - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'replace') - txt.write('LF\n') - - txt.reconfigure(newline='\r\n') - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'replace') - - txt.reconfigure(errors='ignore') - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'ignore') - txt.write('CRLF\n') - - txt.reconfigure(encoding='utf-8', newline=None) - self.assertEqual(txt.errors, 'strict') - txt.seek(0) - self.assertEqual(txt.read(), 'LF\nCRLF\n') - - self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n') - - def test_reconfigure_errors(self): - txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r') - with self.assertRaises(TypeError): # there was a crash - txt.reconfigure(encoding=42) - if self.is_C: - with self.assertRaises(UnicodeEncodeError): - txt.reconfigure(encoding='\udcfe') - with self.assertRaises(LookupError): - txt.reconfigure(encoding='locale\0') - # TODO: txt.reconfigure(encoding='utf-8\0') - # TODO: txt.reconfigure(encoding='nonexisting') - with self.assertRaises(TypeError): - txt.reconfigure(errors=42) - if self.is_C: - with self.assertRaises(UnicodeEncodeError): - txt.reconfigure(errors='\udcfe') - # TODO: txt.reconfigure(errors='ignore\0') - # TODO: txt.reconfigure(errors='nonexisting') - with self.assertRaises(TypeError): - txt.reconfigure(newline=42) - with self.assertRaises(ValueError): - txt.reconfigure(newline='\udcfe') - with self.assertRaises(ValueError): - txt.reconfigure(newline='xyz') - if not self.is_C: - # TODO: Should fail in C too. - with self.assertRaises(ValueError): - txt.reconfigure(newline='\n\0') - if self.is_C: - # TODO: Use __bool__(), not __index__(). - with self.assertRaises(ZeroDivisionError): - txt.reconfigure(line_buffering=BadIndex()) - with self.assertRaises(OverflowError): - txt.reconfigure(line_buffering=2**1000) - with self.assertRaises(ZeroDivisionError): - txt.reconfigure(write_through=BadIndex()) - with self.assertRaises(OverflowError): - txt.reconfigure(write_through=2**1000) - with self.assertRaises(ZeroDivisionError): # there was a crash - txt.reconfigure(line_buffering=BadIndex(), - write_through=BadIndex()) - self.assertEqual(txt.encoding, 'ascii') - self.assertEqual(txt.errors, 'replace') - self.assertIs(txt.line_buffering, False) - self.assertIs(txt.write_through, False) - - txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n', - line_buffering=True, write_through=True) - self.assertEqual(txt.encoding, 'latin1') - self.assertEqual(txt.errors, 'ignore') - self.assertIs(txt.line_buffering, True) - self.assertIs(txt.write_through, True) - - def test_reconfigure_newline(self): - raw = self.BytesIO(b'CR\rEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\n') - txt.reconfigure(newline=None) - self.assertEqual(txt.readline(), 'CR\n') - raw = self.BytesIO(b'CR\rEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\n') - txt.reconfigure(newline='') - self.assertEqual(txt.readline(), 'CR\r') - raw = self.BytesIO(b'CR\rLF\nEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\r') - txt.reconfigure(newline='\n') - self.assertEqual(txt.readline(), 'CR\rLF\n') - raw = self.BytesIO(b'LF\nCR\rEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\n') - txt.reconfigure(newline='\r') - self.assertEqual(txt.readline(), 'LF\nCR\r') - raw = self.BytesIO(b'CR\rCRLF\r\nEOF') - txt = self.TextIOWrapper(raw, 'ascii', newline='\r') - txt.reconfigure(newline='\r\n') - self.assertEqual(txt.readline(), 'CR\rCRLF\r\n') - - txt = self.TextIOWrapper(self.BytesIO(), 'ascii', newline='\r') - txt.reconfigure(newline=None) - txt.write('linesep\n') - txt.reconfigure(newline='') - txt.write('LF\n') - txt.reconfigure(newline='\n') - txt.write('LF\n') - txt.reconfigure(newline='\r') - txt.write('CR\n') - txt.reconfigure(newline='\r\n') - txt.write('CRLF\n') - expected = 'linesep' + os.linesep + 'LF\nLF\nCR\rCRLF\r\n' - self.assertEqual(txt.detach().getvalue().decode('ascii'), expected) - - def test_issue25862(self): - # Assertion failures occurred in tell() after read() and write(). - t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') - t.read(1) - t.read() - t.tell() - t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') - t.read(1) - t.write('x') - t.tell() - - def test_issue35928(self): - p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO()) - f = self.TextIOWrapper(p) - res = f.readline() - self.assertEqual(res, 'foo\n') - f.write(res) - self.assertEqual(res + f.readline(), 'foo\nbar\n') - - def test_pickling_subclass(self): - global MyTextIO - class MyTextIO(self.TextIOWrapper): - def __init__(self, raw, tag): - super().__init__(raw) - self.tag = tag - def __getstate__(self): - return self.tag, self.buffer.getvalue() - def __setstate__(slf, state): - tag, value = state - slf.__init__(self.BytesIO(value), tag) - - raw = self.BytesIO(b'data') - txt = MyTextIO(raw, 'ham') - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(protocol=proto): - pickled = pickle.dumps(txt, proto) - newtxt = pickle.loads(pickled) - self.assertEqual(newtxt.buffer.getvalue(), b'data') - self.assertEqual(newtxt.tag, 'ham') - del MyTextIO - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_read_non_blocking(self): - import os - r, w = os.pipe() - try: - os.set_blocking(r, False) - with self.io.open(r, 'rt') as textfile: - r = None - # Nothing has been written so a non-blocking read raises a BlockingIOError exception. - with self.assertRaises(BlockingIOError): - textfile.read() - finally: - if r is not None: - os.close(r) - os.close(w) - - -class MemviewBytesIO(io.BytesIO): - '''A BytesIO object whose read method returns memoryviews - rather than bytes''' - - def read1(self, len_): - return _to_memoryview(super().read1(len_)) - - def read(self, len_): - return _to_memoryview(super().read(len_)) - -def _to_memoryview(buf): - '''Convert bytes-object *buf* to a non-trivial memoryview''' - - arr = array.array('i') - idx = len(buf) - len(buf) % arr.itemsize - arr.frombytes(buf[:idx]) - return memoryview(arr) - - -class CTextIOWrapperTest(TextIOWrapperTest): - io = io - shutdown_error = "LookupError: unknown encoding: ascii" - - def test_initialization(self): - r = self.BytesIO(b"\xc3\xa9\n\n") - b = self.BufferedReader(r, 1000) - t = self.TextIOWrapper(b, encoding="utf-8") - self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') - self.assertRaises(ValueError, t.read) - - t = self.TextIOWrapper.__new__(self.TextIOWrapper) - self.assertRaises(Exception, repr, t) - - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") - def test_garbage_collection(self): - # C TextIOWrapper objects are collected, and collecting them flushes - # all data to disk. - # The Python version has __del__, so it ends in gc.garbage instead. - with warnings_helper.check_warnings(('', ResourceWarning)): - rawio = self.FileIO(os_helper.TESTFN, "wb") - b = self.BufferedWriter(rawio) - t = self.TextIOWrapper(b, encoding="ascii") - t.write("456def") - t.x = t - wr = weakref.ref(t) - del t - support.gc_collect() - self.assertIsNone(wr(), wr) - with self.open(os_helper.TESTFN, "rb") as f: - self.assertEqual(f.read(), b"456def") - - def test_rwpair_cleared_before_textio(self): - # Issue 13070: TextIOWrapper's finalization would crash when called - # after the reference to the underlying BufferedRWPair's writer got - # cleared by the GC. - for i in range(1000): - b1 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) - t1 = self.TextIOWrapper(b1, encoding="ascii") - b2 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) - t2 = self.TextIOWrapper(b2, encoding="ascii") - # circular references - t1.buddy = t2 - t2.buddy = t1 - support.gc_collect() - - def test_del__CHUNK_SIZE_SystemError(self): - t = self.TextIOWrapper(self.BytesIO(), encoding='ascii') - with self.assertRaises(AttributeError): - del t._CHUNK_SIZE - - def test_internal_buffer_size(self): - # bpo-43260: TextIOWrapper's internal buffer should not store - # data larger than chunk size. - chunk_size = 8192 # default chunk size, updated later - - class MockIO(self.MockRawIO): - def write(self, data): - if len(data) > chunk_size: - raise RuntimeError - return super().write(data) - - buf = MockIO() - t = self.TextIOWrapper(buf, encoding="ascii") - chunk_size = t._CHUNK_SIZE - t.write("abc") - t.write("def") - # default chunk size is 8192 bytes so t don't write data to buf. - self.assertEqual([], buf._write_stack) - - with self.assertRaises(RuntimeError): - t.write("x"*(chunk_size+1)) - - self.assertEqual([b"abcdef"], buf._write_stack) - t.write("ghi") - t.write("x"*chunk_size) - self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) - - def test_issue119506(self): - chunk_size = 8192 - - class MockIO(self.MockRawIO): - written = False - def write(self, data): - if not self.written: - self.written = True - t.write("middle") - return super().write(data) - - buf = MockIO() - t = self.TextIOWrapper(buf) - t.write("abc") - t.write("def") - # writing data which size >= chunk_size cause flushing buffer before write. - t.write("g" * chunk_size) - t.flush() - - self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], - buf._write_stack) - - def test_issue142594(self): - wrapper = None - detached = False - class ReentrantRawIO(self.RawIOBase): - @property - def closed(self): - nonlocal detached - if wrapper is not None and not detached: - detached = True - wrapper.detach() - return False - - raw = ReentrantRawIO() - wrapper = self.TextIOWrapper(raw) - wrapper.close() # should not crash - - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'NoneType' - def test_read_non_blocking(self): - return super().test_read_non_blocking() - - -class PyTextIOWrapperTest(TextIOWrapperTest): - io = pyio - shutdown_error = "LookupError: unknown encoding: ascii" - - -class IncrementalNewlineDecoderTest(unittest.TestCase): - - def check_newline_decoding_utf8(self, decoder): - # UTF-8 specific tests for a newline decoder - def _check_decode(b, s, **kwargs): - # We exercise getstate() / setstate() as well as decode() - state = decoder.getstate() - self.assertEqual(decoder.decode(b, **kwargs), s) - decoder.setstate(state) - self.assertEqual(decoder.decode(b, **kwargs), s) - - _check_decode(b'\xe8\xa2\x88', "\u8888") - - _check_decode(b'\xe8', "") - _check_decode(b'\xa2', "") - _check_decode(b'\x88', "\u8888") - - _check_decode(b'\xe8', "") - _check_decode(b'\xa2', "") - _check_decode(b'\x88', "\u8888") - - _check_decode(b'\xe8', "") - self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True) - - decoder.reset() - _check_decode(b'\n', "\n") - _check_decode(b'\r', "") - _check_decode(b'', "\n", final=True) - _check_decode(b'\r', "\n", final=True) - - _check_decode(b'\r', "") - _check_decode(b'a', "\na") - - _check_decode(b'\r\r\n', "\n\n") - _check_decode(b'\r', "") - _check_decode(b'\r', "\n") - _check_decode(b'\na', "\na") - - _check_decode(b'\xe8\xa2\x88\r\n', "\u8888\n") - _check_decode(b'\xe8\xa2\x88', "\u8888") - _check_decode(b'\n', "\n") - _check_decode(b'\xe8\xa2\x88\r', "\u8888") - _check_decode(b'\n', "\n") - - def check_newline_decoding(self, decoder, encoding): - result = [] - if encoding is not None: - encoder = codecs.getincrementalencoder(encoding)() - def _decode_bytewise(s): - # Decode one byte at a time - for b in encoder.encode(s): - result.append(decoder.decode(bytes([b]))) - else: - encoder = None - def _decode_bytewise(s): - # Decode one char at a time - for c in s: - result.append(decoder.decode(c)) - self.assertEqual(decoder.newlines, None) - _decode_bytewise("abc\n\r") - self.assertEqual(decoder.newlines, '\n') - _decode_bytewise("\nabc") - self.assertEqual(decoder.newlines, ('\n', '\r\n')) - _decode_bytewise("abc\r") - self.assertEqual(decoder.newlines, ('\n', '\r\n')) - _decode_bytewise("abc") - self.assertEqual(decoder.newlines, ('\r', '\n', '\r\n')) - _decode_bytewise("abc\r") - self.assertEqual("".join(result), "abc\n\nabcabc\nabcabc") - decoder.reset() - input = "abc" - if encoder is not None: - encoder.reset() - input = encoder.encode(input) - self.assertEqual(decoder.decode(input), "abc") - self.assertEqual(decoder.newlines, None) - - def test_newline_decoder(self): - encodings = ( - # None meaning the IncrementalNewlineDecoder takes unicode input - # rather than bytes input - None, 'utf-8', 'latin-1', - 'utf-16', 'utf-16-le', 'utf-16-be', - 'utf-32', 'utf-32-le', 'utf-32-be', - ) - for enc in encodings: - decoder = enc and codecs.getincrementaldecoder(enc)() - decoder = self.IncrementalNewlineDecoder(decoder, translate=True) - self.check_newline_decoding(decoder, enc) - decoder = codecs.getincrementaldecoder("utf-8")() - decoder = self.IncrementalNewlineDecoder(decoder, translate=True) - self.check_newline_decoding_utf8(decoder) - self.assertRaises(TypeError, decoder.setstate, 42) - - def test_newline_bytes(self): - # Issue 5433: Excessive optimization in IncrementalNewlineDecoder - def _check(dec): - self.assertEqual(dec.newlines, None) - self.assertEqual(dec.decode("\u0D00"), "\u0D00") - self.assertEqual(dec.newlines, None) - self.assertEqual(dec.decode("\u0A00"), "\u0A00") - self.assertEqual(dec.newlines, None) - dec = self.IncrementalNewlineDecoder(None, translate=False) - _check(dec) - dec = self.IncrementalNewlineDecoder(None, translate=True) - _check(dec) - - def test_translate(self): - # issue 35062 - for translate in (-2, -1, 1, 2): - decoder = codecs.getincrementaldecoder("utf-8")() - decoder = self.IncrementalNewlineDecoder(decoder, translate) - self.check_newline_decoding_utf8(decoder) - decoder = codecs.getincrementaldecoder("utf-8")() - decoder = self.IncrementalNewlineDecoder(decoder, translate=0) - self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n") - -class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): - @support.cpython_only - def test_uninitialized(self): - uninitialized = self.IncrementalNewlineDecoder.__new__( - self.IncrementalNewlineDecoder) - self.assertRaises(ValueError, uninitialized.decode, b'bar') - self.assertRaises(ValueError, uninitialized.getstate) - self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0)) - self.assertRaises(ValueError, uninitialized.reset) - - -class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): - pass - - -# XXX Tests for open() - -class MiscIOTest(unittest.TestCase): - - # for test__all__, actual values are set in subclasses - name_of_module = None - extra_exported = () - not_exported = () - - def tearDown(self): - os_helper.unlink(os_helper.TESTFN) - - def test___all__(self): - support.check__all__(self, self.io, self.name_of_module, - extra=self.extra_exported, - not_exported=self.not_exported) - - def test_attributes(self): - f = self.open(os_helper.TESTFN, "wb", buffering=0) - self.assertEqual(f.mode, "wb") - f.close() - - f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") - self.assertEqual(f.mode, "w+") - self.assertEqual(f.buffer.mode, "wb+") - self.assertEqual(f.buffer.raw.mode, "wb+") - - g = self.open(f.fileno(), "wb", closefd=False) - self.assertEqual(g.mode, "wb") - self.assertEqual(g.raw.mode, "wb") - self.assertEqual(g.name, f.fileno()) - self.assertEqual(g.raw.name, f.fileno()) - f.close() - g.close() - - def test_removed_u_mode(self): - # bpo-37330: The "U" mode has been removed in Python 3.11 - for mode in ("U", "rU", "r+U"): - with self.assertRaises(ValueError) as cm: - self.open(os_helper.TESTFN, mode) - self.assertIn('invalid mode', str(cm.exception)) - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_open_pipe_with_append(self): - # bpo-27805: Ignore ESPIPE from lseek() in open(). - r, w = os.pipe() - self.addCleanup(os.close, r) - f = self.open(w, 'a', encoding="utf-8") - self.addCleanup(f.close) - # Check that the file is marked non-seekable. On Windows, however, lseek - # somehow succeeds on pipes. - if sys.platform != 'win32': - self.assertFalse(f.seekable()) - - def test_io_after_close(self): - for kwargs in [ - {"mode": "w"}, - {"mode": "wb"}, - {"mode": "w", "buffering": 1}, - {"mode": "w", "buffering": 2}, - {"mode": "wb", "buffering": 0}, - {"mode": "r"}, - {"mode": "rb"}, - {"mode": "r", "buffering": 1}, - {"mode": "r", "buffering": 2}, - {"mode": "rb", "buffering": 0}, - {"mode": "w+"}, - {"mode": "w+b"}, - {"mode": "w+", "buffering": 1}, - {"mode": "w+", "buffering": 2}, - {"mode": "w+b", "buffering": 0}, - ]: - if "b" not in kwargs["mode"]: - kwargs["encoding"] = "utf-8" - f = self.open(os_helper.TESTFN, **kwargs) - f.close() - self.assertRaises(ValueError, f.flush) - self.assertRaises(ValueError, f.fileno) - self.assertRaises(ValueError, f.isatty) - self.assertRaises(ValueError, f.__iter__) - if hasattr(f, "peek"): - self.assertRaises(ValueError, f.peek, 1) - self.assertRaises(ValueError, f.read) - if hasattr(f, "read1"): - self.assertRaises(ValueError, f.read1, 1024) - self.assertRaises(ValueError, f.read1) - if hasattr(f, "readall"): - self.assertRaises(ValueError, f.readall) - if hasattr(f, "readinto"): - self.assertRaises(ValueError, f.readinto, bytearray(1024)) - if hasattr(f, "readinto1"): - self.assertRaises(ValueError, f.readinto1, bytearray(1024)) - self.assertRaises(ValueError, f.readline) - self.assertRaises(ValueError, f.readlines) - self.assertRaises(ValueError, f.readlines, 1) - self.assertRaises(ValueError, f.seek, 0) - self.assertRaises(ValueError, f.tell) - self.assertRaises(ValueError, f.truncate) - self.assertRaises(ValueError, f.write, - b"" if "b" in kwargs['mode'] else "") - self.assertRaises(ValueError, f.writelines, []) - self.assertRaises(ValueError, next, f) - - def test_blockingioerror(self): - # Various BlockingIOError issues - class C(str): - pass - c = C("") - b = self.BlockingIOError(1, c) - c.b = b - b.c = c - wr = weakref.ref(c) - del c, b - support.gc_collect() - self.assertIsNone(wr(), wr) - - def test_abcs(self): - # Test the visible base classes are ABCs. - self.assertIsInstance(self.IOBase, abc.ABCMeta) - self.assertIsInstance(self.RawIOBase, abc.ABCMeta) - self.assertIsInstance(self.BufferedIOBase, abc.ABCMeta) - self.assertIsInstance(self.TextIOBase, abc.ABCMeta) - - def _check_abc_inheritance(self, abcmodule): - with self.open(os_helper.TESTFN, "wb", buffering=0) as f: - self.assertIsInstance(f, abcmodule.IOBase) - self.assertIsInstance(f, abcmodule.RawIOBase) - self.assertNotIsInstance(f, abcmodule.BufferedIOBase) - self.assertNotIsInstance(f, abcmodule.TextIOBase) - with self.open(os_helper.TESTFN, "wb") as f: - self.assertIsInstance(f, abcmodule.IOBase) - self.assertNotIsInstance(f, abcmodule.RawIOBase) - self.assertIsInstance(f, abcmodule.BufferedIOBase) - self.assertNotIsInstance(f, abcmodule.TextIOBase) - with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: - self.assertIsInstance(f, abcmodule.IOBase) - self.assertNotIsInstance(f, abcmodule.RawIOBase) - self.assertNotIsInstance(f, abcmodule.BufferedIOBase) - self.assertIsInstance(f, abcmodule.TextIOBase) - - def test_abc_inheritance(self): - # Test implementations inherit from their respective ABCs - self._check_abc_inheritance(self) - - def test_abc_inheritance_official(self): - # Test implementations inherit from the official ABCs of the - # baseline "io" module. - self._check_abc_inheritance(io) - - def _check_warn_on_dealloc(self, *args, **kwargs): - f = self.open(*args, **kwargs) - r = repr(f) - with self.assertWarns(ResourceWarning) as cm: - f = None - support.gc_collect() - self.assertIn(r, str(cm.warning.args[0])) - - def test_warn_on_dealloc(self): - self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) - self._check_warn_on_dealloc(os_helper.TESTFN, "wb") - self._check_warn_on_dealloc(os_helper.TESTFN, "w", encoding="utf-8") - - def _check_warn_on_dealloc_fd(self, *args, **kwargs): - fds = [] - def cleanup_fds(): - for fd in fds: - try: - os.close(fd) - except OSError as e: - if e.errno != errno.EBADF: - raise - self.addCleanup(cleanup_fds) - r, w = os.pipe() - fds += r, w - self._check_warn_on_dealloc(r, *args, **kwargs) - # When using closefd=False, there's no warning - r, w = os.pipe() - fds += r, w - with warnings_helper.check_no_resource_warning(self): - self.open(r, *args, closefd=False, **kwargs) - - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_warn_on_dealloc_fd(self): - self._check_warn_on_dealloc_fd("rb", buffering=0) - self._check_warn_on_dealloc_fd("rb") - self._check_warn_on_dealloc_fd("r", encoding="utf-8") - - - def test_pickling(self): - # Pickling file objects is forbidden - msg = "cannot pickle" - for kwargs in [ - {"mode": "w"}, - {"mode": "wb"}, - {"mode": "wb", "buffering": 0}, - {"mode": "r"}, - {"mode": "rb"}, - {"mode": "rb", "buffering": 0}, - {"mode": "w+"}, - {"mode": "w+b"}, - {"mode": "w+b", "buffering": 0}, - ]: - if "b" not in kwargs["mode"]: - kwargs["encoding"] = "utf-8" - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(protocol=protocol, kwargs=kwargs): - with self.open(os_helper.TESTFN, **kwargs) as f: - with self.assertRaisesRegex(TypeError, msg): - pickle.dumps(f, protocol) - - @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") - def test_nonblock_pipe_write_bigbuf(self): - self._test_nonblock_pipe_write(16*1024) - - @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") - def test_nonblock_pipe_write_smallbuf(self): - self._test_nonblock_pipe_write(1024) - - @unittest.skipUnless(hasattr(os, 'set_blocking'), - 'os.set_blocking() required for this test') - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def _test_nonblock_pipe_write(self, bufsize): - sent = [] - received = [] - r, w = os.pipe() - os.set_blocking(r, False) - os.set_blocking(w, False) - - # To exercise all code paths in the C implementation we need - # to play with buffer sizes. For instance, if we choose a - # buffer size less than or equal to _PIPE_BUF (4096 on Linux) - # then we will never get a partial write of the buffer. - rf = self.open(r, mode='rb', closefd=True, buffering=bufsize) - wf = self.open(w, mode='wb', closefd=True, buffering=bufsize) - - with rf, wf: - for N in 9999, 73, 7574: - try: - i = 0 - while True: - msg = bytes([i % 26 + 97]) * N - sent.append(msg) - wf.write(msg) - i += 1 - - except self.BlockingIOError as e: - self.assertEqual(e.args[0], errno.EAGAIN) - self.assertEqual(e.args[2], e.characters_written) - sent[-1] = sent[-1][:e.characters_written] - received.append(rf.read()) - msg = b'BLOCKED' - wf.write(msg) - sent.append(msg) - - while True: - try: - wf.flush() - break - except self.BlockingIOError as e: - self.assertEqual(e.args[0], errno.EAGAIN) - self.assertEqual(e.args[2], e.characters_written) - self.assertEqual(e.characters_written, 0) - received.append(rf.read()) - - received += iter(rf.read, None) - - sent, received = b''.join(sent), b''.join(received) - self.assertEqual(sent, received) - self.assertTrue(wf.closed) - self.assertTrue(rf.closed) - - def test_create_fail(self): - # 'x' mode fails if file is existing - with self.open(os_helper.TESTFN, 'w', encoding="utf-8"): - pass - self.assertRaises(FileExistsError, self.open, os_helper.TESTFN, 'x', encoding="utf-8") - - def test_create_writes(self): - # 'x' mode opens for writing - with self.open(os_helper.TESTFN, 'xb') as f: - f.write(b"spam") - with self.open(os_helper.TESTFN, 'rb') as f: - self.assertEqual(b"spam", f.read()) - - def test_open_allargs(self): - # there used to be a buffer overflow in the parser for rawmode - self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'rwax+', encoding="utf-8") - - def test_check_encoding_errors(self): - # bpo-37388: open() and TextIOWrapper must check encoding and errors - # arguments in dev mode - mod = self.io.__name__ - filename = __file__ - invalid = 'Boom, Shaka Laka, Boom!' - code = textwrap.dedent(f''' - import sys - from {mod} import open, TextIOWrapper - - try: - open({filename!r}, encoding={invalid!r}) - except LookupError: - pass - else: - sys.exit(21) - - try: - open({filename!r}, errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(22) - - fp = open({filename!r}, "rb") - with fp: - try: - TextIOWrapper(fp, encoding={invalid!r}) - except LookupError: - pass - else: - sys.exit(23) - - try: - TextIOWrapper(fp, errors={invalid!r}) - except LookupError: - pass - else: - sys.exit(24) - - sys.exit(10) - ''') - proc = assert_python_failure('-X', 'dev', '-c', code) - self.assertEqual(proc.rc, 10, proc) - - def test_check_encoding_warning(self): - # PEP 597: Raise warning when encoding is not specified - # and sys.flags.warn_default_encoding is set. - mod = self.io.__name__ - filename = __file__ - code = textwrap.dedent(f'''\ - import sys - from {mod} import open, TextIOWrapper - import pathlib - - with open({filename!r}) as f: # line 5 - pass - - pathlib.Path({filename!r}).read_text() # line 8 - ''') - proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) - warnings = proc.err.splitlines() - self.assertEqual(len(warnings), 2) - self.assertStartsWith(warnings[0], b":5: EncodingWarning: ") - self.assertStartsWith(warnings[1], b":8: EncodingWarning: ") - - def test_text_encoding(self): - # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" - # based on sys.flags.utf8_mode - code = "import io; print(io.text_encoding(None))" - - proc = assert_python_ok('-X', 'utf8=0', '-c', code) - self.assertEqual(b"locale", proc.out.strip()) - - proc = assert_python_ok('-X', 'utf8=1', '-c', code) - self.assertEqual(b"utf-8", proc.out.strip()) - - -class CMiscIOTest(MiscIOTest): - io = io - name_of_module = "io", "_io" - extra_exported = "BlockingIOError", - - def test_readinto_buffer_overflow(self): - # Issue #18025 - class BadReader(self.io.BufferedIOBase): - def read(self, n=-1): - return b'x' * 10**6 - bufio = BadReader() - b = bytearray(2) - self.assertRaises(ValueError, bufio.readinto, b) - - def check_daemon_threads_shutdown_deadlock(self, stream_name): - # Issue #23309: deadlocks at shutdown should be avoided when a - # daemon thread and the main thread both write to a file. - code = """if 1: - import sys - import time - import threading - from test.support import SuppressCrashReport - - file = sys.{stream_name} - - def run(): - while True: - file.write('.') - file.flush() - - crash = SuppressCrashReport() - crash.__enter__() - # don't call __exit__(): the crash occurs at Python shutdown - - thread = threading.Thread(target=run) - thread.daemon = True - thread.start() - - time.sleep(0.5) - file.write('!') - file.flush() - """.format_map(locals()) - res, _ = run_python_until_end("-c", code) - err = res.err.decode() - if res.rc != 0: - # Failure: should be a fatal error - pattern = (r"Fatal Python error: _enter_buffered_busy: " - r"could not acquire lock " - r"for <(_io\.)?BufferedWriter name='<{stream_name}>'> " - r"at interpreter shutdown, possibly due to " - r"daemon threads".format_map(locals())) - self.assertRegex(err, pattern) - else: - self.assertFalse(err.strip('.!')) - - @threading_helper.requires_working_threading() - @support.requires_resource('walltime') - def test_daemon_threads_shutdown_stdout_deadlock(self): - self.check_daemon_threads_shutdown_deadlock('stdout') - - @threading_helper.requires_working_threading() - @support.requires_resource('walltime') - def test_daemon_threads_shutdown_stderr_deadlock(self): - self.check_daemon_threads_shutdown_deadlock('stderr') - - -class PyMiscIOTest(MiscIOTest): - io = pyio - name_of_module = "_pyio", "io" - extra_exported = "BlockingIOError", "open_code", - not_exported = "valid_seek_flags", - - -@unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.') -class SignalsTest(unittest.TestCase): - - def setUp(self): - self.oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) - - def tearDown(self): - signal.signal(signal.SIGALRM, self.oldalrm) - - def alarm_interrupt(self, sig, frame): - 1/0 - - def check_interrupted_write(self, item, bytes, **fdopen_kwargs): - """Check that a partial write, when it gets interrupted, properly - invokes the signal handler, and bubbles up the exception raised - in the latter.""" - - # XXX This test has three flaws that appear when objects are - # XXX not reference counted. - - # - if wio.write() happens to trigger a garbage collection, - # the signal exception may be raised when some __del__ - # method is running; it will not reach the assertRaises() - # call. - - # - more subtle, if the wio object is not destroyed at once - # and survives this function, the next opened file is likely - # to have the same fileno (since the file descriptor was - # actively closed). When wio.__del__ is finally called, it - # will close the other's test file... To trigger this with - # CPython, try adding "global wio" in this function. - - # - This happens only for streams created by the _pyio module, - # because a wio.close() that fails still consider that the - # file needs to be closed again. You can try adding an - # "assert wio.closed" at the end of the function. - - # Fortunately, a little gc.collect() seems to be enough to - # work around all these issues. - support.gc_collect() # For PyPy or other GCs. - - read_results = [] - def _read(): - s = os.read(r, 1) - read_results.append(s) - - t = threading.Thread(target=_read) - t.daemon = True - r, w = os.pipe() - fdopen_kwargs["closefd"] = False - large_data = item * (support.PIPE_MAX_SIZE // len(item) + 1) - try: - wio = self.io.open(w, **fdopen_kwargs) - if hasattr(signal, 'pthread_sigmask'): - # create the thread with SIGALRM signal blocked - signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) - t.start() - signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGALRM]) - else: - t.start() - - # Fill the pipe enough that the write will be blocking. - # It will be interrupted by the timer armed above. Since the - # other thread has read one byte, the low-level write will - # return with a successful (partial) result rather than an EINTR. - # The buffered IO layer must check for pending signal - # handlers, which in this case will invoke alarm_interrupt(). - signal.alarm(1) - try: - self.assertRaises(ZeroDivisionError, wio.write, large_data) - finally: - signal.alarm(0) - t.join() - # We got one byte, get another one and check that it isn't a - # repeat of the first one. - read_results.append(os.read(r, 1)) - self.assertEqual(read_results, [bytes[0:1], bytes[1:2]]) - finally: - os.close(w) - os.close(r) - # This is deliberate. If we didn't close the file descriptor - # before closing wio, wio would try to flush its internal - # buffer, and block again. - try: - wio.close() - except OSError as e: - if e.errno != errno.EBADF: - raise - - @requires_alarm - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_interrupted_write_unbuffered(self): - self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0) - - @requires_alarm - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_interrupted_write_buffered(self): - self.check_interrupted_write(b"xy", b"xy", mode="wb") - - @requires_alarm - @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") - def test_interrupted_write_text(self): - self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") - - @support.no_tracing - def check_reentrant_write(self, data, **fdopen_kwargs): - def on_alarm(*args): - # Will be called reentrantly from the same thread - wio.write(data) - 1/0 - signal.signal(signal.SIGALRM, on_alarm) - r, w = os.pipe() - wio = self.io.open(w, **fdopen_kwargs) - try: - signal.alarm(1) - # Either the reentrant call to wio.write() fails with RuntimeError, - # or the signal handler raises ZeroDivisionError. - with self.assertRaises((ZeroDivisionError, RuntimeError)) as cm: - while 1: - for i in range(100): - wio.write(data) - wio.flush() - # Make sure the buffer doesn't fill up and block further writes - os.read(r, len(data) * 100) - exc = cm.exception - if isinstance(exc, RuntimeError): - self.assertStartsWith(str(exc), "reentrant call") - finally: - signal.alarm(0) - wio.close() - os.close(r) - - @requires_alarm - def test_reentrant_write_buffered(self): - self.check_reentrant_write(b"xy", mode="wb") - - @requires_alarm - def test_reentrant_write_text(self): - self.check_reentrant_write("xy", mode="w", encoding="ascii") - - def check_interrupted_read_retry(self, decode, **fdopen_kwargs): - """Check that a buffered read, when it gets interrupted (either - returning a partial result or EINTR), properly invokes the signal - handler and retries if the latter returned successfully.""" - r, w = os.pipe() - fdopen_kwargs["closefd"] = False - def alarm_handler(sig, frame): - os.write(w, b"bar") - signal.signal(signal.SIGALRM, alarm_handler) - try: - rio = self.io.open(r, **fdopen_kwargs) - os.write(w, b"foo") - signal.alarm(1) - # Expected behaviour: - # - first raw read() returns partial b"foo" - # - second raw read() returns EINTR - # - third raw read() returns b"bar" - self.assertEqual(decode(rio.read(6)), "foobar") - finally: - signal.alarm(0) - rio.close() - os.close(w) - os.close(r) - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_read_retry_buffered(self): - self.check_interrupted_read_retry(lambda x: x.decode('latin1'), - mode="rb") - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_read_retry_text(self): - self.check_interrupted_read_retry(lambda x: x, - mode="r", encoding="latin1") - - def check_interrupted_write_retry(self, item, **fdopen_kwargs): - """Check that a buffered write, when it gets interrupted (either - returning a partial result or EINTR), properly invokes the signal - handler and retries if the latter returned successfully.""" - select = import_helper.import_module("select") - - # A quantity that exceeds the buffer size of an anonymous pipe's - # write end. - N = support.PIPE_MAX_SIZE - r, w = os.pipe() - fdopen_kwargs["closefd"] = False - - # We need a separate thread to read from the pipe and allow the - # write() to finish. This thread is started after the SIGALRM is - # received (forcing a first EINTR in write()). - read_results = [] - write_finished = False - error = None - def _read(): - try: - while not write_finished: - while r in select.select([r], [], [], 1.0)[0]: - s = os.read(r, 1024) - read_results.append(s) - except BaseException as exc: - nonlocal error - error = exc - t = threading.Thread(target=_read) - t.daemon = True - def alarm1(sig, frame): - signal.signal(signal.SIGALRM, alarm2) - signal.alarm(1) - def alarm2(sig, frame): - t.start() - - large_data = item * N - signal.signal(signal.SIGALRM, alarm1) - try: - wio = self.io.open(w, **fdopen_kwargs) - signal.alarm(1) - # Expected behaviour: - # - first raw write() is partial (because of the limited pipe buffer - # and the first alarm) - # - second raw write() returns EINTR (because of the second alarm) - # - subsequent write()s are successful (either partial or complete) - written = wio.write(large_data) - self.assertEqual(N, written) - - wio.flush() - write_finished = True - t.join() - - self.assertIsNone(error) - self.assertEqual(N, sum(len(x) for x in read_results)) - finally: - signal.alarm(0) - write_finished = True - os.close(w) - os.close(r) - # This is deliberate. If we didn't close the file descriptor - # before closing wio, wio would try to flush its internal - # buffer, and could block (in case of failure). - try: - wio.close() - except OSError as e: - if e.errno != errno.EBADF: - raise - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_write_retry_buffered(self): - self.check_interrupted_write_retry(b"x", mode="wb") - - @requires_alarm - @support.requires_resource('walltime') - def test_interrupted_write_retry_text(self): - self.check_interrupted_write_retry("x", mode="w", encoding="latin1") - - -class CSignalsTest(SignalsTest): - io = io - -class PySignalsTest(SignalsTest): - io = pyio - - # Handling reentrancy issues would slow down _pyio even more, so the - # tests are disabled. - test_reentrant_write_buffered = None - test_reentrant_write_text = None - - -class ProtocolsTest(unittest.TestCase): - class MyReader: - def read(self, sz=-1): - return b"" - - class MyWriter: - def write(self, b: bytes): - pass - - def test_reader_subclass(self): - self.assertIsSubclass(self.MyReader, io.Reader) - self.assertNotIsSubclass(str, io.Reader) - - def test_writer_subclass(self): - self.assertIsSubclass(self.MyWriter, io.Writer) - self.assertNotIsSubclass(str, io.Writer) - - -def load_tests(loader, tests, pattern): - tests = (CIOTest, PyIOTest, APIMismatchTest, - CBufferedReaderTest, PyBufferedReaderTest, - CBufferedWriterTest, PyBufferedWriterTest, - CBufferedRWPairTest, PyBufferedRWPairTest, - CBufferedRandomTest, PyBufferedRandomTest, - StatefulIncrementalDecoderTest, - CIncrementalNewlineDecoderTest, PyIncrementalNewlineDecoderTest, - CTextIOWrapperTest, PyTextIOWrapperTest, - CMiscIOTest, PyMiscIOTest, - CSignalsTest, PySignalsTest, TestIOCTypes, - ProtocolsTest, - ) - - # Put the namespaces of the IO module we are testing and some useful mock - # classes in the __dict__ of each test. - mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO, - MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead, - SlowFlushRawIO, MockCharPseudoDevFileIO) - all_members = io.__all__ - c_io_ns = {name : getattr(io, name) for name in all_members} - py_io_ns = {name : getattr(pyio, name) for name in all_members} - globs = globals() - c_io_ns.update((x.__name__, globs["C" + x.__name__]) for x in mocks) - py_io_ns.update((x.__name__, globs["Py" + x.__name__]) for x in mocks) - for test in tests: - if test.__name__.startswith("C"): - for name, obj in c_io_ns.items(): - setattr(test, name, obj) - test.is_C = True - elif test.__name__.startswith("Py"): - for name, obj in py_io_ns.items(): - setattr(test, name, obj) - test.is_C = False - - suite = loader.suiteClass() - for test in tests: - suite.addTest(loader.loadTestsFromTestCase(test)) - return suite - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_io/__init__.py b/Lib/test/test_io/__init__.py new file mode 100644 index 00000000000..c94fad3e779 --- /dev/null +++ b/Lib/test/test_io/__init__.py @@ -0,0 +1,28 @@ +"""Tests for the io module and its implementations (_io and _pyio) + +Tests are split across multiple files to increase +parallelism and focus on specific implementation pieces. + +* test_io + * test_bufferedio - tests file buffering + * test_memoryio - tests BytesIO and StringIO + * test_fileio - tests FileIO + * test_file - tests the file interface + * test_general - tests everything else in the io module + * test_univnewlines - tests universal newline support + * test_largefile - tests operations on a file greater than 2**32 bytes + (only enabled with -ulargefile) +* test_free_threading/test_io - tests thread safety of io objects + +.. attention:: + When writing tests for io, it's important to test both the C and Python + implementations. This is usually done by writing a base test that refers to + the type it is testing as an attribute. Then it provides custom subclasses to + test both implementations. This directory contains lots of examples. +""" + +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_io/__main__.py b/Lib/test/test_io/__main__.py new file mode 100644 index 00000000000..40a23a297ec --- /dev/null +++ b/Lib/test/test_io/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py new file mode 100644 index 00000000000..5aaaf13a3d4 --- /dev/null +++ b/Lib/test/test_io/test_bufferedio.py @@ -0,0 +1,1570 @@ +import array +import pickle +import random +import sys +import threading +import time +import unittest +import warnings +import weakref +from collections import deque, UserList +from itertools import cycle, count +from test import support +from test.support import check_sanitizer, os_helper, threading_helper +from .utils import byteslike, CTestCase, PyTestCase + + +import io # C implementation. +import _pyio as pyio # Python implementation. + + +class CommonBufferedTests: + # Tests common to BufferedReader, BufferedWriter and BufferedRandom + + def test_detach(self): + raw = self.MockRawIO() + buf = self.tp(raw) + self.assertIs(buf.detach(), raw) + self.assertRaises(ValueError, buf.detach) + + repr(buf) # Should still work + + def test_fileno(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + + self.assertEqual(42, bufio.fileno()) + + def test_invalid_args(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + # Invalid whence + self.assertRaises(ValueError, bufio.seek, 0, -1) + self.assertRaises(ValueError, bufio.seek, 0, 9) + + def test_override_destructor(self): + tp = self.tp + record = [] + class MyBufferedIO(tp): + def __del__(self): + record.append(1) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(2) + super().close() + def flush(self): + record.append(3) + super().flush() + rawio = self.MockRawIO() + bufio = MyBufferedIO(rawio) + del bufio + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + + def test_context_manager(self): + # Test usability as a context manager + rawio = self.MockRawIO() + bufio = self.tp(rawio) + def _with(): + with bufio: + pass + _with() + # bufio should now be closed, and using it a second time should raise + # a ValueError. + self.assertRaises(ValueError, _with) + + def test_error_through_destructor(self): + # Test that the exception state is not modified by a destructor, + # even if close() fails. + rawio = self.CloseFailureIO() + with support.catch_unraisable_exception() as cm: + with self.assertRaises(AttributeError): + self.tp(rawio).xyzzy + + self.assertEqual(cm.unraisable.exc_type, OSError) + + def test_repr(self): + raw = self.MockRawIO() + b = self.tp(raw) + clsname = r"(%s\.)?%s" % (self.tp.__module__, self.tp.__qualname__) + self.assertRegex(repr(b), "<%s>" % clsname) + raw.name = "dummy" + self.assertRegex(repr(b), "<%s name='dummy'>" % clsname) + raw.name = b"dummy" + self.assertRegex(repr(b), "<%s name=b'dummy'>" % clsname) + + def test_recursive_repr(self): + # Issue #25455 + raw = self.MockRawIO() + b = self.tp(raw) + with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): + repr(b) # Should not crash + + def test_flush_error_on_close(self): + # Test that buffered file is closed despite failed flush + # and that flush() is called before file closed. + raw = self.MockRawIO() + closed = [] + def bad_flush(): + closed[:] = [b.closed, raw.closed] + raise OSError() + raw.flush = bad_flush + b = self.tp(raw) + self.assertRaises(OSError, b.close) # exception not swallowed + self.assertTrue(b.closed) + self.assertTrue(raw.closed) + self.assertTrue(closed) # flush() called + self.assertFalse(closed[0]) # flush() called before file closed + self.assertFalse(closed[1]) + raw.flush = lambda: None # break reference loop + + def test_close_error_on_close(self): + raw = self.MockRawIO() + def bad_flush(): + raise OSError('flush') + def bad_close(): + raise OSError('close') + raw.close = bad_close + b = self.tp(raw) + b.flush = bad_flush + with self.assertRaises(OSError) as err: # exception not swallowed + b.close() + self.assertEqual(err.exception.args, ('close',)) + self.assertIsInstance(err.exception.__context__, OSError) + self.assertEqual(err.exception.__context__.args, ('flush',)) + self.assertFalse(b.closed) + + # Silence destructor error + raw.close = lambda: None + b.flush = lambda: None + + def test_nonnormalized_close_error_on_close(self): + # Issue #21677 + raw = self.MockRawIO() + def bad_flush(): + raise non_existing_flush + def bad_close(): + raise non_existing_close + raw.close = bad_close + b = self.tp(raw) + b.flush = bad_flush + with self.assertRaises(NameError) as err: # exception not swallowed + b.close() + self.assertIn('non_existing_close', str(err.exception)) + self.assertIsInstance(err.exception.__context__, NameError) + self.assertIn('non_existing_flush', str(err.exception.__context__)) + self.assertFalse(b.closed) + + # Silence destructor error + b.flush = lambda: None + raw.close = lambda: None + + def test_multi_close(self): + raw = self.MockRawIO() + b = self.tp(raw) + b.close() + b.close() + b.close() + self.assertRaises(ValueError, b.flush) + + def test_unseekable(self): + bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) + self.assertRaises(self.UnsupportedOperation, bufio.tell) + self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) + + def test_readonly_attributes(self): + raw = self.MockRawIO() + buf = self.tp(raw) + x = self.MockRawIO() + with self.assertRaises(AttributeError): + buf.raw = x + + def test_pickling_subclass(self): + global MyBufferedIO + class MyBufferedIO(self.tp): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.raw.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + buf = MyBufferedIO(raw, tag='ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(buf, proto) + newbuf = pickle.loads(pickled) + self.assertEqual(newbuf.raw.getvalue(), b'data') + self.assertEqual(newbuf.tag, 'ham') + del MyBufferedIO + + +class SizeofTest: + + @support.cpython_only + def test_sizeof(self): + bufsize1 = 4096 + bufsize2 = 8192 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize1) + size = sys.getsizeof(bufio) - bufsize1 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize2) + self.assertEqual(sys.getsizeof(bufio), size + bufsize2) + + @support.cpython_only + def test_buffer_freeing(self) : + bufsize = 4096 + rawio = self.MockRawIO() + bufio = self.tp(rawio, buffer_size=bufsize) + size = sys.getsizeof(bufio) - bufsize + bufio.close() + self.assertEqual(sys.getsizeof(bufio), size) + +class BufferedReaderTest(CommonBufferedTests): + read_mode = "rb" + + def test_constructor(self): + rawio = self.MockRawIO([b"abc"]) + bufio = self.tp(rawio) + bufio.__init__(rawio) + bufio.__init__(rawio, buffer_size=1024) + bufio.__init__(rawio, buffer_size=16) + self.assertEqual(b"abc", bufio.read()) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + rawio = self.MockRawIO([b"abc"]) + bufio.__init__(rawio) + self.assertEqual(b"abc", bufio.read()) + + def test_uninitialized(self): + bufio = self.tp.__new__(self.tp) + del bufio + bufio = self.tp.__new__(self.tp) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + bufio.read, 0) + bufio.__init__(self.MockRawIO()) + self.assertEqual(bufio.read(0), b'') + + def test_read(self): + for arg in (None, 7): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertEqual(b"abcdefg", bufio.read(arg)) + # Invalid args + self.assertRaises(ValueError, bufio.read, -2) + + def test_read1(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertEqual(b"a", bufio.read(1)) + self.assertEqual(b"b", bufio.read1(1)) + self.assertEqual(rawio._reads, 1) + self.assertEqual(b"", bufio.read1(0)) + self.assertEqual(b"c", bufio.read1(100)) + self.assertEqual(rawio._reads, 1) + self.assertEqual(b"d", bufio.read1(100)) + self.assertEqual(rawio._reads, 2) + self.assertEqual(b"efg", bufio.read1(100)) + self.assertEqual(rawio._reads, 3) + self.assertEqual(b"", bufio.read1(100)) + self.assertEqual(rawio._reads, 4) + + def test_read1_arbitrary(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertEqual(b"a", bufio.read(1)) + self.assertEqual(b"bc", bufio.read1()) + self.assertEqual(b"d", bufio.read1()) + self.assertEqual(b"efg", bufio.read1(-1)) + self.assertEqual(rawio._reads, 3) + self.assertEqual(b"", bufio.read1()) + self.assertEqual(rawio._reads, 4) + + def test_readinto(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + b = bytearray(2) + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"cd") + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ef") + self.assertEqual(bufio.readinto(b), 1) + self.assertEqual(b, b"gf") + self.assertEqual(bufio.readinto(b), 0) + self.assertEqual(b, b"gf") + rawio = self.MockRawIO((b"abc", None)) + bufio = self.tp(rawio) + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(bufio.readinto(b), 1) + self.assertEqual(b, b"cb") + + def test_readinto1(self): + buffer_size = 10 + rawio = self.MockRawIO((b"abc", b"de", b"fgh", b"jkl")) + bufio = self.tp(rawio, buffer_size=buffer_size) + b = bytearray(2) + self.assertEqual(bufio.peek(3), b'abc') + self.assertEqual(rawio._reads, 1) + self.assertEqual(bufio.readinto1(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(rawio._reads, 1) + self.assertEqual(bufio.readinto1(b), 1) + self.assertEqual(b[:1], b"c") + self.assertEqual(rawio._reads, 1) + self.assertEqual(bufio.readinto1(b), 2) + self.assertEqual(b, b"de") + self.assertEqual(rawio._reads, 2) + b = bytearray(2*buffer_size) + self.assertEqual(bufio.peek(3), b'fgh') + self.assertEqual(rawio._reads, 3) + self.assertEqual(bufio.readinto1(b), 6) + self.assertEqual(b[:6], b"fghjkl") + self.assertEqual(rawio._reads, 4) + + def test_readinto_array(self): + buffer_size = 60 + data = b"a" * 26 + rawio = self.MockRawIO((data,)) + bufio = self.tp(rawio, buffer_size=buffer_size) + + # Create an array with element size > 1 byte + b = array.array('i', b'x' * 32) + assert len(b) != 16 + + # Read into it. We should get as many *bytes* as we can fit into b + # (which is more than the number of elements) + n = bufio.readinto(b) + self.assertGreater(n, len(b)) + + # Check that old contents of b are preserved + bm = memoryview(b).cast('B') + self.assertLess(n, len(bm)) + self.assertEqual(bm[:n], data[:n]) + self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) + + def test_readinto1_array(self): + buffer_size = 60 + data = b"a" * 26 + rawio = self.MockRawIO((data,)) + bufio = self.tp(rawio, buffer_size=buffer_size) + + # Create an array with element size > 1 byte + b = array.array('i', b'x' * 32) + assert len(b) != 16 + + # Read into it. We should get as many *bytes* as we can fit into b + # (which is more than the number of elements) + n = bufio.readinto1(b) + self.assertGreater(n, len(b)) + + # Check that old contents of b are preserved + bm = memoryview(b).cast('B') + self.assertLess(n, len(bm)) + self.assertEqual(bm[:n], data[:n]) + self.assertEqual(bm[n:], b'x' * (len(bm[n:]))) + + def test_readlines(self): + def bufio(): + rawio = self.MockRawIO((b"abc\n", b"d\n", b"ef")) + return self.tp(rawio) + self.assertEqual(bufio().readlines(), [b"abc\n", b"d\n", b"ef"]) + self.assertEqual(bufio().readlines(5), [b"abc\n", b"d\n"]) + self.assertEqual(bufio().readlines(None), [b"abc\n", b"d\n", b"ef"]) + + def test_buffering(self): + data = b"abcdefghi" + dlen = len(data) + + tests = [ + [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ], + [ 100, [ 3, 3, 3], [ dlen ] ], + [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ], + ] + + for bufsize, buf_read_sizes, raw_read_sizes in tests: + rawio = self.MockFileIO(data) + bufio = self.tp(rawio, buffer_size=bufsize) + pos = 0 + for nbytes in buf_read_sizes: + self.assertEqual(bufio.read(nbytes), data[pos:pos+nbytes]) + pos += nbytes + # this is mildly implementation-dependent + self.assertEqual(rawio.read_history, raw_read_sizes) + + def test_read_non_blocking(self): + # Inject some None's in there to simulate EWOULDBLOCK + rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None)) + bufio = self.tp(rawio) + self.assertEqual(b"abcd", bufio.read(6)) + self.assertEqual(b"e", bufio.read(1)) + self.assertEqual(b"fg", bufio.read()) + self.assertEqual(b"", bufio.peek(1)) + self.assertIsNone(bufio.read()) + self.assertEqual(b"", bufio.read()) + + rawio = self.MockRawIO((b"a", None, None)) + self.assertEqual(b"a", rawio.readall()) + self.assertIsNone(rawio.readall()) + + def test_read_past_eof(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + + self.assertEqual(b"abcdefg", bufio.read(9000)) + + def test_read_all(self): + rawio = self.MockRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + + self.assertEqual(b"abcdefg", bufio.read()) + + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') + def test_threads(self): + try: + # Write out many bytes with exactly the same number of 0's, + # 1's... 255's. This will help us check that concurrent reading + # doesn't duplicate or forget contents. + N = 1000 + l = list(range(256)) * N + random.shuffle(l) + s = bytes(bytearray(l)) + with self.open(os_helper.TESTFN, "wb") as f: + f.write(s) + with self.open(os_helper.TESTFN, self.read_mode, buffering=0) as raw: + bufio = self.tp(raw, 8) + errors = [] + results = [] + def f(): + try: + # Intra-buffer read then buffer-flushing read + for n in cycle([1, 19]): + s = bufio.read(n) + if not s: + break + # list.append() is atomic + results.append(s) + except Exception as e: + errors.append(e) + raise + threads = [threading.Thread(target=f) for x in range(20)] + with threading_helper.start_threads(threads): + time.sleep(0.02) # yield + self.assertFalse(errors, + "the following exceptions were caught: %r" % errors) + s = b''.join(results) + for i in range(256): + c = bytes(bytearray([i])) + self.assertEqual(s.count(c), N) + finally: + os_helper.unlink(os_helper.TESTFN) + + def test_unseekable(self): + bufio = self.tp(self.MockUnseekableIO(b"A" * 10)) + self.assertRaises(self.UnsupportedOperation, bufio.tell) + self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) + bufio.read(1) + self.assertRaises(self.UnsupportedOperation, bufio.seek, 0) + self.assertRaises(self.UnsupportedOperation, bufio.tell) + + def test_misbehaved_io(self): + rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + self.assertRaises(OSError, bufio.seek, 0) + self.assertRaises(OSError, bufio.tell) + + # Silence destructor error + bufio.close = lambda: None + + def test_no_extraneous_read(self): + # Issue #9550; when the raw IO object has satisfied the read request, + # we should not issue any additional reads, otherwise it may block + # (e.g. socket). + bufsize = 16 + for n in (2, bufsize - 1, bufsize, bufsize + 1, bufsize * 2): + rawio = self.MockRawIO([b"x" * n]) + bufio = self.tp(rawio, bufsize) + self.assertEqual(bufio.read(n), b"x" * n) + # Simple case: one raw read is enough to satisfy the request. + self.assertEqual(rawio._extraneous_reads, 0, + "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) + # A more complex case where two raw reads are needed to satisfy + # the request. + rawio = self.MockRawIO([b"x" * (n - 1), b"x"]) + bufio = self.tp(rawio, bufsize) + self.assertEqual(bufio.read(n), b"x" * n) + self.assertEqual(rawio._extraneous_reads, 0, + "failed for {}: {} != 0".format(n, rawio._extraneous_reads)) + + def test_read_on_closed(self): + # Issue #23796 + b = self.BufferedReader(self.BytesIO(b"12")) + b.read(1) + b.close() + with self.subTest('peek'): + self.assertRaises(ValueError, b.peek) + with self.subTest('read1'): + self.assertRaises(ValueError, b.read1, 1) + with self.subTest('read'): + self.assertRaises(ValueError, b.read) + with self.subTest('readinto'): + self.assertRaises(ValueError, b.readinto, bytearray()) + with self.subTest('readinto1'): + self.assertRaises(ValueError, b.readinto1, bytearray()) + with self.subTest('flush'): + self.assertRaises(ValueError, b.flush) + with self.subTest('truncate'): + self.assertRaises(ValueError, b.truncate) + with self.subTest('seek'): + self.assertRaises(ValueError, b.seek, 0) + + def test_truncate_on_read_only(self): + rawio = self.MockFileIO(b"abc") + bufio = self.tp(rawio) + self.assertFalse(bufio.writable()) + self.assertRaises(self.UnsupportedOperation, bufio.truncate) + self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0) + + def test_tell_character_device_file(self): + # GH-95782 + # For the (former) bug in BufferedIO to manifest, the wrapped IO obj + # must be able to produce at least 2 bytes. + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.read(1), b"1") + self.assertEqual(buf.tell(), 0) + + def test_seek_character_device_file(self): + raw = self.MockCharPseudoDevFileIO(b"12") + buf = self.tp(raw) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.seek(1, io.SEEK_SET), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + self.assertEqual(buf.read(1), b"1") + + # In the C implementation, tell() sets the BufferedIO's abs_pos to 0, + # which means that the next seek() could return a negative offset if it + # does not sanity-check: + self.assertEqual(buf.tell(), 0) + self.assertEqual(buf.seek(0, io.SEEK_CUR), 0) + + +class CBufferedReaderTest(BufferedReaderTest, SizeofTest, CTestCase): + tp = io.BufferedReader + + def test_initialization(self): + rawio = self.MockRawIO([b"abc"]) + bufio = self.tp(rawio) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.read) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.read) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + self.assertRaises(ValueError, bufio.read) + + def test_misbehaved_io_read(self): + rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg")) + bufio = self.tp(rawio) + # _pyio.BufferedReader seems to implement reading different, so that + # checking this is not so easy. + self.assertRaises(OSError, bufio.read, 10) + + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") + def test_garbage_collection(self): + # C BufferedReader objects are collected. + # The Python version has __del__, so it ends into gc.garbage instead + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + rawio = self.FileIO(os_helper.TESTFN, "w+b") + f = self.tp(rawio) + f.f = f + wr = weakref.ref(f) + del f + support.gc_collect() + self.assertIsNone(wr(), wr) + + def test_args_error(self): + # Issue #17275 + with self.assertRaisesRegex(TypeError, "BufferedReader"): + self.tp(self.BytesIO(), 1024, 1024, 1024) + + def test_bad_readinto_value(self): + rawio = self.tp(self.BytesIO(b"12")) + rawio.readinto = lambda buf: -1 + bufio = self.tp(rawio) + with self.assertRaises(OSError) as cm: + bufio.readline() + self.assertIsNone(cm.exception.__cause__) + + def test_bad_readinto_type(self): + rawio = self.tp(self.BytesIO(b"12")) + rawio.readinto = lambda buf: b'' + bufio = self.tp(rawio) + with self.assertRaises(OSError) as cm: + bufio.readline() + self.assertIsInstance(cm.exception.__cause__, TypeError) + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @unittest.skipIf(check_sanitizer(thread=True), + 'ThreadSanitizer aborts on huge allocations (exit code 66).') + def test_read1_error_does_not_cause_reentrant_failure(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"hello") + + with self.open(os_helper.TESTFN, "rb", buffering=0) as raw: + bufio = self.tp(raw, buffer_size=8) + # To request a size that is far too huge to ever be satisfied, + # so that the internal buffer allocation reliably fails with MemoryError. + huge = sys.maxsize // 2 + 1 + with self.assertRaises(MemoryError): + bufio.read1(huge) + + # Used to crash before gh-143689: + self.assertEqual(bufio.read1(1), b"h") + + +class PyBufferedReaderTest(BufferedReaderTest, PyTestCase): + tp = pyio.BufferedReader + + +class BufferedWriterTest(CommonBufferedTests): + write_mode = "wb" + + def test_constructor(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + bufio.__init__(rawio) + bufio.__init__(rawio, buffer_size=1024) + bufio.__init__(rawio, buffer_size=16) + self.assertEqual(3, bufio.write(b"abc")) + bufio.flush() + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + bufio.__init__(rawio) + self.assertEqual(3, bufio.write(b"ghi")) + bufio.flush() + self.assertEqual(b"".join(rawio._write_stack), b"abcghi") + + def test_uninitialized(self): + bufio = self.tp.__new__(self.tp) + del bufio + bufio = self.tp.__new__(self.tp) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + bufio.write, b'') + bufio.__init__(self.MockRawIO()) + self.assertEqual(bufio.write(b''), 0) + + def test_detach_flush(self): + raw = self.MockRawIO() + buf = self.tp(raw) + buf.write(b"howdy!") + self.assertFalse(raw._write_stack) + buf.detach() + self.assertEqual(raw._write_stack, [b"howdy!"]) + + def test_write(self): + # Write to the buffered IO but don't overflow the buffer. + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.write(b"abc") + self.assertFalse(writer._write_stack) + buffer = bytearray(b"def") + bufio.write(buffer) + buffer[:] = b"***" # Overwrite our copy of the data + bufio.flush() + self.assertEqual(b"".join(writer._write_stack), b"abcdef") + + def test_write_overflow(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + contents = b"abcdefghijklmnop" + for n in range(0, len(contents), 3): + bufio.write(contents[n:n+3]) + flushed = b"".join(writer._write_stack) + # At least (total - 8) bytes were implicitly flushed, perhaps more + # depending on the implementation. + self.assertStartsWith(flushed, contents[:-8]) + + def check_writes(self, intermediate_func): + # Lots of writes, test the flushed output is as expected. + contents = bytes(range(256)) * 1000 + n = 0 + writer = self.MockRawIO() + bufio = self.tp(writer, 13) + # Generator of write sizes: repeat each N 15 times then proceed to N+1 + def gen_sizes(): + for size in count(1): + for i in range(15): + yield size + sizes = gen_sizes() + while n < len(contents): + size = min(next(sizes), len(contents) - n) + self.assertEqual(bufio.write(contents[n:n+size]), size) + intermediate_func(bufio) + n += size + bufio.flush() + self.assertEqual(contents, b"".join(writer._write_stack)) + + def test_writes(self): + self.check_writes(lambda bufio: None) + + def test_writes_and_flushes(self): + self.check_writes(lambda bufio: bufio.flush()) + + def test_writes_and_seeks(self): + def _seekabs(bufio): + pos = bufio.tell() + bufio.seek(pos + 1, 0) + bufio.seek(pos - 1, 0) + bufio.seek(pos, 0) + self.check_writes(_seekabs) + def _seekrel(bufio): + pos = bufio.seek(0, 1) + bufio.seek(+1, 1) + bufio.seek(-1, 1) + bufio.seek(pos, 0) + self.check_writes(_seekrel) + + def test_writes_and_truncates(self): + self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) + + def test_write_non_blocking(self): + raw = self.MockNonBlockWriterIO() + bufio = self.tp(raw, 8) + + self.assertEqual(bufio.write(b"abcd"), 4) + self.assertEqual(bufio.write(b"efghi"), 5) + # 1 byte will be written, the rest will be buffered + raw.block_on(b"k") + self.assertEqual(bufio.write(b"jklmn"), 5) + + # 8 bytes will be written, 8 will be buffered and the rest will be lost + raw.block_on(b"0") + try: + bufio.write(b"opqrwxyz0123456789") + except self.BlockingIOError as e: + written = e.characters_written + else: + self.fail("BlockingIOError should have been raised") + self.assertEqual(written, 16) + self.assertEqual(raw.pop_written(), + b"abcdefghijklmnopqrwxyz") + + self.assertEqual(bufio.write(b"ABCDEFGHI"), 9) + s = raw.pop_written() + # Previously buffered bytes were flushed + self.assertStartsWith(s, b"01234567A") + + def test_write_and_rewind(self): + raw = self.BytesIO() + bufio = self.tp(raw, 4) + self.assertEqual(bufio.write(b"abcdef"), 6) + self.assertEqual(bufio.tell(), 6) + bufio.seek(0, 0) + self.assertEqual(bufio.write(b"XY"), 2) + bufio.seek(6, 0) + self.assertEqual(raw.getvalue(), b"XYcdef") + self.assertEqual(bufio.write(b"123456"), 6) + bufio.flush() + self.assertEqual(raw.getvalue(), b"XYcdef123456") + + def test_flush(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.write(b"abc") + bufio.flush() + self.assertEqual(b"abc", writer._write_stack[0]) + + def test_writelines(self): + l = [b'ab', b'cd', b'ef'] + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.writelines(l) + bufio.flush() + self.assertEqual(b''.join(writer._write_stack), b'abcdef') + + def test_writelines_userlist(self): + l = UserList([b'ab', b'cd', b'ef']) + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.writelines(l) + bufio.flush() + self.assertEqual(b''.join(writer._write_stack), b'abcdef') + + def test_writelines_error(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + self.assertRaises(TypeError, bufio.writelines, [1, 2, 3]) + self.assertRaises(TypeError, bufio.writelines, None) + self.assertRaises(TypeError, bufio.writelines, 'abc') + + def test_destructor(self): + writer = self.MockRawIO() + bufio = self.tp(writer, 8) + bufio.write(b"abc") + del bufio + support.gc_collect() + self.assertEqual(b"abc", writer._write_stack[0]) + + def test_truncate(self): + # Truncate implicitly flushes the buffer. + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: + bufio = self.tp(raw, 8) + bufio.write(b"abcdef") + self.assertEqual(bufio.truncate(3), 3) + self.assertEqual(bufio.tell(), 6) + with self.open(os_helper.TESTFN, "rb", buffering=0) as f: + self.assertEqual(f.read(), b"abc") + + def test_truncate_after_write(self): + # Ensure that truncate preserves the file position after + # writes longer than the buffer size. + # Issue: https://bugs.python.org/issue32228 + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with self.open(os_helper.TESTFN, "wb") as f: + # Fill with some buffer + f.write(b'\x00' * 10000) + buffer_sizes = [8192, 4096, 200] + for buffer_size in buffer_sizes: + with self.open(os_helper.TESTFN, "r+b", buffering=buffer_size) as f: + f.write(b'\x00' * (buffer_size + 1)) + # After write write_pos and write_end are set to 0 + f.read(1) + # read operation makes sure that pos != raw_pos + f.truncate() + self.assertEqual(f.tell(), buffer_size + 2) + + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') + def test_threads(self): + try: + # Write out many bytes from many threads and test they were + # all flushed. + N = 1000 + contents = bytes(range(256)) * N + sizes = cycle([1, 19]) + n = 0 + queue = deque() + while n < len(contents): + size = next(sizes) + queue.append(contents[n:n+size]) + n += size + del contents + # We use a real file object because it allows us to + # exercise situations where the GIL is released before + # writing the buffer to the raw streams. This is in addition + # to concurrency issues due to switching threads in the middle + # of Python code. + with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw: + bufio = self.tp(raw, 8) + errors = [] + def f(): + try: + while True: + try: + s = queue.popleft() + except IndexError: + return + bufio.write(s) + except Exception as e: + errors.append(e) + raise + threads = [threading.Thread(target=f) for x in range(20)] + with threading_helper.start_threads(threads): + time.sleep(0.02) # yield + self.assertFalse(errors, + "the following exceptions were caught: %r" % errors) + bufio.close() + with self.open(os_helper.TESTFN, "rb") as f: + s = f.read() + for i in range(256): + self.assertEqual(s.count(bytes([i])), N) + finally: + os_helper.unlink(os_helper.TESTFN) + + def test_misbehaved_io(self): + rawio = self.MisbehavedRawIO() + bufio = self.tp(rawio, 5) + self.assertRaises(OSError, bufio.seek, 0) + self.assertRaises(OSError, bufio.tell) + self.assertRaises(OSError, bufio.write, b"abcdef") + + # Silence destructor error + bufio.close = lambda: None + + def test_max_buffer_size_removal(self): + with self.assertRaises(TypeError): + self.tp(self.MockRawIO(), 8, 12) + + def test_write_error_on_close(self): + raw = self.MockRawIO() + def bad_write(b): + raise OSError() + raw.write = bad_write + b = self.tp(raw) + b.write(b'spam') + self.assertRaises(OSError, b.close) # exception not swallowed + self.assertTrue(b.closed) + + @threading_helper.requires_working_threading() + def test_slow_close_from_thread(self): + # Issue #31976 + rawio = self.SlowFlushRawIO() + bufio = self.tp(rawio, 8) + t = threading.Thread(target=bufio.close) + t.start() + rawio.in_flush.wait() + self.assertRaises(ValueError, bufio.write, b'spam') + self.assertTrue(bufio.closed) + t.join() + + +class CBufferedWriterTest(BufferedWriterTest, SizeofTest, CTestCase): + tp = io.BufferedWriter + + def test_initialization(self): + rawio = self.MockRawIO() + bufio = self.tp(rawio) + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0) + self.assertRaises(ValueError, bufio.write, b"def") + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16) + self.assertRaises(ValueError, bufio.write, b"def") + self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) + self.assertRaises(ValueError, bufio.write, b"def") + + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") + def test_garbage_collection(self): + # C BufferedWriter objects are collected, and collecting them flushes + # all data to disk. + # The Python version has __del__, so it ends into gc.garbage instead + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + rawio = self.FileIO(os_helper.TESTFN, "w+b") + f = self.tp(rawio) + f.write(b"123xxx") + f.x = f + wr = weakref.ref(f) + del f + support.gc_collect() + self.assertIsNone(wr(), wr) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"123xxx") + + def test_args_error(self): + # Issue #17275 + with self.assertRaisesRegex(TypeError, "BufferedWriter"): + self.tp(self.BytesIO(), 1024, 1024, 1024) + + def test_non_boolean_closed_attr(self): + # gh-140650: check TypeError is raised + class MockRawIOWithoutClosed(self.MockRawIO): + closed = NotImplemented + + bufio = self.tp(MockRawIOWithoutClosed()) + self.assertRaises(TypeError, bufio.write, b"") + self.assertRaises(TypeError, bufio.flush) + self.assertRaises(TypeError, bufio.close) + + def test_closed_attr_raises(self): + class MockRawIOClosedRaises(self.MockRawIO): + @property + def closed(self): + raise ValueError("test") + + bufio = self.tp(MockRawIOClosedRaises()) + self.assertRaisesRegex(ValueError, "test", bufio.write, b"") + self.assertRaisesRegex(ValueError, "test", bufio.flush) + self.assertRaisesRegex(ValueError, "test", bufio.close) + + +class PyBufferedWriterTest(BufferedWriterTest, PyTestCase): + tp = pyio.BufferedWriter + +class BufferedRWPairTest: + + def test_constructor(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertFalse(pair.closed) + + def test_uninitialized(self): + pair = self.tp.__new__(self.tp) + del pair + pair = self.tp.__new__(self.tp) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + pair.read, 0) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + pair.write, b'') + pair.__init__(self.MockRawIO(), self.MockRawIO()) + self.assertEqual(pair.read(0), b'') + self.assertEqual(pair.write(b''), 0) + + def test_detach(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertRaises(self.UnsupportedOperation, pair.detach) + + def test_constructor_max_buffer_size_removal(self): + with self.assertRaises(TypeError): + self.tp(self.MockRawIO(), self.MockRawIO(), 8, 12) + + def test_constructor_with_not_readable(self): + class NotReadable(self.MockRawIO): + def readable(self): + return False + + self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO()) + + def test_constructor_with_not_writeable(self): + class NotWriteable(self.MockRawIO): + def writable(self): + return False + + self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable()) + + def test_read(self): + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + self.assertEqual(pair.read(3), b"abc") + self.assertEqual(pair.read(1), b"d") + self.assertEqual(pair.read(), b"ef") + pair = self.tp(self.BytesIO(b"abc"), self.MockRawIO()) + self.assertEqual(pair.read(None), b"abc") + + def test_readlines(self): + pair = lambda: self.tp(self.BytesIO(b"abc\ndef\nh"), self.MockRawIO()) + self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) + self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) + self.assertEqual(pair().readlines(5), [b"abc\n", b"def\n"]) + + def test_read1(self): + # .read1() is delegated to the underlying reader object, so this test + # can be shallow. + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + self.assertEqual(pair.read1(3), b"abc") + self.assertEqual(pair.read1(), b"def") + + def test_readinto(self): + for method in ("readinto", "readinto1"): + with self.subTest(method): + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + data = byteslike(b'\0' * 5) + self.assertEqual(getattr(pair, method)(data), 5) + self.assertEqual(bytes(data), b"abcde") + + # gh-138720: C BufferedRWPair would destruct in a bad order resulting in + # an unraisable exception. + support.gc_collect() + + def test_write(self): + w = self.MockRawIO() + pair = self.tp(self.MockRawIO(), w) + + pair.write(b"abc") + pair.flush() + buffer = bytearray(b"def") + pair.write(buffer) + buffer[:] = b"***" # Overwrite our copy of the data + pair.flush() + self.assertEqual(w._write_stack, [b"abc", b"def"]) + + def test_peek(self): + pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) + + self.assertStartsWith(pair.peek(3), b"abc") + self.assertEqual(pair.read(3), b"abc") + + def test_readable(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertTrue(pair.readable()) + + def test_writeable(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertTrue(pair.writable()) + + def test_seekable(self): + # BufferedRWPairs are never seekable, even if their readers and writers + # are. + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertFalse(pair.seekable()) + + # .flush() is delegated to the underlying writer object and has been + # tested in the test_write method. + + def test_close_and_closed(self): + pair = self.tp(self.MockRawIO(), self.MockRawIO()) + self.assertFalse(pair.closed) + pair.close() + self.assertTrue(pair.closed) + + def test_reader_close_error_on_close(self): + def reader_close(): + reader_non_existing + reader = self.MockRawIO() + reader.close = reader_close + writer = self.MockRawIO() + pair = self.tp(reader, writer) + with self.assertRaises(NameError) as err: + pair.close() + self.assertIn('reader_non_existing', str(err.exception)) + self.assertTrue(pair.closed) + self.assertFalse(reader.closed) + self.assertTrue(writer.closed) + + # Silence destructor error + reader.close = lambda: None + + def test_writer_close_error_on_close(self): + def writer_close(): + writer_non_existing + reader = self.MockRawIO() + writer = self.MockRawIO() + writer.close = writer_close + pair = self.tp(reader, writer) + with self.assertRaises(NameError) as err: + pair.close() + self.assertIn('writer_non_existing', str(err.exception)) + self.assertFalse(pair.closed) + self.assertTrue(reader.closed) + self.assertFalse(writer.closed) + + # Silence destructor error + writer.close = lambda: None + writer = None + + # Ignore BufferedWriter (of the BufferedRWPair) unraisable exception + with support.catch_unraisable_exception(): + # Ignore BufferedRWPair unraisable exception + with support.catch_unraisable_exception(): + pair = None + support.gc_collect() + support.gc_collect() + + def test_reader_writer_close_error_on_close(self): + def reader_close(): + reader_non_existing + def writer_close(): + writer_non_existing + reader = self.MockRawIO() + reader.close = reader_close + writer = self.MockRawIO() + writer.close = writer_close + pair = self.tp(reader, writer) + with self.assertRaises(NameError) as err: + pair.close() + self.assertIn('reader_non_existing', str(err.exception)) + self.assertIsInstance(err.exception.__context__, NameError) + self.assertIn('writer_non_existing', str(err.exception.__context__)) + self.assertFalse(pair.closed) + self.assertFalse(reader.closed) + self.assertFalse(writer.closed) + + # Silence destructor error + reader.close = lambda: None + writer.close = lambda: None + + def test_isatty(self): + class SelectableIsAtty(self.MockRawIO): + def __init__(self, isatty): + super().__init__() + self._isatty = isatty + + def isatty(self): + return self._isatty + + pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(False)) + self.assertFalse(pair.isatty()) + + pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(False)) + self.assertTrue(pair.isatty()) + + pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(True)) + self.assertTrue(pair.isatty()) + + pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(True)) + self.assertTrue(pair.isatty()) + + def test_weakref_clearing(self): + brw = self.tp(self.MockRawIO(), self.MockRawIO()) + ref = weakref.ref(brw) + brw = None + ref = None # Shouldn't segfault. + +class CBufferedRWPairTest(BufferedRWPairTest, CTestCase): + tp = io.BufferedRWPair + +class PyBufferedRWPairTest(BufferedRWPairTest, PyTestCase): + tp = pyio.BufferedRWPair + + +class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): + read_mode = "rb+" + write_mode = "wb+" + + def test_constructor(self): + BufferedReaderTest.test_constructor(self) + BufferedWriterTest.test_constructor(self) + + def test_uninitialized(self): + BufferedReaderTest.test_uninitialized(self) + BufferedWriterTest.test_uninitialized(self) + + def test_read_and_write(self): + raw = self.MockRawIO((b"asdf", b"ghjk")) + rw = self.tp(raw, 8) + + self.assertEqual(b"as", rw.read(2)) + rw.write(b"ddd") + rw.write(b"eee") + self.assertFalse(raw._write_stack) # Buffer writes + self.assertEqual(b"ghjk", rw.read()) + self.assertEqual(b"dddeee", raw._write_stack[0]) + + def test_seek_and_tell(self): + raw = self.BytesIO(b"asdfghjkl") + rw = self.tp(raw) + + self.assertEqual(b"as", rw.read(2)) + self.assertEqual(2, rw.tell()) + rw.seek(0, 0) + self.assertEqual(b"asdf", rw.read(4)) + + rw.write(b"123f") + rw.seek(0, 0) + self.assertEqual(b"asdf123fl", rw.read()) + self.assertEqual(9, rw.tell()) + rw.seek(-4, 2) + self.assertEqual(5, rw.tell()) + rw.seek(2, 1) + self.assertEqual(7, rw.tell()) + self.assertEqual(b"fl", rw.read(11)) + rw.flush() + self.assertEqual(b"asdf123fl", raw.getvalue()) + + self.assertRaises(TypeError, rw.seek, 0.0) + + def check_flush_and_read(self, read_func): + raw = self.BytesIO(b"abcdefghi") + bufio = self.tp(raw) + + self.assertEqual(b"ab", read_func(bufio, 2)) + bufio.write(b"12") + self.assertEqual(b"ef", read_func(bufio, 2)) + self.assertEqual(6, bufio.tell()) + bufio.flush() + self.assertEqual(6, bufio.tell()) + self.assertEqual(b"ghi", read_func(bufio)) + raw.seek(0, 0) + raw.write(b"XYZ") + # flush() resets the read buffer + bufio.flush() + bufio.seek(0, 0) + self.assertEqual(b"XYZ", read_func(bufio, 3)) + + def test_flush_and_read(self): + self.check_flush_and_read(lambda bufio, *args: bufio.read(*args)) + + def test_flush_and_readinto(self): + def _readinto(bufio, n=-1): + b = bytearray(n if n >= 0 else 9999) + n = bufio.readinto(b) + b.resize(n) + return b.take_bytes() + self.check_flush_and_read(_readinto) + + def test_flush_and_peek(self): + def _peek(bufio, n=-1): + # This relies on the fact that the buffer can contain the whole + # raw stream, otherwise peek() can return less. + b = bufio.peek(n) + if n != -1: + b = b[:n] + bufio.seek(len(b), 1) + return b + self.check_flush_and_read(_peek) + + def test_flush_and_write(self): + raw = self.BytesIO(b"abcdefghi") + bufio = self.tp(raw) + + bufio.write(b"123") + bufio.flush() + bufio.write(b"45") + bufio.flush() + bufio.seek(0, 0) + self.assertEqual(b"12345fghi", raw.getvalue()) + self.assertEqual(b"12345fghi", bufio.read()) + + def test_threads(self): + BufferedReaderTest.test_threads(self) + BufferedWriterTest.test_threads(self) + + def test_writes_and_peek(self): + def _peek(bufio): + bufio.peek(1) + self.check_writes(_peek) + def _peek(bufio): + pos = bufio.tell() + bufio.seek(-1, 1) + bufio.peek(1) + bufio.seek(pos, 0) + self.check_writes(_peek) + + def test_writes_and_reads(self): + def _read(bufio): + bufio.seek(-1, 1) + bufio.read(1) + self.check_writes(_read) + + def test_writes_and_read1s(self): + def _read1(bufio): + bufio.seek(-1, 1) + bufio.read1(1) + self.check_writes(_read1) + + def test_writes_and_readintos(self): + def _read(bufio): + bufio.seek(-1, 1) + bufio.readinto(bytearray(1)) + self.check_writes(_read) + + def test_write_after_readahead(self): + # Issue #6629: writing after the buffer was filled by readahead should + # first rewind the raw stream. + for overwrite_size in [1, 5]: + raw = self.BytesIO(b"A" * 10) + bufio = self.tp(raw, 4) + # Trigger readahead + self.assertEqual(bufio.read(1), b"A") + self.assertEqual(bufio.tell(), 1) + # Overwriting should rewind the raw stream if it needs so + bufio.write(b"B" * overwrite_size) + self.assertEqual(bufio.tell(), overwrite_size + 1) + # If the write size was smaller than the buffer size, flush() and + # check that rewind happens. + bufio.flush() + self.assertEqual(bufio.tell(), overwrite_size + 1) + s = raw.getvalue() + self.assertEqual(s, + b"A" + b"B" * overwrite_size + b"A" * (9 - overwrite_size)) + + def test_write_rewind_write(self): + # Various combinations of reading / writing / seeking backwards / writing again + def mutate(bufio, pos1, pos2): + assert pos2 >= pos1 + # Fill the buffer + bufio.seek(pos1) + bufio.read(pos2 - pos1) + bufio.write(b'\x02') + # This writes earlier than the previous write, but still inside + # the buffer. + bufio.seek(pos1) + bufio.write(b'\x01') + + b = b"\x80\x81\x82\x83\x84" + for i in range(0, len(b)): + for j in range(i, len(b)): + raw = self.BytesIO(b) + bufio = self.tp(raw, 100) + mutate(bufio, i, j) + bufio.flush() + expected = bytearray(b) + expected[j] = 2 + expected[i] = 1 + self.assertEqual(raw.getvalue(), expected, + "failed result for i=%d, j=%d" % (i, j)) + + def test_truncate_after_read_or_write(self): + raw = self.BytesIO(b"A" * 10) + bufio = self.tp(raw, 100) + self.assertEqual(bufio.read(2), b"AA") # the read buffer gets filled + self.assertEqual(bufio.truncate(), 2) + self.assertEqual(bufio.write(b"BB"), 2) # the write buffer increases + self.assertEqual(bufio.truncate(), 4) + + def test_misbehaved_io(self): + BufferedReaderTest.test_misbehaved_io(self) + BufferedWriterTest.test_misbehaved_io(self) + + def test_interleaved_read_write(self): + # Test for issue #12213 + with self.BytesIO(b'abcdefgh') as raw: + with self.tp(raw, 100) as f: + f.write(b"1") + self.assertEqual(f.read(1), b'b') + f.write(b'2') + self.assertEqual(f.read1(1), b'd') + f.write(b'3') + buf = bytearray(1) + f.readinto(buf) + self.assertEqual(buf, b'f') + f.write(b'4') + self.assertEqual(f.peek(1), b'h') + f.flush() + self.assertEqual(raw.getvalue(), b'1b2d3f4h') + + with self.BytesIO(b'abc') as raw: + with self.tp(raw, 100) as f: + self.assertEqual(f.read(1), b'a') + f.write(b"2") + self.assertEqual(f.read(1), b'c') + f.flush() + self.assertEqual(raw.getvalue(), b'a2c') + + def test_read1_after_write(self): + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(1), b'b') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + self.assertEqual(f.read1(), b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + with self.BytesIO(b'abcdef') as raw: + with self.tp(raw, 3) as f: + f.write(b"1") + # XXX: read(100) returns different numbers of bytes + # in Python and C implementations. + self.assertEqual(f.read1(100)[:3], b'bcd') + f.flush() + self.assertEqual(raw.getvalue(), b'1bcdef') + + def test_interleaved_readline_write(self): + with self.BytesIO(b'ab\ncdef\ng\n') as raw: + with self.tp(raw) as f: + f.write(b'1') + self.assertEqual(f.readline(), b'b\n') + f.write(b'2') + self.assertEqual(f.readline(), b'def\n') + f.write(b'3') + self.assertEqual(f.readline(), b'\n') + f.flush() + self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n') + + # You can't construct a BufferedRandom over a non-seekable stream. + test_unseekable = None + + # writable() returns True, so there's no point to test it over + # a writable stream. + test_truncate_on_read_only = None + + +class CBufferedRandomTest(BufferedRandomTest, SizeofTest, CTestCase): + tp = io.BufferedRandom + + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") + def test_garbage_collection(self): + CBufferedReaderTest.test_garbage_collection(self) + CBufferedWriterTest.test_garbage_collection(self) + + def test_args_error(self): + # Issue #17275 + with self.assertRaisesRegex(TypeError, "BufferedRandom"): + self.tp(self.BytesIO(), 1024, 1024, 1024) + + +class PyBufferedRandomTest(BufferedRandomTest, PyTestCase): + tp = pyio.BufferedRandom + + +# Simple test to ensure that optimizations in the IO library deliver the +# expected results. For best testing, run this under a debug-build Python too +# (to exercise asserts in the C code). + +lengths = list(range(1, 257)) + [512, 1000, 1024, 2048, 4096, 8192, 10000, + 16384, 32768, 65536, 1000000] + +class BufferSizeTest: + def try_one(self, s): + # Write s + "\n" + s to file, then open it and ensure that successive + # .readline()s deliver what we wrote. + + # Ensure we can open TESTFN for writing. + os_helper.unlink(os_helper.TESTFN) + + # Since C doesn't guarantee we can write/read arbitrary bytes in text + # files, use binary mode. + f = self.open(os_helper.TESTFN, "wb") + try: + # write once with \n and once without + f.write(s) + f.write(b"\n") + f.write(s) + f.close() + f = self.open(os_helper.TESTFN, "rb") + line = f.readline() + self.assertEqual(line, s + b"\n") + line = f.readline() + self.assertEqual(line, s) + line = f.readline() + self.assertFalse(line) # Must be at EOF + f.close() + finally: + os_helper.unlink(os_helper.TESTFN) + + def drive_one(self, pattern): + for length in lengths: + # Repeat string 'pattern' as often as needed to reach total length + # 'length'. Then call try_one with that string, a string one larger + # than that, and a string one smaller than that. Try this with all + # small sizes and various powers of 2, so we exercise all likely + # stdio buffer sizes, and "off by one" errors on both sides. + q, r = divmod(length, len(pattern)) + teststring = pattern * q + pattern[:r] + self.assertEqual(len(teststring), length) + self.try_one(teststring) + self.try_one(teststring + b"x") + self.try_one(teststring[:-1]) + + def test_primepat(self): + # A pattern with prime length, to avoid simple relationships with + # stdio buffer sizes. + self.drive_one(b"1234567890\00\01\02\03\04\05\06") + + def test_nullpat(self): + self.drive_one(b'\0' * 1000) + + +class CBufferSizeTest(BufferSizeTest, unittest.TestCase): + open = io.open + +class PyBufferSizeTest(BufferSizeTest, unittest.TestCase): + open = staticmethod(pyio.open) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_file.py b/Lib/test/test_io/test_file.py similarity index 100% rename from Lib/test/test_file.py rename to Lib/test/test_io/test_file.py diff --git a/Lib/test/test_fileio.py b/Lib/test/test_io/test_fileio.py similarity index 100% rename from Lib/test/test_fileio.py rename to Lib/test/test_io/test_fileio.py diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py new file mode 100644 index 00000000000..6a6671186ba --- /dev/null +++ b/Lib/test/test_io/test_general.py @@ -0,0 +1,1432 @@ +"""General tests for the io module. + +New tests should go in more specific modules; see test_io/__init__.py +""" + +import abc +import array +import errno +import os +import pickle +import sys +import textwrap +import threading +import unittest +import warnings +import weakref +from test import support +from test.support.script_helper import ( + assert_python_ok, assert_python_failure, run_python_until_end) +from test.support import ( + import_helper, is_apple, os_helper, threading_helper, warnings_helper, +) +from test.support.os_helper import FakePath +from .utils import byteslike, CTestCase, PyTestCase + +import io # C implementation of io +import _pyio as pyio # Python implementation of io + + +class IOTest: + + def setUp(self): + os_helper.unlink(os_helper.TESTFN) + + def tearDown(self): + os_helper.unlink(os_helper.TESTFN) + + def write_ops(self, f): + self.assertEqual(f.write(b"blah."), 5) + f.truncate(0) + self.assertEqual(f.tell(), 5) + f.seek(0) + + self.assertEqual(f.write(b"blah."), 5) + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.write(b"Hello."), 6) + self.assertEqual(f.tell(), 6) + self.assertEqual(f.seek(-1, 1), 5) + self.assertEqual(f.tell(), 5) + buffer = bytearray(b" world\n\n\n") + self.assertEqual(f.write(buffer), 9) + buffer[:] = b"*" * 9 # Overwrite our copy of the data + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.write(b"h"), 1) + self.assertEqual(f.seek(-1, 2), 13) + self.assertEqual(f.tell(), 13) + + self.assertEqual(f.truncate(12), 12) + self.assertEqual(f.tell(), 13) + self.assertRaises(TypeError, f.seek, 0.0) + + def read_ops(self, f, buffered=False): + data = f.read(5) + self.assertEqual(data, b"hello") + data = byteslike(data) + self.assertEqual(f.readinto(data), 5) + self.assertEqual(bytes(data), b" worl") + data = bytearray(5) + self.assertEqual(f.readinto(data), 2) + self.assertEqual(len(data), 5) + self.assertEqual(data[:2], b"d\n") + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.read(20), b"hello world\n") + self.assertEqual(f.read(1), b"") + self.assertEqual(f.readinto(byteslike(b"x")), 0) + self.assertEqual(f.seek(-6, 2), 6) + self.assertEqual(f.read(5), b"world") + self.assertEqual(f.read(0), b"") + self.assertEqual(f.readinto(byteslike()), 0) + self.assertEqual(f.seek(-6, 1), 5) + self.assertEqual(f.read(5), b" worl") + self.assertEqual(f.tell(), 10) + self.assertRaises(TypeError, f.seek, 0.0) + if buffered: + f.seek(0) + self.assertEqual(f.read(), b"hello world\n") + f.seek(6) + self.assertEqual(f.read(), b"world\n") + self.assertEqual(f.read(), b"") + f.seek(0) + data = byteslike(5) + self.assertEqual(f.readinto1(data), 5) + self.assertEqual(bytes(data), b"hello") + + LARGE = 2**31 + + def large_file_ops(self, f): + assert f.readable() + assert f.writable() + try: + self.assertEqual(f.seek(self.LARGE), self.LARGE) + except (OverflowError, ValueError): + self.skipTest("no largefile support") + self.assertEqual(f.tell(), self.LARGE) + self.assertEqual(f.write(b"xxx"), 3) + self.assertEqual(f.tell(), self.LARGE + 3) + self.assertEqual(f.seek(-1, 1), self.LARGE + 2) + self.assertEqual(f.truncate(), self.LARGE + 2) + self.assertEqual(f.tell(), self.LARGE + 2) + self.assertEqual(f.seek(0, 2), self.LARGE + 2) + self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1) + self.assertEqual(f.tell(), self.LARGE + 2) + self.assertEqual(f.seek(0, 2), self.LARGE + 1) + self.assertEqual(f.seek(-1, 2), self.LARGE) + self.assertEqual(f.read(2), b"x") + + def test_invalid_operations(self): + # Try writing on a file opened in read mode and vice-versa. + exc = self.UnsupportedOperation + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as fp: + self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readline) + with self.open(os_helper.TESTFN, "wb") as fp: + self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readline) + with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: + self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readall) + self.assertRaises(exc, fp.readline) + with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: + self.assertRaises(exc, fp.write, b"blah") + self.assertRaises(exc, fp.writelines, [b"blah\n"]) + with self.open(os_helper.TESTFN, "rb") as fp: + self.assertRaises(exc, fp.write, b"blah") + self.assertRaises(exc, fp.writelines, [b"blah\n"]) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as fp: + self.assertRaises(exc, fp.write, "blah") + self.assertRaises(exc, fp.writelines, ["blah\n"]) + # Non-zero seeking from current or end pos + self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR) + self.assertRaises(exc, fp.seek, -1, self.SEEK_END) + + @support.cpython_only + def test_startup_optimization(self): + # gh-132952: Test that `io` is not imported at startup and that the + # __module__ of UnsupportedOperation is set to "io". + assert_python_ok("-S", "-c", textwrap.dedent( + """ + import sys + assert "io" not in sys.modules + try: + sys.stdin.truncate() + except Exception as e: + typ = type(e) + assert typ.__module__ == "io", (typ, typ.__module__) + assert typ.__name__ == "UnsupportedOperation", (typ, typ.__name__) + else: + raise AssertionError("Expected UnsupportedOperation") + """ + )) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_optional_abilities(self): + # Test for OSError when optional APIs are not supported + # The purpose of this test is to try fileno(), reading, writing and + # seeking operations with various objects that indicate they do not + # support these operations. + + def pipe_reader(): + [r, w] = os.pipe() + os.close(w) # So that read() is harmless + return self.FileIO(r, "r") + + def pipe_writer(): + [r, w] = os.pipe() + self.addCleanup(os.close, r) + # Guarantee that we can write into the pipe without blocking + thread = threading.Thread(target=os.read, args=(r, 100)) + thread.start() + self.addCleanup(thread.join) + return self.FileIO(w, "w") + + def buffered_reader(): + return self.BufferedReader(self.MockUnseekableIO()) + + def buffered_writer(): + return self.BufferedWriter(self.MockUnseekableIO()) + + def buffered_random(): + return self.BufferedRandom(self.BytesIO()) + + def buffered_rw_pair(): + return self.BufferedRWPair(self.MockUnseekableIO(), + self.MockUnseekableIO()) + + def text_reader(): + class UnseekableReader(self.MockUnseekableIO): + writable = self.BufferedIOBase.writable + write = self.BufferedIOBase.write + return self.TextIOWrapper(UnseekableReader(), "ascii") + + def text_writer(): + class UnseekableWriter(self.MockUnseekableIO): + readable = self.BufferedIOBase.readable + read = self.BufferedIOBase.read + return self.TextIOWrapper(UnseekableWriter(), "ascii") + + tests = ( + (pipe_reader, "fr"), (pipe_writer, "fw"), + (buffered_reader, "r"), (buffered_writer, "w"), + (buffered_random, "rws"), (buffered_rw_pair, "rw"), + (text_reader, "r"), (text_writer, "w"), + (self.BytesIO, "rws"), (self.StringIO, "rws"), + ) + + def do_test(test, obj, abilities): + readable = "r" in abilities + self.assertEqual(obj.readable(), readable) + writable = "w" in abilities + self.assertEqual(obj.writable(), writable) + + if isinstance(obj, self.TextIOBase): + data = "3" + elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)): + data = b"3" + else: + self.fail("Unknown base class") + + if "f" in abilities: + obj.fileno() + else: + self.assertRaises(OSError, obj.fileno) + + if readable: + obj.read(1) + obj.read() + else: + self.assertRaises(OSError, obj.read, 1) + self.assertRaises(OSError, obj.read) + + if writable: + obj.write(data) + else: + self.assertRaises(OSError, obj.write, data) + + if sys.platform.startswith("win") and test in ( + pipe_reader, pipe_writer): + # Pipes seem to appear as seekable on Windows + return + seekable = "s" in abilities + self.assertEqual(obj.seekable(), seekable) + + if seekable: + obj.tell() + obj.seek(0) + else: + self.assertRaises(OSError, obj.tell) + self.assertRaises(OSError, obj.seek, 0) + + if writable and seekable: + obj.truncate() + obj.truncate(0) + else: + self.assertRaises(OSError, obj.truncate) + self.assertRaises(OSError, obj.truncate, 0) + + for [test, abilities] in tests: + with self.subTest(test): + if test == pipe_writer and not threading_helper.can_start_thread: + self.skipTest("Need threads") + with test() as obj: + do_test(test, obj, abilities) + + + def test_open_handles_NUL_chars(self): + fn_with_NUL = 'foo\0bar' + self.assertRaises(ValueError, self.open, fn_with_NUL, 'w', encoding="utf-8") + + bytes_fn = bytes(fn_with_NUL, 'ascii') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertRaises(ValueError, self.open, bytes_fn, 'w', encoding="utf-8") + + def test_raw_file_io(self): + with self.open(os_helper.TESTFN, "wb", buffering=0) as f: + self.assertEqual(f.readable(), False) + self.assertEqual(f.writable(), True) + self.assertEqual(f.seekable(), True) + self.write_ops(f) + with self.open(os_helper.TESTFN, "rb", buffering=0) as f: + self.assertEqual(f.readable(), True) + self.assertEqual(f.writable(), False) + self.assertEqual(f.seekable(), True) + self.read_ops(f) + + def test_buffered_file_io(self): + with self.open(os_helper.TESTFN, "wb") as f: + self.assertEqual(f.readable(), False) + self.assertEqual(f.writable(), True) + self.assertEqual(f.seekable(), True) + self.write_ops(f) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.readable(), True) + self.assertEqual(f.writable(), False) + self.assertEqual(f.seekable(), True) + self.read_ops(f, True) + + def test_readline(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"abc\ndef\nxyzzy\nfoo\x00bar\nanother line") + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.readline(), b"abc\n") + self.assertEqual(f.readline(10), b"def\n") + self.assertEqual(f.readline(2), b"xy") + self.assertEqual(f.readline(4), b"zzy\n") + self.assertEqual(f.readline(), b"foo\x00bar\n") + self.assertEqual(f.readline(None), b"another line") + self.assertRaises(TypeError, f.readline, 5.3) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + self.assertRaises(TypeError, f.readline, 5.3) + + def test_readline_nonsizeable(self): + # Issue #30061 + # Crash when readline() returns an object without __len__ + class R(self.IOBase): + def readline(self): + return None + self.assertRaises((TypeError, StopIteration), next, R()) + + def test_next_nonsizeable(self): + # Issue #30061 + # Crash when __next__() returns an object without __len__ + class R(self.IOBase): + def __next__(self): + return None + self.assertRaises(TypeError, R().readlines, 1) + + def test_raw_bytes_io(self): + f = self.BytesIO() + self.write_ops(f) + data = f.getvalue() + self.assertEqual(data, b"hello world\n") + f = self.BytesIO(data) + self.read_ops(f, True) + + def test_large_file_ops(self): + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: + support.requires( + 'largefile', + 'test requires %s bytes and a long time to run' % self.LARGE) + with self.open(os_helper.TESTFN, "w+b", 0) as f: + self.large_file_ops(f) + with self.open(os_helper.TESTFN, "w+b") as f: + self.large_file_ops(f) + + def test_with_open(self): + for bufsize in (0, 100): + with self.open(os_helper.TESTFN, "wb", bufsize) as f: + f.write(b"xxx") + self.assertEqual(f.closed, True) + try: + with self.open(os_helper.TESTFN, "wb", bufsize) as f: + 1/0 + except ZeroDivisionError: + self.assertEqual(f.closed, True) + else: + self.fail("1/0 didn't raise an exception") + + # issue 5008 + def test_append_mode_tell(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"xxx") + with self.open(os_helper.TESTFN, "ab", buffering=0) as f: + self.assertEqual(f.tell(), 3) + with self.open(os_helper.TESTFN, "ab") as f: + self.assertEqual(f.tell(), 3) + with self.open(os_helper.TESTFN, "a", encoding="utf-8") as f: + self.assertGreater(f.tell(), 0) + + def test_destructor(self): + record = [] + class MyFileIO(self.FileIO): + def __del__(self): + record.append(1) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(2) + super().close() + def flush(self): + record.append(3) + super().flush() + with warnings_helper.check_warnings(('', ResourceWarning)): + f = MyFileIO(os_helper.TESTFN, "wb") + f.write(b"xxx") + del f + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"xxx") + + def _check_base_destructor(self, base): + record = [] + class MyIO(base): + def __init__(self): + # This exercises the availability of attributes on object + # destruction. + # (in the C version, close() is called by the tp_dealloc + # function, not by __del__) + self.on_del = 1 + self.on_close = 2 + self.on_flush = 3 + def __del__(self): + record.append(self.on_del) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(self.on_close) + super().close() + def flush(self): + record.append(self.on_flush) + super().flush() + f = MyIO() + del f + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + + def test_IOBase_destructor(self): + self._check_base_destructor(self.IOBase) + + def test_RawIOBase_destructor(self): + self._check_base_destructor(self.RawIOBase) + + def test_BufferedIOBase_destructor(self): + self._check_base_destructor(self.BufferedIOBase) + + def test_TextIOBase_destructor(self): + self._check_base_destructor(self.TextIOBase) + + def test_close_flushes(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"xxx") + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"xxx") + + def test_array_writes(self): + a = array.array('i', range(10)) + n = len(a.tobytes()) + def check(f): + with f: + self.assertEqual(f.write(a), n) + f.writelines((a,)) + check(self.BytesIO()) + check(self.FileIO(os_helper.TESTFN, "w")) + check(self.BufferedWriter(self.MockRawIO())) + check(self.BufferedRandom(self.MockRawIO())) + check(self.BufferedRWPair(self.MockRawIO(), self.MockRawIO())) + + def test_closefd(self): + self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'w', + encoding="utf-8", closefd=False) + + def test_read_closed(self): + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + f.write("egg\n") + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) + self.assertEqual(file.read(), "egg\n") + file.seek(0) + file.close() + self.assertRaises(ValueError, file.read) + with self.open(os_helper.TESTFN, "rb") as f: + file = self.open(f.fileno(), "rb", closefd=False) + self.assertEqual(file.read()[:3], b"egg") + file.close() + self.assertRaises(ValueError, file.readinto, bytearray(1)) + + def test_no_closefd_with_filename(self): + # can't use closefd in combination with a file name + self.assertRaises(ValueError, self.open, os_helper.TESTFN, "r", + encoding="utf-8", closefd=False) + + def test_closefd_attr(self): + with self.open(os_helper.TESTFN, "wb") as f: + f.write(b"egg\n") + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + self.assertEqual(f.buffer.raw.closefd, True) + file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) + self.assertEqual(file.buffer.raw.closefd, False) + + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") + def test_garbage_collection(self): + # FileIO objects are collected, and collecting them flushes + # all data to disk. + # + # Note that using warnings_helper.check_warnings() will keep the + # file alive due to the `source` argument to warn(). So, use + # catch_warnings() instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + f = self.FileIO(os_helper.TESTFN, "wb") + f.write(b"abcxxx") + f.f = f + wr = weakref.ref(f) + del f + support.gc_collect() + self.assertIsNone(wr(), wr) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"abcxxx") + + def test_unbounded_file(self): + # Issue #1174606: reading from an unbounded stream such as /dev/zero. + zero = "/dev/zero" + if not os.path.exists(zero): + self.skipTest("{0} does not exist".format(zero)) + if sys.maxsize > 0x7FFFFFFF: + self.skipTest("test can only run in a 32-bit address space") + if support.real_max_memuse < support._2G: + self.skipTest("test requires at least 2 GiB of memory") + with self.open(zero, "rb", buffering=0) as f: + self.assertRaises(OverflowError, f.read) + with self.open(zero, "rb") as f: + self.assertRaises(OverflowError, f.read) + with self.open(zero, "r") as f: + self.assertRaises(OverflowError, f.read) + + def check_flush_error_on_close(self, *args, **kwargs): + # Test that the file is closed despite failed flush + # and that flush() is called before file closed. + f = self.open(*args, **kwargs) + closed = [] + def bad_flush(): + closed[:] = [f.closed] + raise OSError() + f.flush = bad_flush + self.assertRaises(OSError, f.close) # exception not swallowed + self.assertTrue(f.closed) + self.assertTrue(closed) # flush() called + self.assertFalse(closed[0]) # flush() called before file closed + f.flush = lambda: None # break reference loop + + def test_flush_error_on_close(self): + # raw file + # Issue #5700: io.FileIO calls flush() after file closed + self.check_flush_error_on_close(os_helper.TESTFN, 'wb', buffering=0) + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb', buffering=0) + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb', buffering=0, closefd=False) + os.close(fd) + # buffered io + self.check_flush_error_on_close(os_helper.TESTFN, 'wb') + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb') + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'wb', closefd=False) + os.close(fd) + # text io + self.check_flush_error_on_close(os_helper.TESTFN, 'w', encoding="utf-8") + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'w', encoding="utf-8") + fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT) + self.check_flush_error_on_close(fd, 'w', encoding="utf-8", closefd=False) + os.close(fd) + + def test_multi_close(self): + f = self.open(os_helper.TESTFN, "wb", buffering=0) + f.close() + f.close() + f.close() + self.assertRaises(ValueError, f.flush) + + def test_RawIOBase_read(self): + # Exercise the default limited RawIOBase.read(n) implementation (which + # calls readinto() internally). + rawio = self.MockRawIOWithoutRead((b"abc", b"d", None, b"efg", None)) + self.assertEqual(rawio.read(2), b"ab") + self.assertEqual(rawio.read(2), b"c") + self.assertEqual(rawio.read(2), b"d") + self.assertEqual(rawio.read(2), None) + self.assertEqual(rawio.read(2), b"ef") + self.assertEqual(rawio.read(2), b"g") + self.assertEqual(rawio.read(2), None) + self.assertEqual(rawio.read(2), b"") + + def test_RawIOBase_read_bounds_checking(self): + # Make sure a `.readinto` call which returns a value outside + # (0, len(buffer)) raises. + class Misbehaved(self.io.RawIOBase): + def __init__(self, readinto_return) -> None: + self._readinto_return = readinto_return + def readinto(self, b): + return self._readinto_return + + with self.assertRaises(ValueError) as cm: + Misbehaved(2).read(1) + self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1") + for bad_size in (2147483647, sys.maxsize, -1, -1000): + with self.assertRaises(ValueError): + Misbehaved(bad_size).read() + + def test_RawIOBase_read_gh60107(self): + # gh-60107: Ensure a "Raw I/O" which keeps a reference to the + # mutable memory doesn't allow making a mutable bytes. + class RawIOKeepsReference(self.MockRawIOWithoutRead): + def __init__(self, *args, **kwargs): + self.buf = None + super().__init__(*args, **kwargs) + + def readinto(self, buf): + # buf is the bytearray so keeping a reference to it doesn't keep + # the memory alive; a memoryview does. + self.buf = memoryview(buf) + buf[0:4] = self._read_stack.pop() + return 3 + + with self.assertRaises(BufferError): + rawio = RawIOKeepsReference([b"1234"]) + rawio.read(4) + + def test_types_have_dict(self): + test = ( + self.IOBase(), + self.RawIOBase(), + self.TextIOBase(), + self.StringIO(), + self.BytesIO() + ) + for obj in test: + self.assertHasAttr(obj, "__dict__") + + def test_opener(self): + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + f.write("egg\n") + fd = os.open(os_helper.TESTFN, os.O_RDONLY) + def opener(path, flags): + return fd + with self.open("non-existent", "r", encoding="utf-8", opener=opener) as f: + self.assertEqual(f.read(), "egg\n") + + def test_bad_opener_negative_1(self): + # Issue #27066. + def badopener(fname, flags): + return -1 + with self.assertRaises(ValueError) as cm: + self.open('non-existent', 'r', opener=badopener) + self.assertEqual(str(cm.exception), 'opener returned -1') + + def test_bad_opener_other_negative(self): + # Issue #27066. + def badopener(fname, flags): + return -2 + with self.assertRaises(ValueError) as cm: + self.open('non-existent', 'r', opener=badopener) + self.assertEqual(str(cm.exception), 'opener returned -2') + + def test_opener_invalid_fd(self): + # Check that OSError is raised with error code EBADF if the + # opener returns an invalid file descriptor (see gh-82212). + fd = os_helper.make_bad_fd() + with self.assertRaises(OSError) as cm: + self.open('foo', opener=lambda name, flags: fd) + self.assertEqual(cm.exception.errno, errno.EBADF) + + def test_fileio_closefd(self): + # Issue #4841 + with self.open(__file__, 'rb') as f1, \ + self.open(__file__, 'rb') as f2: + fileio = self.FileIO(f1.fileno(), closefd=False) + # .__init__() must not close f1 + fileio.__init__(f2.fileno(), closefd=False) + f1.readline() + # .close() must not close f2 + fileio.close() + f2.readline() + + def test_nonbuffered_textio(self): + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError): + self.open(os_helper.TESTFN, 'w', encoding="utf-8", buffering=0) + + def test_invalid_newline(self): + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError): + self.open(os_helper.TESTFN, 'w', encoding="utf-8", newline='invalid') + + def test_buffered_readinto_mixin(self): + # Test the implementation provided by BufferedIOBase + class Stream(self.BufferedIOBase): + def read(self, size): + return b"12345" + read1 = read + stream = Stream() + for method in ("readinto", "readinto1"): + with self.subTest(method): + buffer = byteslike(5) + self.assertEqual(getattr(stream, method)(buffer), 5) + self.assertEqual(bytes(buffer), b"12345") + + def test_fspath_support(self): + def check_path_succeeds(path): + with self.open(path, "w", encoding="utf-8") as f: + f.write("egg\n") + + with self.open(path, "r", encoding="utf-8") as f: + self.assertEqual(f.read(), "egg\n") + + check_path_succeeds(FakePath(os_helper.TESTFN)) + check_path_succeeds(FakePath(os.fsencode(os_helper.TESTFN))) + + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + bad_path = FakePath(f.fileno()) + with self.assertRaises(TypeError): + self.open(bad_path, 'w', encoding="utf-8") + + bad_path = FakePath(None) + with self.assertRaises(TypeError): + self.open(bad_path, 'w', encoding="utf-8") + + bad_path = FakePath(FloatingPointError) + with self.assertRaises(FloatingPointError): + self.open(bad_path, 'w', encoding="utf-8") + + # ensure that refcounting is correct with some error conditions + with self.assertRaisesRegex(ValueError, 'read/write/append mode'): + self.open(FakePath(os_helper.TESTFN), 'rwxa', encoding="utf-8") + + def test_RawIOBase_readall(self): + # Exercise the default unlimited RawIOBase.read() and readall() + # implementations. + rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) + self.assertEqual(rawio.read(), b"abcdefg") + rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg")) + self.assertEqual(rawio.readall(), b"abcdefg") + + def test_BufferedIOBase_readinto(self): + # Exercise the default BufferedIOBase.readinto() and readinto1() + # implementations (which call read() or read1() internally). + class Reader(self.BufferedIOBase): + def __init__(self, avail): + self.avail = avail + def read(self, size): + result = self.avail[:size] + self.avail = self.avail[size:] + return result + def read1(self, size): + """Returns no more than 5 bytes at once""" + return self.read(min(size, 5)) + tests = ( + # (test method, total data available, read buffer size, expected + # read size) + ("readinto", 10, 5, 5), + ("readinto", 10, 6, 6), # More than read1() can return + ("readinto", 5, 6, 5), # Buffer larger than total available + ("readinto", 6, 7, 6), + ("readinto", 10, 0, 0), # Empty buffer + ("readinto1", 10, 5, 5), # Result limited to single read1() call + ("readinto1", 10, 6, 5), # Buffer larger than read1() can return + ("readinto1", 5, 6, 5), # Buffer larger than total available + ("readinto1", 6, 7, 5), + ("readinto1", 10, 0, 0), # Empty buffer + ) + UNUSED_BYTE = 0x81 + for test in tests: + with self.subTest(test): + method, avail, request, result = test + reader = Reader(bytes(range(avail))) + buffer = bytearray((UNUSED_BYTE,) * request) + method = getattr(reader, method) + self.assertEqual(method(buffer), result) + self.assertEqual(len(buffer), request) + self.assertSequenceEqual(buffer[:result], range(result)) + unused = (UNUSED_BYTE,) * (request - result) + self.assertSequenceEqual(buffer[result:], unused) + self.assertEqual(len(reader.avail), avail - result) + + def test_close_assert(self): + class R(self.IOBase): + def __setattr__(self, name, value): + pass + def flush(self): + raise OSError() + f = R() + # This would cause an assertion failure. + self.assertRaises(OSError, f.close) + + # Silence destructor error + R.flush = lambda self: None + + @threading_helper.requires_working_threading() + def test_write_readline_races(self): + # gh-134908: Concurrent iteration over a file caused races + thread_count = 2 + write_count = 100 + read_count = 100 + + def writer(file, barrier): + barrier.wait() + for _ in range(write_count): + file.write("x") + + def reader(file, barrier): + barrier.wait() + for _ in range(read_count): + for line in file: + self.assertEqual(line, "") + + with self.open(os_helper.TESTFN, "w+") as f: + barrier = threading.Barrier(thread_count + 1) + reader = threading.Thread(target=reader, args=(f, barrier)) + writers = [threading.Thread(target=writer, args=(f, barrier)) + for _ in range(thread_count)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(writers + [reader]): + pass + self.assertIsNone(cm.exc_type) + + self.assertEqual(os.stat(os_helper.TESTFN).st_size, + write_count * thread_count) + + +class CIOTest(IOTest, CTestCase): + + def test_IOBase_finalize(self): + # Issue #12149: segmentation fault on _PyIOBase_finalize when both a + # class which inherits IOBase and an object of this class are caught + # in a reference cycle and close() is already in the method cache. + class MyIO(self.IOBase): + def close(self): + pass + + # create an instance to populate the method cache + MyIO() + obj = MyIO() + obj.obj = obj + wr = weakref.ref(obj) + del MyIO + del obj + support.gc_collect() + self.assertIsNone(wr(), wr) + +@support.cpython_only +class TestIOCTypes(unittest.TestCase): + def setUp(self): + _io = import_helper.import_module("_io") + self.types = [ + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + _io.FileIO, + _io.IncrementalNewlineDecoder, + _io.StringIO, + _io.TextIOWrapper, + _io._BufferedIOBase, + _io._BytesIOBuffer, + _io._IOBase, + _io._RawIOBase, + _io._TextIOBase, + ] + if sys.platform == "win32": + self.types.append(_io._WindowsConsoleIO) + self._io = _io + + def test_immutable_types(self): + for tp in self.types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = "bar" + + def test_class_hierarchy(self): + def check_subs(types, base): + for tp in types: + with self.subTest(tp=tp, base=base): + self.assertIsSubclass(tp, base) + + def recursive_check(d): + for k, v in d.items(): + if isinstance(v, dict): + recursive_check(v) + elif isinstance(v, set): + check_subs(v, k) + else: + self.fail("corrupt test dataset") + + _io = self._io + hierarchy = { + _io._IOBase: { + _io._BufferedIOBase: { + _io.BufferedRWPair, + _io.BufferedRandom, + _io.BufferedReader, + _io.BufferedWriter, + _io.BytesIO, + }, + _io._RawIOBase: { + _io.FileIO, + }, + _io._TextIOBase: { + _io.StringIO, + _io.TextIOWrapper, + }, + }, + } + if sys.platform == "win32": + hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO) + + recursive_check(hierarchy) + + def test_subclassing(self): + _io = self._io + dataset = {k: True for k in self.types} + dataset[_io._BytesIOBuffer] = False + + for tp, is_basetype in dataset.items(): + with self.subTest(tp=tp, is_basetype=is_basetype): + name = f"{tp.__name__}_subclass" + bases = (tp,) + if is_basetype: + _ = type(name, bases, {}) + else: + msg = "not an acceptable base type" + with self.assertRaisesRegex(TypeError, msg): + _ = type(name, bases, {}) + + def test_disallow_instantiation(self): + _io = self._io + support.check_disallow_instantiation(self, _io._BytesIOBuffer) + + def test_stringio_setstate(self): + # gh-127182: Calling __setstate__() with invalid arguments must not crash + obj = self._io.StringIO() + with self.assertRaisesRegex( + TypeError, + 'initial_value must be str or None, not int', + ): + obj.__setstate__((1, '', 0, {})) + + obj.__setstate__((None, '', 0, {})) # should not crash + self.assertEqual(obj.getvalue(), '') + + obj.__setstate__(('', '', 0, {})) + self.assertEqual(obj.getvalue(), '') + +class PyIOTest(IOTest, PyTestCase): + pass + + +@support.cpython_only +class APIMismatchTest(unittest.TestCase): + + def test_RawIOBase_io_in_pyio_match(self): + """Test that pyio RawIOBase class has all c RawIOBase methods""" + mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, + ignore=('__weakref__', '__static_attributes__')) + self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') + + def test_RawIOBase_pyio_in_io_match(self): + """Test that c RawIOBase class has all pyio RawIOBase methods""" + mismatch = support.detect_api_mismatch(io.RawIOBase, pyio.RawIOBase) + self.assertEqual(mismatch, set(), msg='C RawIOBase does not have all Python RawIOBase methods') + + +# XXX Tests for open() + +class MiscIOTest: + + # for test__all__, actual values are set in subclasses + name_of_module = None + extra_exported = () + not_exported = () + + def tearDown(self): + os_helper.unlink(os_helper.TESTFN) + + def test___all__(self): + support.check__all__(self, self.io, self.name_of_module, + extra=self.extra_exported, + not_exported=self.not_exported) + + def test_attributes(self): + f = self.open(os_helper.TESTFN, "wb", buffering=0) + self.assertEqual(f.mode, "wb") + f.close() + + f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") + self.assertEqual(f.mode, "w+") + self.assertEqual(f.buffer.mode, "wb+") + self.assertEqual(f.buffer.raw.mode, "wb+") + + g = self.open(f.fileno(), "wb", closefd=False) + self.assertEqual(g.mode, "wb") + self.assertEqual(g.raw.mode, "wb") + self.assertEqual(g.name, f.fileno()) + self.assertEqual(g.raw.name, f.fileno()) + f.close() + g.close() + + def test_removed_u_mode(self): + # bpo-37330: The "U" mode has been removed in Python 3.11 + for mode in ("U", "rU", "r+U"): + with self.assertRaises(ValueError) as cm: + self.open(os_helper.TESTFN, mode) + self.assertIn('invalid mode', str(cm.exception)) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_open_pipe_with_append(self): + # bpo-27805: Ignore ESPIPE from lseek() in open(). + r, w = os.pipe() + self.addCleanup(os.close, r) + f = self.open(w, 'a', encoding="utf-8") + self.addCleanup(f.close) + # Check that the file is marked non-seekable. On Windows, however, lseek + # somehow succeeds on pipes. + if sys.platform != 'win32': + self.assertFalse(f.seekable()) + + def test_io_after_close(self): + for kwargs in [ + {"mode": "w"}, + {"mode": "wb"}, + {"mode": "w", "buffering": 1}, + {"mode": "w", "buffering": 2}, + {"mode": "wb", "buffering": 0}, + {"mode": "r"}, + {"mode": "rb"}, + {"mode": "r", "buffering": 1}, + {"mode": "r", "buffering": 2}, + {"mode": "rb", "buffering": 0}, + {"mode": "w+"}, + {"mode": "w+b"}, + {"mode": "w+", "buffering": 1}, + {"mode": "w+", "buffering": 2}, + {"mode": "w+b", "buffering": 0}, + ]: + if "b" not in kwargs["mode"]: + kwargs["encoding"] = "utf-8" + f = self.open(os_helper.TESTFN, **kwargs) + f.close() + self.assertRaises(ValueError, f.flush) + self.assertRaises(ValueError, f.fileno) + self.assertRaises(ValueError, f.isatty) + self.assertRaises(ValueError, f.__iter__) + if hasattr(f, "peek"): + self.assertRaises(ValueError, f.peek, 1) + self.assertRaises(ValueError, f.read) + if hasattr(f, "read1"): + self.assertRaises(ValueError, f.read1, 1024) + self.assertRaises(ValueError, f.read1) + if hasattr(f, "readall"): + self.assertRaises(ValueError, f.readall) + if hasattr(f, "readinto"): + self.assertRaises(ValueError, f.readinto, bytearray(1024)) + if hasattr(f, "readinto1"): + self.assertRaises(ValueError, f.readinto1, bytearray(1024)) + self.assertRaises(ValueError, f.readline) + self.assertRaises(ValueError, f.readlines) + self.assertRaises(ValueError, f.readlines, 1) + self.assertRaises(ValueError, f.seek, 0) + self.assertRaises(ValueError, f.tell) + self.assertRaises(ValueError, f.truncate) + self.assertRaises(ValueError, f.write, + b"" if "b" in kwargs['mode'] else "") + self.assertRaises(ValueError, f.writelines, []) + self.assertRaises(ValueError, next, f) + + def test_blockingioerror(self): + # Various BlockingIOError issues + class C(str): + pass + c = C("") + b = self.BlockingIOError(1, c) + c.b = b + b.c = c + wr = weakref.ref(c) + del c, b + support.gc_collect() + self.assertIsNone(wr(), wr) + + def test_abcs(self): + # Test the visible base classes are ABCs. + self.assertIsInstance(self.IOBase, abc.ABCMeta) + self.assertIsInstance(self.RawIOBase, abc.ABCMeta) + self.assertIsInstance(self.BufferedIOBase, abc.ABCMeta) + self.assertIsInstance(self.TextIOBase, abc.ABCMeta) + + def _check_abc_inheritance(self, abcmodule): + with self.open(os_helper.TESTFN, "wb", buffering=0) as f: + self.assertIsInstance(f, abcmodule.IOBase) + self.assertIsInstance(f, abcmodule.RawIOBase) + self.assertNotIsInstance(f, abcmodule.BufferedIOBase) + self.assertNotIsInstance(f, abcmodule.TextIOBase) + with self.open(os_helper.TESTFN, "wb") as f: + self.assertIsInstance(f, abcmodule.IOBase) + self.assertNotIsInstance(f, abcmodule.RawIOBase) + self.assertIsInstance(f, abcmodule.BufferedIOBase) + self.assertNotIsInstance(f, abcmodule.TextIOBase) + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + self.assertIsInstance(f, abcmodule.IOBase) + self.assertNotIsInstance(f, abcmodule.RawIOBase) + self.assertNotIsInstance(f, abcmodule.BufferedIOBase) + self.assertIsInstance(f, abcmodule.TextIOBase) + + def test_abc_inheritance(self): + # Test implementations inherit from their respective ABCs + self._check_abc_inheritance(self) + + def test_abc_inheritance_official(self): + # Test implementations inherit from the official ABCs of the + # baseline "io" module. + self._check_abc_inheritance(io) + + def _check_warn_on_dealloc(self, *args, **kwargs): + f = self.open(*args, **kwargs) + r = repr(f) + with self.assertWarns(ResourceWarning) as cm: + f = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_warn_on_dealloc(self): + self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) + self._check_warn_on_dealloc(os_helper.TESTFN, "wb") + self._check_warn_on_dealloc(os_helper.TESTFN, "w", encoding="utf-8") + + def _check_warn_on_dealloc_fd(self, *args, **kwargs): + fds = [] + def cleanup_fds(): + for fd in fds: + try: + os.close(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + self.addCleanup(cleanup_fds) + r, w = os.pipe() + fds += r, w + self._check_warn_on_dealloc(r, *args, **kwargs) + # When using closefd=False, there's no warning + r, w = os.pipe() + fds += r, w + with warnings_helper.check_no_resource_warning(self): + self.open(r, *args, closefd=False, **kwargs) + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_warn_on_dealloc_fd(self): + self._check_warn_on_dealloc_fd("rb", buffering=0) + self._check_warn_on_dealloc_fd("rb") + self._check_warn_on_dealloc_fd("r", encoding="utf-8") + + + def test_pickling(self): + # Pickling file objects is forbidden + msg = "cannot pickle" + for kwargs in [ + {"mode": "w"}, + {"mode": "wb"}, + {"mode": "wb", "buffering": 0}, + {"mode": "r"}, + {"mode": "rb"}, + {"mode": "rb", "buffering": 0}, + {"mode": "w+"}, + {"mode": "w+b"}, + {"mode": "w+b", "buffering": 0}, + ]: + if "b" not in kwargs["mode"]: + kwargs["encoding"] = "utf-8" + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol, kwargs=kwargs): + with self.open(os_helper.TESTFN, **kwargs) as f: + with self.assertRaisesRegex(TypeError, msg): + pickle.dumps(f, protocol) + + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") + def test_nonblock_pipe_write_bigbuf(self): + self._test_nonblock_pipe_write(16*1024) + + @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd") + def test_nonblock_pipe_write_smallbuf(self): + self._test_nonblock_pipe_write(1024) + + @unittest.skipUnless(hasattr(os, 'set_blocking'), + 'os.set_blocking() required for this test') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def _test_nonblock_pipe_write(self, bufsize): + sent = [] + received = [] + r, w = os.pipe() + os.set_blocking(r, False) + os.set_blocking(w, False) + + # To exercise all code paths in the C implementation we need + # to play with buffer sizes. For instance, if we choose a + # buffer size less than or equal to _PIPE_BUF (4096 on Linux) + # then we will never get a partial write of the buffer. + rf = self.open(r, mode='rb', closefd=True, buffering=bufsize) + wf = self.open(w, mode='wb', closefd=True, buffering=bufsize) + + with rf, wf: + for N in 9999, 73, 7574: + try: + i = 0 + while True: + msg = bytes([i % 26 + 97]) * N + sent.append(msg) + wf.write(msg) + i += 1 + + except self.BlockingIOError as e: + self.assertEqual(e.args[0], errno.EAGAIN) + self.assertEqual(e.args[2], e.characters_written) + sent[-1] = sent[-1][:e.characters_written] + received.append(rf.read()) + msg = b'BLOCKED' + wf.write(msg) + sent.append(msg) + + while True: + try: + wf.flush() + break + except self.BlockingIOError as e: + self.assertEqual(e.args[0], errno.EAGAIN) + self.assertEqual(e.args[2], e.characters_written) + self.assertEqual(e.characters_written, 0) + received.append(rf.read()) + + received += iter(rf.read, None) + + sent, received = b''.join(sent), b''.join(received) + self.assertEqual(sent, received) + self.assertTrue(wf.closed) + self.assertTrue(rf.closed) + + def test_create_fail(self): + # 'x' mode fails if file is existing + with self.open(os_helper.TESTFN, 'w', encoding="utf-8"): + pass + self.assertRaises(FileExistsError, self.open, os_helper.TESTFN, 'x', encoding="utf-8") + + def test_create_writes(self): + # 'x' mode opens for writing + with self.open(os_helper.TESTFN, 'xb') as f: + f.write(b"spam") + with self.open(os_helper.TESTFN, 'rb') as f: + self.assertEqual(b"spam", f.read()) + + def test_open_allargs(self): + # there used to be a buffer overflow in the parser for rawmode + self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'rwax+', encoding="utf-8") + + def test_check_encoding_errors(self): + # bpo-37388: open() and TextIOWrapper must check encoding and errors + # arguments in dev mode + mod = self.io.__name__ + filename = __file__ + invalid = 'Boom, Shaka Laka, Boom!' + code = textwrap.dedent(f''' + import sys + from {mod} import open, TextIOWrapper + + try: + open({filename!r}, encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(21) + + try: + open({filename!r}, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(22) + + fp = open({filename!r}, "rb") + with fp: + try: + TextIOWrapper(fp, encoding={invalid!r}) + except LookupError: + pass + else: + sys.exit(23) + + try: + TextIOWrapper(fp, errors={invalid!r}) + except LookupError: + pass + else: + sys.exit(24) + + sys.exit(10) + ''') + proc = assert_python_failure('-X', 'dev', '-c', code) + self.assertEqual(proc.rc, 10, proc) + + def test_check_encoding_warning(self): + # PEP 597: Raise warning when encoding is not specified + # and sys.flags.warn_default_encoding is set. + mod = self.io.__name__ + filename = __file__ + code = textwrap.dedent(f'''\ + import sys + from {mod} import open, TextIOWrapper + import pathlib + + with open({filename!r}) as f: # line 5 + pass + + pathlib.Path({filename!r}).read_text() # line 8 + ''') + proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) + warnings = proc.err.splitlines() + self.assertEqual(len(warnings), 2) + self.assertStartsWith(warnings[0], b":5: EncodingWarning: ") + self.assertStartsWith(warnings[1], b":8: EncodingWarning: ") + + def test_text_encoding(self): + # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" + # based on sys.flags.utf8_mode + code = "import io; print(io.text_encoding(None))" + + proc = assert_python_ok('-X', 'utf8=0', '-c', code) + self.assertEqual(b"locale", proc.out.strip()) + + proc = assert_python_ok('-X', 'utf8=1', '-c', code) + self.assertEqual(b"utf-8", proc.out.strip()) + + +class CMiscIOTest(MiscIOTest, CTestCase): + name_of_module = "io", "_io" + extra_exported = "BlockingIOError", + + def test_readinto_buffer_overflow(self): + # Issue #18025 + class BadReader(self.io.BufferedIOBase): + def read(self, n=-1): + return b'x' * 10**6 + bufio = BadReader() + b = bytearray(2) + self.assertRaises(ValueError, bufio.readinto, b) + + def check_daemon_threads_shutdown_deadlock(self, stream_name): + # Issue #23309: deadlocks at shutdown should be avoided when a + # daemon thread and the main thread both write to a file. + code = """if 1: + import sys + import time + import threading + from test.support import SuppressCrashReport + + file = sys.{stream_name} + + def run(): + while True: + file.write('.') + file.flush() + + crash = SuppressCrashReport() + crash.__enter__() + # don't call __exit__(): the crash occurs at Python shutdown + + thread = threading.Thread(target=run) + thread.daemon = True + thread.start() + + time.sleep(0.5) + file.write('!') + file.flush() + """.format_map(locals()) + res, _ = run_python_until_end("-c", code) + err = res.err.decode() + if res.rc != 0: + # Failure: should be a fatal error + pattern = (r"Fatal Python error: _enter_buffered_busy: " + r"could not acquire lock " + r"for <(_io\.)?BufferedWriter name='<{stream_name}>'> " + r"at interpreter shutdown, possibly due to " + r"daemon threads".format_map(locals())) + self.assertRegex(err, pattern) + else: + self.assertFalse(err.strip('.!')) + + @threading_helper.requires_working_threading() + @support.requires_resource('walltime') + def test_daemon_threads_shutdown_stdout_deadlock(self): + self.check_daemon_threads_shutdown_deadlock('stdout') + + @threading_helper.requires_working_threading() + @support.requires_resource('walltime') + def test_daemon_threads_shutdown_stderr_deadlock(self): + self.check_daemon_threads_shutdown_deadlock('stderr') + + +class PyMiscIOTest(MiscIOTest, PyTestCase): + name_of_module = "_pyio", "io" + extra_exported = "BlockingIOError", "open_code", + not_exported = "valid_seek_flags", + + +class ProtocolsTest(unittest.TestCase): + class MyReader: + def read(self, sz=-1): + return b"" + + class MyWriter: + def write(self, b: bytes): + pass + + def test_reader_subclass(self): + self.assertIsSubclass(self.MyReader, io.Reader) + self.assertNotIsSubclass(str, io.Reader) + + def test_writer_subclass(self): + self.assertIsSubclass(self.MyWriter, io.Writer) + self.assertNotIsSubclass(str, io.Writer) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_largefile.py b/Lib/test/test_io/test_largefile.py similarity index 98% rename from Lib/test/test_largefile.py rename to Lib/test/test_io/test_largefile.py index 41f7b70e5cf..438a90a92ed 100644 --- a/Lib/test/test_largefile.py +++ b/Lib/test/test_io/test_largefile.py @@ -56,9 +56,7 @@ class TestFileMethods(LargeFileTest): (i.e. > 2 GiB) files. """ - # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, - # so memuse=2 is needed - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: @@ -154,7 +152,7 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_seek_readall(self, _size): # Seek which doesn't change position should readall successfully. with self.open(TESTFN, 'rb') as f: diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_io/test_memoryio.py similarity index 100% rename from Lib/test/test_memoryio.py rename to Lib/test/test_io/test_memoryio.py diff --git a/Lib/test/test_io/test_signals.py b/Lib/test/test_io/test_signals.py new file mode 100644 index 00000000000..03f1da1eb1c --- /dev/null +++ b/Lib/test/test_io/test_signals.py @@ -0,0 +1,280 @@ +import errno +import os +import signal +import threading +import unittest +from test import support +from test.support import import_helper +from .utils import PyTestCase, CTestCase + + +requires_alarm = unittest.skipUnless( + hasattr(signal, "alarm"), "test requires signal.alarm()" +) + + +@unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.') +class SignalsTest: + + def setUp(self): + self.oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt) + + def tearDown(self): + signal.signal(signal.SIGALRM, self.oldalrm) + + def alarm_interrupt(self, sig, frame): + 1/0 + + def check_interrupted_write(self, item, bytes, **fdopen_kwargs): + """Check that a partial write, when it gets interrupted, properly + invokes the signal handler, and bubbles up the exception raised + in the latter.""" + + # XXX This test has three flaws that appear when objects are + # XXX not reference counted. + + # - if wio.write() happens to trigger a garbage collection, + # the signal exception may be raised when some __del__ + # method is running; it will not reach the assertRaises() + # call. + + # - more subtle, if the wio object is not destroyed at once + # and survives this function, the next opened file is likely + # to have the same fileno (since the file descriptor was + # actively closed). When wio.__del__ is finally called, it + # will close the other's test file... To trigger this with + # CPython, try adding "global wio" in this function. + + # - This happens only for streams created by the _pyio module, + # because a wio.close() that fails still consider that the + # file needs to be closed again. You can try adding an + # "assert wio.closed" at the end of the function. + + # Fortunately, a little gc.collect() seems to be enough to + # work around all these issues. + support.gc_collect() # For PyPy or other GCs. + + read_results = [] + def _read(): + s = os.read(r, 1) + read_results.append(s) + + t = threading.Thread(target=_read) + t.daemon = True + r, w = os.pipe() + fdopen_kwargs["closefd"] = False + large_data = item * (support.PIPE_MAX_SIZE // len(item) + 1) + try: + wio = self.io.open(w, **fdopen_kwargs) + if hasattr(signal, 'pthread_sigmask'): + # create the thread with SIGALRM signal blocked + signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) + t.start() + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGALRM]) + else: + t.start() + + # Fill the pipe enough that the write will be blocking. + # It will be interrupted by the timer armed above. Since the + # other thread has read one byte, the low-level write will + # return with a successful (partial) result rather than an EINTR. + # The buffered IO layer must check for pending signal + # handlers, which in this case will invoke alarm_interrupt(). + signal.alarm(1) + try: + self.assertRaises(ZeroDivisionError, wio.write, large_data) + finally: + signal.alarm(0) + t.join() + # We got one byte, get another one and check that it isn't a + # repeat of the first one. + read_results.append(os.read(r, 1)) + self.assertEqual(read_results, [bytes[0:1], bytes[1:2]]) + finally: + os.close(w) + os.close(r) + # This is deliberate. If we didn't close the file descriptor + # before closing wio, wio would try to flush its internal + # buffer, and block again. + try: + wio.close() + except OSError as e: + if e.errno != errno.EBADF: + raise + + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_interrupted_write_unbuffered(self): + self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0) + + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_interrupted_write_buffered(self): + self.check_interrupted_write(b"xy", b"xy", mode="wb") + + @requires_alarm + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_interrupted_write_text(self): + self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") + + @support.no_tracing + def check_reentrant_write(self, data, **fdopen_kwargs): + def on_alarm(*args): + # Will be called reentrantly from the same thread + wio.write(data) + 1/0 + signal.signal(signal.SIGALRM, on_alarm) + r, w = os.pipe() + wio = self.io.open(w, **fdopen_kwargs) + try: + signal.alarm(1) + # Either the reentrant call to wio.write() fails with RuntimeError, + # or the signal handler raises ZeroDivisionError. + with self.assertRaises((ZeroDivisionError, RuntimeError)) as cm: + while 1: + for i in range(100): + wio.write(data) + wio.flush() + # Make sure the buffer doesn't fill up and block further writes + os.read(r, len(data) * 100) + exc = cm.exception + if isinstance(exc, RuntimeError): + self.assertStartsWith(str(exc), "reentrant call") + finally: + signal.alarm(0) + wio.close() + os.close(r) + + @requires_alarm + def test_reentrant_write_buffered(self): + self.check_reentrant_write(b"xy", mode="wb") + + @requires_alarm + def test_reentrant_write_text(self): + self.check_reentrant_write("xy", mode="w", encoding="ascii") + + def check_interrupted_read_retry(self, decode, **fdopen_kwargs): + """Check that a buffered read, when it gets interrupted (either + returning a partial result or EINTR), properly invokes the signal + handler and retries if the latter returned successfully.""" + r, w = os.pipe() + fdopen_kwargs["closefd"] = False + def alarm_handler(sig, frame): + os.write(w, b"bar") + signal.signal(signal.SIGALRM, alarm_handler) + try: + rio = self.io.open(r, **fdopen_kwargs) + os.write(w, b"foo") + signal.alarm(1) + # Expected behaviour: + # - first raw read() returns partial b"foo" + # - second raw read() returns EINTR + # - third raw read() returns b"bar" + self.assertEqual(decode(rio.read(6)), "foobar") + finally: + signal.alarm(0) + rio.close() + os.close(w) + os.close(r) + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_read_retry_buffered(self): + self.check_interrupted_read_retry(lambda x: x.decode('latin1'), + mode="rb") + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_read_retry_text(self): + self.check_interrupted_read_retry(lambda x: x, + mode="r", encoding="latin1") + + def check_interrupted_write_retry(self, item, **fdopen_kwargs): + """Check that a buffered write, when it gets interrupted (either + returning a partial result or EINTR), properly invokes the signal + handler and retries if the latter returned successfully.""" + select = import_helper.import_module("select") + + # A quantity that exceeds the buffer size of an anonymous pipe's + # write end. + N = support.PIPE_MAX_SIZE + r, w = os.pipe() + fdopen_kwargs["closefd"] = False + + # We need a separate thread to read from the pipe and allow the + # write() to finish. This thread is started after the SIGALRM is + # received (forcing a first EINTR in write()). + read_results = [] + write_finished = False + error = None + def _read(): + try: + while not write_finished: + while r in select.select([r], [], [], 1.0)[0]: + s = os.read(r, 1024) + read_results.append(s) + except BaseException as exc: + nonlocal error + error = exc + t = threading.Thread(target=_read) + t.daemon = True + def alarm1(sig, frame): + signal.signal(signal.SIGALRM, alarm2) + signal.alarm(1) + def alarm2(sig, frame): + t.start() + + large_data = item * N + signal.signal(signal.SIGALRM, alarm1) + try: + wio = self.io.open(w, **fdopen_kwargs) + signal.alarm(1) + # Expected behaviour: + # - first raw write() is partial (because of the limited pipe buffer + # and the first alarm) + # - second raw write() returns EINTR (because of the second alarm) + # - subsequent write()s are successful (either partial or complete) + written = wio.write(large_data) + self.assertEqual(N, written) + + wio.flush() + write_finished = True + t.join() + + self.assertIsNone(error) + self.assertEqual(N, sum(len(x) for x in read_results)) + finally: + signal.alarm(0) + write_finished = True + os.close(w) + os.close(r) + # This is deliberate. If we didn't close the file descriptor + # before closing wio, wio would try to flush its internal + # buffer, and could block (in case of failure). + try: + wio.close() + except OSError as e: + if e.errno != errno.EBADF: + raise + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_write_retry_buffered(self): + self.check_interrupted_write_retry(b"x", mode="wb") + + @requires_alarm + @support.requires_resource('walltime') + def test_interrupted_write_retry_text(self): + self.check_interrupted_write_retry("x", mode="w", encoding="latin1") + + +class CSignalsTest(SignalsTest, CTestCase): + pass + +class PySignalsTest(SignalsTest, PyTestCase): + pass + + # Handling reentrancy issues would slow down _pyio even more, so the + # tests are disabled. + test_reentrant_write_buffered = None + test_reentrant_write_text = None diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py new file mode 100644 index 00000000000..dfa77ea23b3 --- /dev/null +++ b/Lib/test/test_io/test_textio.py @@ -0,0 +1,1704 @@ +import array +import codecs +import locale +import os +import pickle +import sys +import threading +import time +import unittest +import warnings +import weakref +from collections import UserList +from test import support +from test.support import os_helper, threading_helper +from test.support.script_helper import assert_python_ok +from .utils import CTestCase, PyTestCase + +import io # C implementation of io +import _pyio as pyio # Python implementation of io + + +def _default_chunk_size(): + """Get the default TextIOWrapper chunk size""" + with open(__file__, "r", encoding="latin-1") as f: + return f._CHUNK_SIZE + + +class BadIndex: + def __index__(self): + 1/0 + + +# To fully exercise seek/tell, the StatefulIncrementalDecoder has these +# properties: +# - A single output character can correspond to many bytes of input. +# - The number of input bytes to complete the character can be +# undetermined until the last input byte is received. +# - The number of input bytes can vary depending on previous input. +# - A single input byte can correspond to many characters of output. +# - The number of output characters can be undetermined until the +# last input byte is received. +# - The number of output characters can vary depending on previous input. + +class StatefulIncrementalDecoder(codecs.IncrementalDecoder): + """ + For testing seek/tell behavior with a stateful, buffering decoder. + + Input is a sequence of words. Words may be fixed-length (length set + by input) or variable-length (period-terminated). In variable-length + mode, extra periods are ignored. Possible words are: + - 'i' followed by a number sets the input length, I (maximum 99). + When I is set to 0, words are space-terminated. + - 'o' followed by a number sets the output length, O (maximum 99). + - Any other word is converted into a word followed by a period on + the output. The output word consists of the input word truncated + or padded out with hyphens to make its length equal to O. If O + is 0, the word is output verbatim without truncating or padding. + I and O are initially set to 1. When I changes, any buffered input is + re-scanned according to the new I. EOF also terminates the last word. + """ + + def __init__(self, errors='strict'): + codecs.IncrementalDecoder.__init__(self, errors) + self.reset() + + def __repr__(self): + return '' % id(self) + + def reset(self): + self.i = 1 + self.o = 1 + self.buffer = bytearray() + + def getstate(self): + i, o = self.i ^ 1, self.o ^ 1 # so that flags = 0 after reset() + return bytes(self.buffer), i*100 + o + + def setstate(self, state): + buffer, io = state + self.buffer = bytearray(buffer) + i, o = divmod(io, 100) + self.i, self.o = i ^ 1, o ^ 1 + + def decode(self, input, final=False): + output = '' + for b in input: + if self.i == 0: # variable-length, terminated with period + if b == ord('.'): + if self.buffer: + output += self.process_word() + else: + self.buffer.append(b) + else: # fixed-length, terminate after self.i bytes + self.buffer.append(b) + if len(self.buffer) == self.i: + output += self.process_word() + if final and self.buffer: # EOF terminates the last word + output += self.process_word() + return output + + def process_word(self): + output = '' + if self.buffer[0] == ord('i'): + self.i = min(99, int(self.buffer[1:] or 0)) # set input length + elif self.buffer[0] == ord('o'): + self.o = min(99, int(self.buffer[1:] or 0)) # set output length + else: + output = self.buffer.decode('ascii') + if len(output) < self.o: + output += '-'*self.o # pad out with hyphens + if self.o: + output = output[:self.o] # truncate to output length + output += '.' + self.buffer = bytearray() + return output + + codecEnabled = False + + +# bpo-41919: This method is separated from StatefulIncrementalDecoder to avoid a resource leak +# when registering codecs and cleanup functions. +def lookupTestDecoder(name): + if StatefulIncrementalDecoder.codecEnabled and name == 'test_decoder': + latin1 = codecs.lookup('latin-1') + return codecs.CodecInfo( + name='test_decoder', encode=latin1.encode, decode=None, + incrementalencoder=None, + streamreader=None, streamwriter=None, + incrementaldecoder=StatefulIncrementalDecoder) + + +class StatefulIncrementalDecoderTest(unittest.TestCase): + """ + Make sure the StatefulIncrementalDecoder actually works. + """ + + test_cases = [ + # I=1, O=1 (fixed-length input == fixed-length output) + (b'abcd', False, 'a.b.c.d.'), + # I=0, O=0 (variable-length input, variable-length output) + (b'oiabcd', True, 'abcd.'), + # I=0, O=0 (should ignore extra periods) + (b'oi...abcd...', True, 'abcd.'), + # I=0, O=6 (variable-length input, fixed-length output) + (b'i.o6.x.xyz.toolongtofit.', False, 'x-----.xyz---.toolon.'), + # I=2, O=6 (fixed-length input < fixed-length output) + (b'i.i2.o6xyz', True, 'xy----.z-----.'), + # I=6, O=3 (fixed-length input > fixed-length output) + (b'i.o3.i6.abcdefghijklmnop', True, 'abc.ghi.mno.'), + # I=0, then 3; O=29, then 15 (with longer output) + (b'i.o29.a.b.cde.o15.abcdefghijabcdefghij.i3.a.b.c.d.ei00k.l.m', True, + 'a----------------------------.' + + 'b----------------------------.' + + 'cde--------------------------.' + + 'abcdefghijabcde.' + + 'a.b------------.' + + '.c.------------.' + + 'd.e------------.' + + 'k--------------.' + + 'l--------------.' + + 'm--------------.') + ] + + def test_decoder(self): + # Try a few one-shot test cases. + for input, eof, output in self.test_cases: + d = StatefulIncrementalDecoder() + self.assertEqual(d.decode(input, eof), output) + + # Also test an unfinished decode, followed by forcing EOF. + d = StatefulIncrementalDecoder() + self.assertEqual(d.decode(b'oiabcd'), '') + self.assertEqual(d.decode(b'', 1), 'abcd.') + +class TextIOWrapperTest: + + def setUp(self): + self.testdata = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n" + self.normalized = b"AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii") + os_helper.unlink(os_helper.TESTFN) + codecs.register(lookupTestDecoder) + self.addCleanup(codecs.unregister, lookupTestDecoder) + + def tearDown(self): + os_helper.unlink(os_helper.TESTFN) + + def test_constructor(self): + r = self.BytesIO(b"\xc3\xa9\n\n") + b = self.BufferedReader(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8") + t.__init__(b, encoding="latin-1", newline="\r\n") + self.assertEqual(t.encoding, "latin-1") + self.assertEqual(t.line_buffering, False) + t.__init__(b, encoding="utf-8", line_buffering=True) + self.assertEqual(t.encoding, "utf-8") + self.assertEqual(t.line_buffering, True) + self.assertEqual("\xe9\n", t.readline()) + invalid_type = TypeError if self.is_C else ValueError + with self.assertRaises(invalid_type): + t.__init__(b, encoding=42) + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding='utf-8\0') + with self.assertRaises(invalid_type): + t.__init__(b, encoding="utf-8", errors=42) + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(UnicodeEncodeError): + t.__init__(b, encoding="utf-8", errors='\udcfe') + if support.Py_DEBUG or sys.flags.dev_mode or self.is_C: + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", errors='replace\0') + with self.assertRaises(TypeError): + t.__init__(b, encoding="utf-8", newline=42) + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\udcfe') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='\n\0') + with self.assertRaises(ValueError): + t.__init__(b, encoding="utf-8", newline='xyzzy') + + def test_uninitialized(self): + t = self.TextIOWrapper.__new__(self.TextIOWrapper) + del t + t = self.TextIOWrapper.__new__(self.TextIOWrapper) + self.assertRaises(Exception, repr, t) + self.assertRaisesRegex((ValueError, AttributeError), + 'uninitialized|has no attribute', + t.read, 0) + t.__init__(self.MockRawIO(), encoding="utf-8") + self.assertEqual(t.read(0), '') + + def test_non_text_encoding_codecs_are_rejected(self): + # Ensure the constructor complains if passed a codec that isn't + # marked as a text encoding + # http://bugs.python.org/issue20404 + r = self.BytesIO() + b = self.BufferedWriter(r) + with self.assertRaisesRegex(LookupError, "is not a text encoding"): + self.TextIOWrapper(b, encoding="hex") + + def test_detach(self): + r = self.BytesIO() + b = self.BufferedWriter(r) + t = self.TextIOWrapper(b, encoding="ascii") + self.assertIs(t.detach(), b) + + t = self.TextIOWrapper(b, encoding="ascii") + t.write("howdy") + self.assertFalse(r.getvalue()) + t.detach() + self.assertEqual(r.getvalue(), b"howdy") + self.assertRaises(ValueError, t.detach) + + # Operations independent of the detached stream should still work + repr(t) + self.assertEqual(t.encoding, "ascii") + self.assertEqual(t.errors, "strict") + self.assertFalse(t.line_buffering) + self.assertFalse(t.write_through) + + def test_repr(self): + raw = self.BytesIO("hello".encode("utf-8")) + b = self.BufferedReader(raw) + t = self.TextIOWrapper(b, encoding="utf-8") + modname = self.TextIOWrapper.__module__ + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper encoding='utf-8'>" % modname) + raw.name = "dummy" + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper name='dummy' encoding='utf-8'>" % modname) + t.mode = "r" + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper name='dummy' mode='r' encoding='utf-8'>" % modname) + raw.name = b"dummy" + self.assertRegex(repr(t), + r"<(%s\.)?TextIOWrapper name=b'dummy' mode='r' encoding='utf-8'>" % modname) + + t.buffer.detach() + repr(t) # Should not raise an exception + + def test_recursive_repr(self): + # Issue #25455 + raw = self.BytesIO() + t = self.TextIOWrapper(raw, encoding="utf-8") + with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): + repr(t) # Should not crash + + def test_subclass_repr(self): + class TestSubclass(self.TextIOWrapper): + pass + + f = TestSubclass(self.StringIO()) + self.assertIn(TestSubclass.__name__, repr(f)) + + def test_line_buffering(self): + r = self.BytesIO() + b = self.BufferedWriter(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=True) + t.write("X") + self.assertEqual(r.getvalue(), b"") # No flush happened + t.write("Y\nZ") + self.assertEqual(r.getvalue(), b"XY\nZ") # All got flushed + t.write("A\rB") + self.assertEqual(r.getvalue(), b"XY\nZA\rB") + + def test_reconfigure_line_buffering(self): + r = self.BytesIO() + b = self.BufferedWriter(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=False) + t.write("AB\nC") + self.assertEqual(r.getvalue(), b"") + + t.reconfigure(line_buffering=True) # implicit flush + self.assertEqual(r.getvalue(), b"AB\nC") + t.write("DEF\nG") + self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") + t.write("H") + self.assertEqual(r.getvalue(), b"AB\nCDEF\nG") + t.reconfigure(line_buffering=False) # implicit flush + self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") + t.write("IJ") + self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH") + + # Keeping default value + t.reconfigure() + t.reconfigure(line_buffering=None) + self.assertEqual(t.line_buffering, False) + t.reconfigure(line_buffering=True) + t.reconfigure() + t.reconfigure(line_buffering=None) + self.assertEqual(t.line_buffering, True) + + @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") + def test_default_encoding(self): + with os_helper.EnvironmentVarGuard() as env: + # try to get a user preferred encoding different than the current + # locale encoding to check that TextIOWrapper() uses the current + # locale encoding and not the user preferred encoding + env.unset('LC_ALL', 'LANG', 'LC_CTYPE') + + current_locale_encoding = locale.getencoding() + b = self.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", EncodingWarning) + t = self.TextIOWrapper(b) + self.assertEqual(t.encoding, current_locale_encoding) + + def test_encoding(self): + # Check the encoding attribute is always set, and valid + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="utf-8") + self.assertEqual(t.encoding, "utf-8") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", EncodingWarning) + t = self.TextIOWrapper(b) + self.assertIsNotNone(t.encoding) + codecs.lookup(t.encoding) + + def test_encoding_errors_reading(self): + # (1) default + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii") + self.assertRaises(UnicodeError, t.read) + # (2) explicit strict + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii", errors="strict") + self.assertRaises(UnicodeError, t.read) + # (3) ignore + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii", errors="ignore") + self.assertEqual(t.read(), "abc\n\n") + # (4) replace + b = self.BytesIO(b"abc\n\xff\n") + t = self.TextIOWrapper(b, encoding="ascii", errors="replace") + self.assertEqual(t.read(), "abc\n\ufffd\n") + + def test_encoding_errors_writing(self): + # (1) default + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii") + self.assertRaises(UnicodeError, t.write, "\xff") + # (2) explicit strict + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii", errors="strict") + self.assertRaises(UnicodeError, t.write, "\xff") + # (3) ignore + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii", errors="ignore", + newline="\n") + t.write("abc\xffdef\n") + t.flush() + self.assertEqual(b.getvalue(), b"abcdef\n") + # (4) replace + b = self.BytesIO() + t = self.TextIOWrapper(b, encoding="ascii", errors="replace", + newline="\n") + t.write("abc\xffdef\n") + t.flush() + self.assertEqual(b.getvalue(), b"abc?def\n") + + def test_newlines(self): + input_lines = [ "unix\n", "windows\r\n", "os9\r", "last\n", "nonl" ] + + tests = [ + [ None, [ 'unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl' ] ], + [ '', input_lines ], + [ '\n', [ "unix\n", "windows\r\n", "os9\rlast\n", "nonl" ] ], + [ '\r\n', [ "unix\nwindows\r\n", "os9\rlast\nnonl" ] ], + [ '\r', [ "unix\nwindows\r", "\nos9\r", "last\nnonl" ] ], + ] + encodings = ( + 'utf-8', 'latin-1', + 'utf-16', 'utf-16-le', 'utf-16-be', + 'utf-32', 'utf-32-le', 'utf-32-be', + ) + + # Try a range of buffer sizes to test the case where \r is the last + # character in TextIOWrapper._pending_line. + for encoding in encodings: + # XXX: str.encode() should return bytes + data = bytes(''.join(input_lines).encode(encoding)) + for do_reads in (False, True): + for bufsize in range(1, 10): + for newline, exp_lines in tests: + bufio = self.BufferedReader(self.BytesIO(data), bufsize) + textio = self.TextIOWrapper(bufio, newline=newline, + encoding=encoding) + if do_reads: + got_lines = [] + while True: + c2 = textio.read(2) + if c2 == '': + break + self.assertEqual(len(c2), 2) + got_lines.append(c2 + textio.readline()) + else: + got_lines = list(textio) + + for got_line, exp_line in zip(got_lines, exp_lines): + self.assertEqual(got_line, exp_line) + self.assertEqual(len(got_lines), len(exp_lines)) + + def test_newlines_input(self): + testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" + normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + for newline, expected in [ + (None, normalized.decode("ascii").splitlines(keepends=True)), + ("", testdata.decode("ascii").splitlines(keepends=True)), + ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), + ]: + buf = self.BytesIO(testdata) + txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) + self.assertEqual(txt.readlines(), expected) + txt.seek(0) + self.assertEqual(txt.read(), "".join(expected)) + + def test_newlines_output(self): + testdict = { + "": b"AAA\nBBB\nCCC\nX\rY\r\nZ", + "\n": b"AAA\nBBB\nCCC\nX\rY\r\nZ", + "\r": b"AAA\rBBB\rCCC\rX\rY\r\rZ", + "\r\n": b"AAA\r\nBBB\r\nCCC\r\nX\rY\r\r\nZ", + } + tests = [(None, testdict[os.linesep])] + sorted(testdict.items()) + for newline, expected in tests: + buf = self.BytesIO() + txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline) + txt.write("AAA\nB") + txt.write("BB\nCCC\n") + txt.write("X\rY\r\nZ") + txt.flush() + self.assertEqual(buf.closed, False) + self.assertEqual(buf.getvalue(), expected) + + def test_destructor(self): + l = [] + base = self.BytesIO + class MyBytesIO(base): + def close(self): + l.append(self.getvalue()) + base.close(self) + b = MyBytesIO() + t = self.TextIOWrapper(b, encoding="ascii") + t.write("abc") + del t + support.gc_collect() + self.assertEqual([b"abc"], l) + + def test_override_destructor(self): + record = [] + class MyTextIO(self.TextIOWrapper): + def __del__(self): + record.append(1) + try: + f = super().__del__ + except AttributeError: + pass + else: + f() + def close(self): + record.append(2) + super().close() + def flush(self): + record.append(3) + super().flush() + b = self.BytesIO() + t = MyTextIO(b, encoding="ascii") + del t + support.gc_collect() + self.assertEqual(record, [1, 2, 3]) + + def test_error_through_destructor(self): + # Test that the exception state is not modified by a destructor, + # even if close() fails. + rawio = self.CloseFailureIO() + with support.catch_unraisable_exception() as cm: + with self.assertRaises(AttributeError): + self.TextIOWrapper(rawio, encoding="utf-8").xyzzy + + self.assertEqual(cm.unraisable.exc_type, OSError) + + # Systematic tests of the text I/O API + + def test_basic_io(self): + for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65): + for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le": + f = self.open(os_helper.TESTFN, "w+", encoding=enc) + f._CHUNK_SIZE = chunksize + self.assertEqual(f.write("abc"), 3) + f.close() + f = self.open(os_helper.TESTFN, "r+", encoding=enc) + f._CHUNK_SIZE = chunksize + self.assertEqual(f.tell(), 0) + self.assertEqual(f.read(), "abc") + cookie = f.tell() + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.read(None), "abc") + f.seek(0) + self.assertEqual(f.read(2), "ab") + self.assertEqual(f.read(1), "c") + self.assertEqual(f.read(1), "") + self.assertEqual(f.read(), "") + self.assertEqual(f.tell(), cookie) + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.seek(0, 2), cookie) + self.assertEqual(f.write("def"), 3) + self.assertEqual(f.seek(cookie), cookie) + self.assertEqual(f.read(), "def") + if enc.startswith("utf"): + self.multi_line_test(f, enc) + f.close() + + def multi_line_test(self, f, enc): + f.seek(0) + f.truncate() + sample = "s\xff\u0fff\uffff" + wlines = [] + for size in (0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 62, 63, 64, 65, 1000): + chars = [] + for i in range(size): + chars.append(sample[i % len(sample)]) + line = "".join(chars) + "\n" + wlines.append((f.tell(), line)) + f.write(line) + f.seek(0) + rlines = [] + while True: + pos = f.tell() + line = f.readline() + if not line: + break + rlines.append((pos, line)) + self.assertEqual(rlines, wlines) + + def test_telling(self): + f = self.open(os_helper.TESTFN, "w+", encoding="utf-8") + p0 = f.tell() + f.write("\xff\n") + p1 = f.tell() + f.write("\xff\n") + p2 = f.tell() + f.seek(0) + self.assertEqual(f.tell(), p0) + self.assertEqual(f.readline(), "\xff\n") + self.assertEqual(f.tell(), p1) + self.assertEqual(f.readline(), "\xff\n") + self.assertEqual(f.tell(), p2) + f.seek(0) + for line in f: + self.assertEqual(line, "\xff\n") + self.assertRaises(OSError, f.tell) + self.assertEqual(f.tell(), p2) + f.close() + + def test_seeking(self): + chunk_size = _default_chunk_size() + prefix_size = chunk_size - 2 + u_prefix = "a" * prefix_size + prefix = bytes(u_prefix.encode("utf-8")) + self.assertEqual(len(u_prefix), len(prefix)) + u_suffix = "\u8888\n" + suffix = bytes(u_suffix.encode("utf-8")) + line = prefix + suffix + with self.open(os_helper.TESTFN, "wb") as f: + f.write(line*2) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + s = f.read(prefix_size) + self.assertEqual(s, str(prefix, "ascii")) + self.assertEqual(f.tell(), prefix_size) + self.assertEqual(f.readline(), u_suffix) + + def test_seeking_too(self): + # Regression test for a specific bug + data = b'\xe0\xbf\xbf\n' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f: + f._CHUNK_SIZE # Just test that it exists + f._CHUNK_SIZE = 2 + f.readline() + f.tell() + + def test_seek_and_tell(self): + #Test seek/tell using the StatefulIncrementalDecoder. + # Make test faster by doing smaller seeks + CHUNK_SIZE = 128 + + def test_seek_and_tell_with_data(data, min_pos=0): + """Tell/seek to various points within a data stream and ensure + that the decoded data returned by read() is consistent.""" + f = self.open(os_helper.TESTFN, 'wb') + f.write(data) + f.close() + f = self.open(os_helper.TESTFN, encoding='test_decoder') + f._CHUNK_SIZE = CHUNK_SIZE + decoded = f.read() + f.close() + + for i in range(min_pos, len(decoded) + 1): # seek positions + for j in [1, 5, len(decoded) - i]: # read lengths + f = self.open(os_helper.TESTFN, encoding='test_decoder') + self.assertEqual(f.read(i), decoded[:i]) + cookie = f.tell() + self.assertEqual(f.read(j), decoded[i:i + j]) + f.seek(cookie) + self.assertEqual(f.read(), decoded[i:]) + f.close() + + # Enable the test decoder. + StatefulIncrementalDecoder.codecEnabled = 1 + + # Run the tests. + try: + # Try each test case. + for input, _, _ in StatefulIncrementalDecoderTest.test_cases: + test_seek_and_tell_with_data(input) + + # Position each test case so that it crosses a chunk boundary. + for input, _, _ in StatefulIncrementalDecoderTest.test_cases: + offset = CHUNK_SIZE - len(input)//2 + prefix = b'.'*offset + # Don't bother seeking into the prefix (takes too long). + min_pos = offset*2 + test_seek_and_tell_with_data(prefix + input, min_pos) + + # Ensure our test decoder won't interfere with subsequent tests. + finally: + StatefulIncrementalDecoder.codecEnabled = 0 + + def test_multibyte_seek_and_tell(self): + f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") + f.write("AB\n\u3046\u3048\n") + f.close() + + f = self.open(os_helper.TESTFN, "r", encoding="euc_jp") + self.assertEqual(f.readline(), "AB\n") + p0 = f.tell() + self.assertEqual(f.readline(), "\u3046\u3048\n") + p1 = f.tell() + f.seek(p0) + self.assertEqual(f.readline(), "\u3046\u3048\n") + self.assertEqual(f.tell(), p1) + f.close() + + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + + + def test_seek_with_encoder_state(self): + f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") + f.write("\u00e6\u0300") + p0 = f.tell() + f.write("\u00e6") + f.seek(p0) + f.write("\u0300") + f.close() + + f = self.open(os_helper.TESTFN, "r", encoding="euc_jis_2004") + self.assertEqual(f.readline(), "\u00e6\u0300\u0300") + f.close() + + def test_encoded_writes(self): + data = "1234567890" + tests = ("utf-16", + "utf-16-le", + "utf-16-be", + "utf-32", + "utf-32-le", + "utf-32-be") + for encoding in tests: + buf = self.BytesIO() + f = self.TextIOWrapper(buf, encoding=encoding) + # Check if the BOM is written only once (see issue1753). + f.write(data) + f.write(data) + f.seek(0) + self.assertEqual(f.read(), data * 2) + f.seek(0) + self.assertEqual(f.read(), data * 2) + self.assertEqual(buf.getvalue(), (data * 2).encode(encoding)) + + def test_unreadable(self): + class UnReadable(self.BytesIO): + def readable(self): + return False + txt = self.TextIOWrapper(UnReadable(), encoding="utf-8") + self.assertRaises(OSError, txt.read) + + def test_read_one_by_one(self): + txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB"), encoding="utf-8") + reads = "" + while True: + c = txt.read(1) + if not c: + break + reads += c + self.assertEqual(reads, "AA\nBB") + + def test_readlines(self): + txt = self.TextIOWrapper(self.BytesIO(b"AA\nBB\nCC"), encoding="utf-8") + self.assertEqual(txt.readlines(), ["AA\n", "BB\n", "CC"]) + txt.seek(0) + self.assertEqual(txt.readlines(None), ["AA\n", "BB\n", "CC"]) + txt.seek(0) + self.assertEqual(txt.readlines(5), ["AA\n", "BB\n"]) + + # read in amounts equal to TextIOWrapper._CHUNK_SIZE which is 128. + def test_read_by_chunk(self): + # make sure "\r\n" straddles 128 char boundary. + txt = self.TextIOWrapper(self.BytesIO(b"A" * 127 + b"\r\nB"), encoding="utf-8") + reads = "" + while True: + c = txt.read(128) + if not c: + break + reads += c + self.assertEqual(reads, "A"*127+"\nB") + + def test_writelines(self): + l = ['ab', 'cd', 'ef'] + buf = self.BytesIO() + txt = self.TextIOWrapper(buf, encoding="utf-8") + txt.writelines(l) + txt.flush() + self.assertEqual(buf.getvalue(), b'abcdef') + + def test_writelines_userlist(self): + l = UserList(['ab', 'cd', 'ef']) + buf = self.BytesIO() + txt = self.TextIOWrapper(buf, encoding="utf-8") + txt.writelines(l) + txt.flush() + self.assertEqual(buf.getvalue(), b'abcdef') + + def test_writelines_error(self): + txt = self.TextIOWrapper(self.BytesIO(), encoding="utf-8") + self.assertRaises(TypeError, txt.writelines, [1, 2, 3]) + self.assertRaises(TypeError, txt.writelines, None) + self.assertRaises(TypeError, txt.writelines, b'abc') + + def test_issue1395_1(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + + # read one char at a time + reads = "" + while True: + c = txt.read(1) + if not c: + break + reads += c + self.assertEqual(reads, self.normalized) + + def test_issue1395_2(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = "" + while True: + c = txt.read(4) + if not c: + break + reads += c + self.assertEqual(reads, self.normalized) + + def test_issue1395_3(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + reads += txt.read(4) + reads += txt.readline() + reads += txt.readline() + reads += txt.readline() + self.assertEqual(reads, self.normalized) + + def test_issue1395_4(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + reads += txt.read() + self.assertEqual(reads, self.normalized) + + def test_issue1395_5(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + pos = txt.tell() + txt.seek(0) + txt.seek(pos) + self.assertEqual(txt.read(4), "BBB\n") + + def test_issue2282(self): + buffer = self.BytesIO(self.testdata) + txt = self.TextIOWrapper(buffer, encoding="ascii") + + self.assertEqual(buffer.seekable(), txt.seekable()) + + def test_append_bom(self): + # The BOM is not written again when appending to a non-empty file + filename = os_helper.TESTFN + for charset in ('utf-8-sig', 'utf-16', 'utf-32'): + with self.open(filename, 'w', encoding=charset) as f: + f.write('aaa') + pos = f.tell() + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'aaa'.encode(charset)) + + with self.open(filename, 'a', encoding=charset) as f: + f.write('xxx') + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) + + def test_seek_bom(self): + # Same test, but when seeking manually + filename = os_helper.TESTFN + for charset in ('utf-8-sig', 'utf-16', 'utf-32'): + with self.open(filename, 'w', encoding=charset) as f: + f.write('aaa') + pos = f.tell() + with self.open(filename, 'r+', encoding=charset) as f: + f.seek(pos) + f.write('zzz') + f.seek(0) + f.write('bbb') + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) + + def test_seek_append_bom(self): + # Same test, but first seek to the start and then to the end + filename = os_helper.TESTFN + for charset in ('utf-8-sig', 'utf-16', 'utf-32'): + with self.open(filename, 'w', encoding=charset) as f: + f.write('aaa') + with self.open(filename, 'a', encoding=charset) as f: + f.seek(0) + f.seek(0, self.SEEK_END) + f.write('xxx') + with self.open(filename, 'rb') as f: + self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) + + def test_errors_property(self): + with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: + self.assertEqual(f.errors, "strict") + with self.open(os_helper.TESTFN, "w", encoding="utf-8", errors="replace") as f: + self.assertEqual(f.errors, "replace") + + @support.no_tracing + @threading_helper.requires_working_threading() + def test_threads_write(self): + # Issue6750: concurrent writes could duplicate data + event = threading.Event() + with self.open(os_helper.TESTFN, "w", encoding="utf-8", buffering=1) as f: + def run(n): + text = "Thread%03d\n" % n + event.wait() + f.write(text) + threads = [threading.Thread(target=run, args=(x,)) + for x in range(20)] + with threading_helper.start_threads(threads, event.set): + time.sleep(0.02) + with self.open(os_helper.TESTFN, encoding="utf-8") as f: + content = f.read() + for n in range(20): + self.assertEqual(content.count("Thread%03d\n" % n), 1) + + def test_flush_error_on_close(self): + # Test that text file is closed despite failed flush + # and that flush() is called before file closed. + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + closed = [] + def bad_flush(): + closed[:] = [txt.closed, txt.buffer.closed] + raise OSError() + txt.flush = bad_flush + self.assertRaises(OSError, txt.close) # exception not swallowed + self.assertTrue(txt.closed) + self.assertTrue(txt.buffer.closed) + self.assertTrue(closed) # flush() called + self.assertFalse(closed[0]) # flush() called before file closed + self.assertFalse(closed[1]) + txt.flush = lambda: None # break reference loop + + def test_close_error_on_close(self): + buffer = self.BytesIO(self.testdata) + def bad_flush(): + raise OSError('flush') + def bad_close(): + raise OSError('close') + buffer.close = bad_close + txt = self.TextIOWrapper(buffer, encoding="ascii") + txt.flush = bad_flush + with self.assertRaises(OSError) as err: # exception not swallowed + txt.close() + self.assertEqual(err.exception.args, ('close',)) + self.assertIsInstance(err.exception.__context__, OSError) + self.assertEqual(err.exception.__context__.args, ('flush',)) + self.assertFalse(txt.closed) + + # Silence destructor error + buffer.close = lambda: None + txt.flush = lambda: None + + def test_nonnormalized_close_error_on_close(self): + # Issue #21677 + buffer = self.BytesIO(self.testdata) + def bad_flush(): + raise non_existing_flush + def bad_close(): + raise non_existing_close + buffer.close = bad_close + txt = self.TextIOWrapper(buffer, encoding="ascii") + txt.flush = bad_flush + with self.assertRaises(NameError) as err: # exception not swallowed + txt.close() + self.assertIn('non_existing_close', str(err.exception)) + self.assertIsInstance(err.exception.__context__, NameError) + self.assertIn('non_existing_flush', str(err.exception.__context__)) + self.assertFalse(txt.closed) + + # Silence destructor error + buffer.close = lambda: None + txt.flush = lambda: None + + def test_multi_close(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + txt.close() + txt.close() + txt.close() + self.assertRaises(ValueError, txt.flush) + + def test_unseekable(self): + txt = self.TextIOWrapper(self.MockUnseekableIO(self.testdata), encoding="utf-8") + self.assertRaises(self.UnsupportedOperation, txt.tell) + self.assertRaises(self.UnsupportedOperation, txt.seek, 0) + + def test_readonly_attributes(self): + txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii") + buf = self.BytesIO(self.testdata) + with self.assertRaises(AttributeError): + txt.buffer = buf + + def test_rawio(self): + # Issue #12591: TextIOWrapper must work with raw I/O objects, so + # that subprocess.Popen() can have the required unbuffered + # semantics with universal_newlines=True. + raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + # Reads + self.assertEqual(txt.read(4), 'abcd') + self.assertEqual(txt.readline(), 'efghi\n') + self.assertEqual(list(txt), ['jkl\n', 'opq\n']) + + def test_rawio_write_through(self): + # Issue #12591: with write_through=True, writes don't need a flush + raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n']) + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n', + write_through=True) + txt.write('1') + txt.write('23\n4') + txt.write('5') + self.assertEqual(b''.join(raw._write_stack), b'123\n45') + + def test_bufio_write_through(self): + # Issue #21396: write_through=True doesn't force a flush() + # on the underlying binary buffered object. + flush_called, write_called = [], [] + class BufferedWriter(self.BufferedWriter): + def flush(self, *args, **kwargs): + flush_called.append(True) + return super().flush(*args, **kwargs) + def write(self, *args, **kwargs): + write_called.append(True) + return super().write(*args, **kwargs) + + rawio = self.BytesIO() + data = b"a" + bufio = BufferedWriter(rawio, len(data)*2) + textio = self.TextIOWrapper(bufio, encoding='ascii', + write_through=True) + # write to the buffered io but don't overflow the buffer + text = data.decode('ascii') + textio.write(text) + + # buffer.flush is not called with write_through=True + self.assertFalse(flush_called) + # buffer.write *is* called with write_through=True + self.assertTrue(write_called) + self.assertEqual(rawio.getvalue(), b"") # no flush + + write_called = [] # reset + textio.write(text * 10) # total content is larger than bufio buffer + self.assertTrue(write_called) + self.assertEqual(rawio.getvalue(), data * 11) # all flushed + + def test_reconfigure_write_through(self): + raw = self.MockRawIO([]) + t = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + t.write('1') + t.reconfigure(write_through=True) # implied flush + self.assertEqual(t.write_through, True) + self.assertEqual(b''.join(raw._write_stack), b'1') + t.write('23') + self.assertEqual(b''.join(raw._write_stack), b'123') + t.reconfigure(write_through=False) + self.assertEqual(t.write_through, False) + t.write('45') + t.flush() + self.assertEqual(b''.join(raw._write_stack), b'12345') + # Keeping default value + t.reconfigure() + t.reconfigure(write_through=None) + self.assertEqual(t.write_through, False) + t.reconfigure(write_through=True) + t.reconfigure() + t.reconfigure(write_through=None) + self.assertEqual(t.write_through, True) + + def test_read_nonbytes(self): + # Issue #17106 + # Crash when underlying read() returns non-bytes + t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") + self.assertRaises(TypeError, t.read, 1) + t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") + self.assertRaises(TypeError, t.readline) + t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8") + self.assertRaises(TypeError, t.read) + + def test_illegal_encoder(self): + # Issue 31271: Calling write() while the return value of encoder's + # encode() is invalid shouldn't cause an assertion failure. + rot13 = codecs.lookup("rot13") + with support.swap_attr(rot13, '_is_text_encoding', True): + t = self.TextIOWrapper(self.BytesIO(b'foo'), encoding="rot13") + self.assertRaises(TypeError, t.write, 'bar') + + def test_illegal_decoder(self): + # Issue #17106 + # Bypass the early encoding check added in issue 20404 + def _make_illegal_wrapper(): + quopri = codecs.lookup("quopri") + quopri._is_text_encoding = True + try: + t = self.TextIOWrapper(self.BytesIO(b'aaaaaa'), + newline='\n', encoding="quopri") + finally: + quopri._is_text_encoding = False + return t + # Crash when decoder returns non-string + t = _make_illegal_wrapper() + self.assertRaises(TypeError, t.read, 1) + t = _make_illegal_wrapper() + self.assertRaises(TypeError, t.readline) + t = _make_illegal_wrapper() + self.assertRaises(TypeError, t.read) + + # Issue 31243: calling read() while the return value of decoder's + # getstate() is invalid should neither crash the interpreter nor + # raise a SystemError. + def _make_very_illegal_wrapper(getstate_ret_val): + class BadDecoder: + def getstate(self): + return getstate_ret_val + def _get_bad_decoder(dummy): + return BadDecoder() + quopri = codecs.lookup("quopri") + with support.swap_attr(quopri, 'incrementaldecoder', + _get_bad_decoder): + return _make_illegal_wrapper() + t = _make_very_illegal_wrapper(42) + self.assertRaises(TypeError, t.read, 42) + t = _make_very_illegal_wrapper(()) + self.assertRaises(TypeError, t.read, 42) + t = _make_very_illegal_wrapper((1, 2)) + self.assertRaises(TypeError, t.read, 42) + + def _check_create_at_shutdown(self, **kwargs): + # Issue #20037: creating a TextIOWrapper at shutdown + # shouldn't crash the interpreter. + iomod = self.io.__name__ + code = """if 1: + import codecs + import {iomod} as io + + # Avoid looking up codecs at shutdown + codecs.lookup('utf-8') + + class C: + def __del__(self): + io.TextIOWrapper(io.BytesIO(), **{kwargs}) + print("ok") + c = C() + """.format(iomod=iomod, kwargs=kwargs) + return assert_python_ok("-c", code) + + def test_create_at_shutdown_without_encoding(self): + rc, out, err = self._check_create_at_shutdown() + if err: + # Can error out with a RuntimeError if the module state + # isn't found. + self.assertIn(self.shutdown_error, err.decode()) + else: + self.assertEqual("ok", out.decode().strip()) + + def test_create_at_shutdown_with_encoding(self): + rc, out, err = self._check_create_at_shutdown(encoding='utf-8', + errors='strict') + self.assertFalse(err) + self.assertEqual("ok", out.decode().strip()) + + def test_read_byteslike(self): + r = MemviewBytesIO(b'Just some random string\n') + t = self.TextIOWrapper(r, 'utf-8') + + # TextIOwrapper will not read the full string, because + # we truncate it to a multiple of the native int size + # so that we can construct a more complex memoryview. + bytes_val = _to_memoryview(r.getvalue()).tobytes() + + self.assertEqual(t.read(200), bytes_val.decode('utf-8')) + + def test_issue22849(self): + class F(object): + def readable(self): return True + def writable(self): return True + def seekable(self): return True + + for i in range(10): + try: + self.TextIOWrapper(F(), encoding='utf-8') + except Exception: + pass + + F.tell = lambda x: 0 + t = self.TextIOWrapper(F(), encoding='utf-8') + + def test_reconfigure_locale(self): + wrapper = self.TextIOWrapper(self.BytesIO(b"test")) + wrapper.reconfigure(encoding="locale") + + def test_reconfigure_encoding_read(self): + # latin1 -> utf8 + # (latin1 can decode utf-8 encoded string) + data = 'abc\xe9\n'.encode('latin1') + 'd\xe9f\n'.encode('utf8') + raw = self.BytesIO(data) + txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') + self.assertEqual(txt.readline(), 'abc\xe9\n') + with self.assertRaises(self.UnsupportedOperation): + txt.reconfigure(encoding='utf-8') + with self.assertRaises(self.UnsupportedOperation): + txt.reconfigure(newline=None) + + def test_reconfigure_write_fromascii(self): + # ascii has a specific encodefunc in the C implementation, + # but utf-8-sig has not. Make sure that we get rid of the + # cached encodefunc when we switch encoders. + raw = self.BytesIO() + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + txt.write('foo\n') + txt.reconfigure(encoding='utf-8-sig') + txt.write('\xe9\n') + txt.flush() + self.assertEqual(raw.getvalue(), b'foo\n\xc3\xa9\n') + + def test_reconfigure_write(self): + # latin -> utf8 + raw = self.BytesIO() + txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n') + txt.write('abc\xe9\n') + txt.reconfigure(encoding='utf-8') + self.assertEqual(raw.getvalue(), b'abc\xe9\n') + txt.write('d\xe9f\n') + txt.flush() + self.assertEqual(raw.getvalue(), b'abc\xe9\nd\xc3\xa9f\n') + + # ascii -> utf-8-sig: ensure that no BOM is written in the middle of + # the file + raw = self.BytesIO() + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + txt.write('abc\n') + txt.reconfigure(encoding='utf-8-sig') + txt.write('d\xe9f\n') + txt.flush() + self.assertEqual(raw.getvalue(), b'abc\nd\xc3\xa9f\n') + + def test_reconfigure_write_non_seekable(self): + raw = self.BytesIO() + raw.seekable = lambda: False + raw.seek = None + txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n') + txt.write('abc\n') + txt.reconfigure(encoding='utf-8-sig') + txt.write('d\xe9f\n') + txt.flush() + + # If the raw stream is not seekable, there'll be a BOM + self.assertEqual(raw.getvalue(), b'abc\n\xef\xbb\xbfd\xc3\xa9f\n') + + def test_reconfigure_defaults(self): + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\n') + txt.reconfigure(encoding=None) + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + txt.write('LF\n') + + txt.reconfigure(newline='\r\n') + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + + txt.reconfigure(errors='ignore') + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'ignore') + txt.write('CRLF\n') + + txt.reconfigure(encoding='utf-8', newline=None) + self.assertEqual(txt.errors, 'strict') + txt.seek(0) + self.assertEqual(txt.read(), 'LF\nCRLF\n') + + self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n') + + def test_reconfigure_errors(self): + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r') + with self.assertRaises(TypeError): # there was a crash + txt.reconfigure(encoding=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(encoding='\udcfe') + with self.assertRaises(LookupError): + txt.reconfigure(encoding='locale\0') + # TODO: txt.reconfigure(encoding='utf-8\0') + # TODO: txt.reconfigure(encoding='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(errors=42) + if self.is_C: + with self.assertRaises(UnicodeEncodeError): + txt.reconfigure(errors='\udcfe') + # TODO: txt.reconfigure(errors='ignore\0') + # TODO: txt.reconfigure(errors='nonexisting') + with self.assertRaises(TypeError): + txt.reconfigure(newline=42) + with self.assertRaises(ValueError): + txt.reconfigure(newline='\udcfe') + with self.assertRaises(ValueError): + txt.reconfigure(newline='xyz') + if not self.is_C: + # TODO: Should fail in C too. + with self.assertRaises(ValueError): + txt.reconfigure(newline='\n\0') + if self.is_C: + # TODO: Use __bool__(), not __index__(). + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(line_buffering=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(line_buffering=2**1000) + with self.assertRaises(ZeroDivisionError): + txt.reconfigure(write_through=BadIndex()) + with self.assertRaises(OverflowError): + txt.reconfigure(write_through=2**1000) + with self.assertRaises(ZeroDivisionError): # there was a crash + txt.reconfigure(line_buffering=BadIndex(), + write_through=BadIndex()) + self.assertEqual(txt.encoding, 'ascii') + self.assertEqual(txt.errors, 'replace') + self.assertIs(txt.line_buffering, False) + self.assertIs(txt.write_through, False) + + txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n', + line_buffering=True, write_through=True) + self.assertEqual(txt.encoding, 'latin1') + self.assertEqual(txt.errors, 'ignore') + self.assertIs(txt.line_buffering, True) + self.assertIs(txt.write_through, True) + + def test_reconfigure_newline(self): + raw = self.BytesIO(b'CR\rEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\n') + txt.reconfigure(newline=None) + self.assertEqual(txt.readline(), 'CR\n') + raw = self.BytesIO(b'CR\rEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\n') + txt.reconfigure(newline='') + self.assertEqual(txt.readline(), 'CR\r') + raw = self.BytesIO(b'CR\rLF\nEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\r') + txt.reconfigure(newline='\n') + self.assertEqual(txt.readline(), 'CR\rLF\n') + raw = self.BytesIO(b'LF\nCR\rEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\n') + txt.reconfigure(newline='\r') + self.assertEqual(txt.readline(), 'LF\nCR\r') + raw = self.BytesIO(b'CR\rCRLF\r\nEOF') + txt = self.TextIOWrapper(raw, 'ascii', newline='\r') + txt.reconfigure(newline='\r\n') + self.assertEqual(txt.readline(), 'CR\rCRLF\r\n') + + txt = self.TextIOWrapper(self.BytesIO(), 'ascii', newline='\r') + txt.reconfigure(newline=None) + txt.write('linesep\n') + txt.reconfigure(newline='') + txt.write('LF\n') + txt.reconfigure(newline='\n') + txt.write('LF\n') + txt.reconfigure(newline='\r') + txt.write('CR\n') + txt.reconfigure(newline='\r\n') + txt.write('CRLF\n') + expected = 'linesep' + os.linesep + 'LF\nLF\nCR\rCRLF\r\n' + self.assertEqual(txt.detach().getvalue().decode('ascii'), expected) + + def test_issue25862(self): + # Assertion failures occurred in tell() after read() and write(). + t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') + t.read(1) + t.read() + t.tell() + t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii') + t.read(1) + t.write('x') + t.tell() + + def test_issue35928(self): + p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO()) + f = self.TextIOWrapper(p) + res = f.readline() + self.assertEqual(res, 'foo\n') + f.write(res) + self.assertEqual(res + f.readline(), 'foo\nbar\n') + + def test_pickling_subclass(self): + global MyTextIO + class MyTextIO(self.TextIOWrapper): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.buffer.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + txt = MyTextIO(raw, 'ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(txt, proto) + newtxt = pickle.loads(pickled) + self.assertEqual(newtxt.buffer.getvalue(), b'data') + self.assertEqual(newtxt.tag, 'ham') + del MyTextIO + + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_read_non_blocking(self): + import os + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.io.open(r, 'rt') as textfile: + r = None + # Nothing has been written so a non-blocking read raises a BlockingIOError exception. + with self.assertRaises(BlockingIOError): + textfile.read() + finally: + if r is not None: + os.close(r) + os.close(w) + + +class MemviewBytesIO(io.BytesIO): + '''A BytesIO object whose read method returns memoryviews + rather than bytes''' + + def read1(self, len_): + return _to_memoryview(super().read1(len_)) + + def read(self, len_): + return _to_memoryview(super().read(len_)) + +def _to_memoryview(buf): + '''Convert bytes-object *buf* to a non-trivial memoryview''' + + arr = array.array('i') + idx = len(buf) - len(buf) % arr.itemsize + arr.frombytes(buf[:idx]) + return memoryview(arr) + + +class CTextIOWrapperTest(TextIOWrapperTest, CTestCase): + shutdown_error = "LookupError: unknown encoding: ascii" + + def test_initialization(self): + r = self.BytesIO(b"\xc3\xa9\n\n") + b = self.BufferedReader(r, 1000) + t = self.TextIOWrapper(b, encoding="utf-8") + self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy') + self.assertRaises(ValueError, t.read) + + t = self.TextIOWrapper.__new__(self.TextIOWrapper) + self.assertRaises(Exception, repr, t) + + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON; cyclic GC not supported, causes file locking") + def test_garbage_collection(self): + # C TextIOWrapper objects are collected, and collecting them flushes + # all data to disk. + # The Python version has __del__, so it ends in gc.garbage instead. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ResourceWarning) + rawio = self.FileIO(os_helper.TESTFN, "wb") + b = self.BufferedWriter(rawio) + t = self.TextIOWrapper(b, encoding="ascii") + t.write("456def") + t.x = t + wr = weakref.ref(t) + del t + support.gc_collect() + self.assertIsNone(wr(), wr) + with self.open(os_helper.TESTFN, "rb") as f: + self.assertEqual(f.read(), b"456def") + + def test_rwpair_cleared_before_textio(self): + # Issue 13070: TextIOWrapper's finalization would crash when called + # after the reference to the underlying BufferedRWPair's writer got + # cleared by the GC. + for i in range(1000): + b1 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) + t1 = self.TextIOWrapper(b1, encoding="ascii") + b2 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()) + t2 = self.TextIOWrapper(b2, encoding="ascii") + # circular references + t1.buddy = t2 + t2.buddy = t1 + support.gc_collect() + + def test_del__CHUNK_SIZE_SystemError(self): + t = self.TextIOWrapper(self.BytesIO(), encoding='ascii') + with self.assertRaises(AttributeError): + del t._CHUNK_SIZE + + def test_internal_buffer_size(self): + # bpo-43260: TextIOWrapper's internal buffer should not store + # data larger than chunk size. + chunk_size = 8192 # default chunk size, updated later + + class MockIO(self.MockRawIO): + def write(self, data): + if len(data) > chunk_size: + raise RuntimeError + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf, encoding="ascii") + chunk_size = t._CHUNK_SIZE + t.write("abc") + t.write("def") + # default chunk size is 8192 bytes so t don't write data to buf. + self.assertEqual([], buf._write_stack) + + with self.assertRaises(RuntimeError): + t.write("x"*(chunk_size+1)) + + self.assertEqual([b"abcdef"], buf._write_stack) + t.write("ghi") + t.write("x"*chunk_size) + self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + + def test_issue142594(self): + wrapper = None + detached = False + class ReentrantRawIO(self.RawIOBase): + @property + def closed(self): + nonlocal detached + if wrapper is not None and not detached: + detached = True + wrapper.detach() + return False + + raw = ReentrantRawIO() + wrapper = self.TextIOWrapper(raw) + wrapper.close() # should not crash + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'NoneType' + def test_read_non_blocking(self): + return super().test_read_non_blocking() + + +class PyTextIOWrapperTest(TextIOWrapperTest, PyTestCase): + shutdown_error = "LookupError: unknown encoding: ascii" + + +class IncrementalNewlineDecoderTest: + + def check_newline_decoding_utf8(self, decoder): + # UTF-8 specific tests for a newline decoder + def _check_decode(b, s, **kwargs): + # We exercise getstate() / setstate() as well as decode() + state = decoder.getstate() + self.assertEqual(decoder.decode(b, **kwargs), s) + decoder.setstate(state) + self.assertEqual(decoder.decode(b, **kwargs), s) + + _check_decode(b'\xe8\xa2\x88', "\u8888") + + _check_decode(b'\xe8', "") + _check_decode(b'\xa2', "") + _check_decode(b'\x88', "\u8888") + + _check_decode(b'\xe8', "") + _check_decode(b'\xa2', "") + _check_decode(b'\x88', "\u8888") + + _check_decode(b'\xe8', "") + self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True) + + decoder.reset() + _check_decode(b'\n', "\n") + _check_decode(b'\r', "") + _check_decode(b'', "\n", final=True) + _check_decode(b'\r', "\n", final=True) + + _check_decode(b'\r', "") + _check_decode(b'a', "\na") + + _check_decode(b'\r\r\n', "\n\n") + _check_decode(b'\r', "") + _check_decode(b'\r', "\n") + _check_decode(b'\na', "\na") + + _check_decode(b'\xe8\xa2\x88\r\n', "\u8888\n") + _check_decode(b'\xe8\xa2\x88', "\u8888") + _check_decode(b'\n', "\n") + _check_decode(b'\xe8\xa2\x88\r', "\u8888") + _check_decode(b'\n', "\n") + + def check_newline_decoding(self, decoder, encoding): + result = [] + if encoding is not None: + encoder = codecs.getincrementalencoder(encoding)() + def _decode_bytewise(s): + # Decode one byte at a time + for b in encoder.encode(s): + result.append(decoder.decode(bytes([b]))) + else: + encoder = None + def _decode_bytewise(s): + # Decode one char at a time + for c in s: + result.append(decoder.decode(c)) + self.assertEqual(decoder.newlines, None) + _decode_bytewise("abc\n\r") + self.assertEqual(decoder.newlines, '\n') + _decode_bytewise("\nabc") + self.assertEqual(decoder.newlines, ('\n', '\r\n')) + _decode_bytewise("abc\r") + self.assertEqual(decoder.newlines, ('\n', '\r\n')) + _decode_bytewise("abc") + self.assertEqual(decoder.newlines, ('\r', '\n', '\r\n')) + _decode_bytewise("abc\r") + self.assertEqual("".join(result), "abc\n\nabcabc\nabcabc") + decoder.reset() + input = "abc" + if encoder is not None: + encoder.reset() + input = encoder.encode(input) + self.assertEqual(decoder.decode(input), "abc") + self.assertEqual(decoder.newlines, None) + + def test_newline_decoder(self): + encodings = ( + # None meaning the IncrementalNewlineDecoder takes unicode input + # rather than bytes input + None, 'utf-8', 'latin-1', + 'utf-16', 'utf-16-le', 'utf-16-be', + 'utf-32', 'utf-32-le', 'utf-32-be', + ) + for enc in encodings: + decoder = enc and codecs.getincrementaldecoder(enc)() + decoder = self.IncrementalNewlineDecoder(decoder, translate=True) + self.check_newline_decoding(decoder, enc) + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = self.IncrementalNewlineDecoder(decoder, translate=True) + self.check_newline_decoding_utf8(decoder) + self.assertRaises(TypeError, decoder.setstate, 42) + + def test_newline_bytes(self): + # Issue 5433: Excessive optimization in IncrementalNewlineDecoder + def _check(dec): + self.assertEqual(dec.newlines, None) + self.assertEqual(dec.decode("\u0D00"), "\u0D00") + self.assertEqual(dec.newlines, None) + self.assertEqual(dec.decode("\u0A00"), "\u0A00") + self.assertEqual(dec.newlines, None) + dec = self.IncrementalNewlineDecoder(None, translate=False) + _check(dec) + dec = self.IncrementalNewlineDecoder(None, translate=True) + _check(dec) + + def test_translate(self): + # issue 35062 + for translate in (-2, -1, 1, 2): + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = self.IncrementalNewlineDecoder(decoder, translate) + self.check_newline_decoding_utf8(decoder) + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = self.IncrementalNewlineDecoder(decoder, translate=0) + self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n") + +class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest, unittest.TestCase): + IncrementalNewlineDecoder = io.IncrementalNewlineDecoder + + @support.cpython_only + def test_uninitialized(self): + uninitialized = self.IncrementalNewlineDecoder.__new__( + self.IncrementalNewlineDecoder) + self.assertRaises(ValueError, uninitialized.decode, b'bar') + self.assertRaises(ValueError, uninitialized.getstate) + self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0)) + self.assertRaises(ValueError, uninitialized.reset) + + +class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest, unittest.TestCase): + IncrementalNewlineDecoder = pyio.IncrementalNewlineDecoder diff --git a/Lib/test/test_univnewlines.py b/Lib/test/test_io/test_univnewlines.py similarity index 100% rename from Lib/test/test_univnewlines.py rename to Lib/test/test_io/test_univnewlines.py diff --git a/Lib/test/test_io/utils.py b/Lib/test/test_io/utils.py new file mode 100644 index 00000000000..3b1faec2140 --- /dev/null +++ b/Lib/test/test_io/utils.py @@ -0,0 +1,318 @@ +import array +import threading +import time +import unittest + +import io # C implementation of io +import _pyio as pyio # Python implementation of io + + +try: + import ctypes +except ImportError: + def byteslike(*pos, **kw): + return array.array("b", bytes(*pos, **kw)) +else: + class EmptyStruct(ctypes.Structure): + pass + + def byteslike(*pos, **kw): + """Create a bytes-like object having no string or sequence methods""" + data = bytes(*pos, **kw) + obj = EmptyStruct() + ctypes.resize(obj, len(data)) + memoryview(obj).cast("B")[:] = data + return obj + + +class MockRawIOWithoutRead: + """A RawIO implementation without read(), so as to exercise the default + RawIO.read() which calls readinto().""" + + def __init__(self, read_stack=()): + self._read_stack = list(read_stack) + self._write_stack = [] + self._reads = 0 + self._extraneous_reads = 0 + + def write(self, b): + self._write_stack.append(bytes(b)) + return len(b) + + def writable(self): + return True + + def fileno(self): + return 42 + + def readable(self): + return True + + def seekable(self): + return True + + def seek(self, pos, whence): + return 0 # wrong but we gotta return something + + def tell(self): + return 0 # same comment as above + + def readinto(self, buf): + self._reads += 1 + max_len = len(buf) + try: + data = self._read_stack[0] + except IndexError: + self._extraneous_reads += 1 + return 0 + if data is None: + del self._read_stack[0] + return None + n = len(data) + if len(data) <= max_len: + del self._read_stack[0] + buf[:n] = data + return n + else: + buf[:] = data[:max_len] + self._read_stack[0] = data[max_len:] + return max_len + + def truncate(self, pos=None): + return pos + +class CMockRawIOWithoutRead(MockRawIOWithoutRead, io.RawIOBase): + pass + +class PyMockRawIOWithoutRead(MockRawIOWithoutRead, pyio.RawIOBase): + pass + + +class MockRawIO(MockRawIOWithoutRead): + + def read(self, n=None): + self._reads += 1 + try: + return self._read_stack.pop(0) + except: + self._extraneous_reads += 1 + return b"" + +class CMockRawIO(MockRawIO, io.RawIOBase): + pass + +class PyMockRawIO(MockRawIO, pyio.RawIOBase): + pass + + +class MisbehavedRawIO(MockRawIO): + def write(self, b): + return super().write(b) * 2 + + def read(self, n=None): + return super().read(n) * 2 + + def seek(self, pos, whence): + return -123 + + def tell(self): + return -456 + + def readinto(self, buf): + super().readinto(buf) + return len(buf) * 5 + +class CMisbehavedRawIO(MisbehavedRawIO, io.RawIOBase): + pass + +class PyMisbehavedRawIO(MisbehavedRawIO, pyio.RawIOBase): + pass + + +class SlowFlushRawIO(MockRawIO): + def __init__(self): + super().__init__() + self.in_flush = threading.Event() + + def flush(self): + self.in_flush.set() + time.sleep(0.25) + +class CSlowFlushRawIO(SlowFlushRawIO, io.RawIOBase): + pass + +class PySlowFlushRawIO(SlowFlushRawIO, pyio.RawIOBase): + pass + + +class CloseFailureIO(MockRawIO): + closed = 0 + + def close(self): + if not self.closed: + self.closed = 1 + raise OSError + +class CCloseFailureIO(CloseFailureIO, io.RawIOBase): + pass + +class PyCloseFailureIO(CloseFailureIO, pyio.RawIOBase): + pass + + +class MockFileIO: + + def __init__(self, data): + self.read_history = [] + super().__init__(data) + + def read(self, n=None): + res = super().read(n) + self.read_history.append(None if res is None else len(res)) + return res + + def readinto(self, b): + res = super().readinto(b) + self.read_history.append(res) + return res + +class CMockFileIO(MockFileIO, io.BytesIO): + pass + +class PyMockFileIO(MockFileIO, pyio.BytesIO): + pass + + +class MockUnseekableIO: + def seekable(self): + return False + + def seek(self, *args): + raise self.UnsupportedOperation("not seekable") + + def tell(self, *args): + raise self.UnsupportedOperation("not seekable") + + def truncate(self, *args): + raise self.UnsupportedOperation("not seekable") + +class CMockUnseekableIO(MockUnseekableIO, io.BytesIO): + UnsupportedOperation = io.UnsupportedOperation + +class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO): + UnsupportedOperation = pyio.UnsupportedOperation + + +class MockCharPseudoDevFileIO(MockFileIO): + # GH-95782 + # ftruncate() does not work on these special files (and CPython then raises + # appropriate exceptions), so truncate() does not have to be accounted for + # here. + def __init__(self, data): + super().__init__(data) + + def seek(self, *args): + return 0 + + def tell(self, *args): + return 0 + +class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO): + pass + +class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO): + pass + + +class MockNonBlockWriterIO: + + def __init__(self): + self._write_stack = [] + self._blocker_char = None + + def pop_written(self): + s = b"".join(self._write_stack) + self._write_stack[:] = [] + return s + + def block_on(self, char): + """Block when a given char is encountered.""" + self._blocker_char = char + + def readable(self): + return True + + def seekable(self): + return True + + def seek(self, pos, whence=0): + # naive implementation, enough for tests + return 0 + + def writable(self): + return True + + def write(self, b): + b = bytes(b) + n = -1 + if self._blocker_char: + try: + n = b.index(self._blocker_char) + except ValueError: + pass + else: + if n > 0: + # write data up to the first blocker + self._write_stack.append(b[:n]) + return n + else: + # cancel blocker and indicate would block + self._blocker_char = None + return None + self._write_stack.append(b) + return len(b) + +class CMockNonBlockWriterIO(MockNonBlockWriterIO, io.RawIOBase): + BlockingIOError = io.BlockingIOError + +class PyMockNonBlockWriterIO(MockNonBlockWriterIO, pyio.RawIOBase): + BlockingIOError = pyio.BlockingIOError + + +# Build classes which point to all the right mocks per io implementation +class CTestCase(unittest.TestCase): + io = io + is_C = True + + MockRawIO = CMockRawIO + MisbehavedRawIO = CMisbehavedRawIO + MockFileIO = CMockFileIO + CloseFailureIO = CCloseFailureIO + MockNonBlockWriterIO = CMockNonBlockWriterIO + MockUnseekableIO = CMockUnseekableIO + MockRawIOWithoutRead = CMockRawIOWithoutRead + SlowFlushRawIO = CSlowFlushRawIO + MockCharPseudoDevFileIO = CMockCharPseudoDevFileIO + + # Use the class as a proxy to the io module members. + def __getattr__(self, name): + return getattr(io, name) + + +class PyTestCase(unittest.TestCase): + io = pyio + is_C = False + + MockRawIO = PyMockRawIO + MisbehavedRawIO = PyMisbehavedRawIO + MockFileIO = PyMockFileIO + CloseFailureIO = PyCloseFailureIO + MockNonBlockWriterIO = PyMockNonBlockWriterIO + MockUnseekableIO = PyMockUnseekableIO + MockRawIOWithoutRead = PyMockRawIOWithoutRead + SlowFlushRawIO = PySlowFlushRawIO + MockCharPseudoDevFileIO = PyMockCharPseudoDevFileIO + + # Use the class as a proxy to the _pyio module members. + def __getattr__(self, name): + return getattr(pyio, name) diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py index 8c1a28ef870..129faf3dea6 100644 --- a/scripts/update_lib/deps.py +++ b/scripts/update_lib/deps.py @@ -570,10 +570,7 @@ def clear_import_graph_caches() -> None: "io": { "hard_deps": ["_pyio.py"], "test": [ - "test_io.py", - "test_bufio.py", - "test_fileio.py", - "test_memoryio.py", + "test_io", ], }, "dbm": { @@ -607,8 +604,7 @@ def clear_import_graph_caches() -> None: "file": { "lib": [], "test": [ - "test_file.py", - "test_largefile.py", + "test_io", ], }, "fcntl": { diff --git a/scripts/update_lib/tests/test_deps.py b/scripts/update_lib/tests/test_deps.py index d97af2867aa..4d5d6c0a44e 100644 --- a/scripts/update_lib/tests/test_deps.py +++ b/scripts/update_lib/tests/test_deps.py @@ -110,6 +110,18 @@ def test_known_dependency(self): self.assertEqual(len(paths), 1) self.assertEqual(paths[0], pathlib.Path("cpython/Lib/test/test_regrtest")) + def test_io_package(self): + self.assertEqual( + get_test_paths("io", "cpython"), + (pathlib.Path("cpython/Lib/test/test_io"),), + ) + + def test_file_alias_uses_io_package(self): + self.assertEqual( + get_test_paths("file", "cpython"), + (pathlib.Path("cpython/Lib/test/test_io"),), + ) + def test_default_directory(self): """Test default to test_name/ directory.""" with tempfile.TemporaryDirectory() as tmpdir: From f8a8d643a7682d613d5fa9d804d71581191263a6 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 17 Oct 2026 15:07:26 +0900 Subject: [PATCH 3/5] Protect weakref callbacks with the original referent stripe Follow CPython's free-threading weakref locking model instead of keeping an independent callback RwLock. Retain the original referent stripe key so callback access stays synchronized after GC clears the referent. Unlink and steal callbacks under that stripe, then release ownership outside the lock. Preserve synchronization for running-world GC paths and gc.get_referents while keeping ordinary weakref upgrades unchanged. Assisted-by: OpenAI Codex:model version unavailable --- crates/vm/src/object/core.rs | 85 ++++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 651496b4ba6..2511b680dc6 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -24,6 +24,7 @@ use alloc::fmt; use core::{ any::TypeId, borrow::Borrow, + cell::UnsafeCell, marker::PhantomData, mem::ManuallyDrop, num::NonZeroUsize, @@ -580,6 +581,9 @@ unsafe impl Traverse for PyObject { } // === Stripe lock for weakref list protection (WEAKREF_LIST_LOCK) === +// Like CPython's _Py_LOCK_DONT_DETACH weakref stripes, these critical sections +// must not run Python, release Python references, or suspend an attached thread. +// Traversal visitors must not recursively acquire a weakref stripe. #[cfg(feature = "threading")] mod weakref_lock { @@ -740,7 +744,8 @@ impl WeakRefList { let weak_payload = PyWeak { pointers: Pointers::new(), wr_object: Radium::new(ptr::null_mut()), - callback: PyRwLock::new(callback), + referent_addr: obj as *const PyObject as usize, + callback: UnsafeCell::new(callback), hash: Radium::new(crate::common::hash::SENTINEL), }; let weak = PyRef::new_ref(weak_payload, cls, dict); @@ -831,7 +836,8 @@ impl WeakRefList { match NonNull::new(candidate_ptr) { Some(candidate) => { let node = unsafe { candidate.as_ref() }; - let has_callback = node.payload.callback.read().is_some(); + // SAFETY: the referent stripe is held by the caller. + let has_callback = unsafe { &*node.payload.callback.get() }.is_some(); let node_cls = node.class(); // PyWeakref_CheckProxy: the basic-proxy slot is reserved for // the canonical proxy type; subclasses and callback-less ref @@ -865,13 +871,14 @@ impl WeakRefList { if wr.ref_count.safe_inc() { let owner = unsafe { PyRef::::from_raw(wr as *const Py) }; callbacks.push((owner, None)); - // Keep the owner in the outer batch before accessing a lock. + // Keep the owner in the outer batch before taking its callback. let entry = callbacks.last_mut().unwrap(); - entry.1 = wr.payload.callback.write().take(); + // SAFETY: the referent stripe protects the callback as well. + entry.1 = unsafe { &mut *wr.payload.callback.get() }.take(); } unsafe { unlink_weakref(self, node) }; - // This must be the last access if safe_inc failed: a concurrent - // deallocator may skip the stripe after observing the null pointer. + // Publish death only after unlinking, while the original stripe + // still protects this node from a concurrent deallocator. wr.payload .wr_object .store(ptr::null_mut(), Ordering::Release); @@ -901,13 +908,15 @@ impl WeakRefList { while let Some(node) = current { let next = unsafe { WeakLink::pointers(node).as_ref().get_next() }; let wr = unsafe { node.as_ref() }; - if wr.payload.callback.read().is_some() { + // SAFETY: the referent stripe protects this borrowed callback. + let callback = unsafe { &*wr.payload.callback.get() }; + if callback.is_some() { owners.reserve(1); callbacks.reserve(1); if !is_unreachable(wr.as_object()) && wr.ref_count.safe_inc() { let owner = unsafe { PyRef::::from_raw(wr as *const Py) }; owners.push(owner); - let callback = wr.payload.callback.read().clone(); + let callback = callback.clone(); if let Some(callback) = callback { callbacks.push((owners.last().unwrap().clone(), callback)); } @@ -1007,14 +1016,20 @@ pub struct PyWeak { /// Direct pointer to the referent object, null when dead. /// Equivalent to wr_object in PyWeakReference. wr_object: PyAtomic<*mut PyObject>, - /// Owned callback edge, protected independently of the referent list. + /// Immutable key for the original referent stripe, like CPython's + /// weakrefs_lock. Used only for locking, never dereferenced, so the same + /// stripe protects the callback even after wr_object becomes null. + referent_addr: usize, + /// Owned callback edge, protected by the original referent stripe. /// GC can retain this edge after unlinking the weakref. - callback: PyRwLock>, + callback: UnsafeCell>, pub(crate) hash: PyAtomic, } cfg_select! { feature = "threading" => { + // SAFETY: mutable callback and list access is serialized by the + // original referent stripe; referent and hash reads are atomic. unsafe impl Send for PyWeak {} unsafe impl Sync for PyWeak {} } @@ -1051,22 +1066,38 @@ impl PyWeak { /// Get the owned callback, which GC can retain after clearing the referent. pub(crate) fn get_callback(&self) -> Option { - self.callback.read().clone() + let _lock = weakref_lock::lock(self.referent_addr); + // SAFETY: the original referent stripe protects the callback. + unsafe { &*self.callback.get() }.clone() } - /// _PyWeakref_ClearRef: unlink without releasing the callback. - fn clear_ref(&self) { - let obj_ptr = self.wr_object.load(Ordering::Acquire); - if obj_ptr.is_null() { - return; // Already cleared by WeakRefList::clear() + /// Unlink and steal the callback under the original referent stripe. + /// Return ownership so callers release it only after unlocking. + fn clear(&self) -> Option { + let _lock = weakref_lock::lock(self.referent_addr); + // SAFETY: the original referent stripe protects both fields, including + // callbacks retained by GC after the weakref was already unlinked. + unsafe { + self.clear_ref_lock_held(); + (&mut *self.callback.get()).take() } + } - let _lock = weakref_lock::lock(obj_ptr as usize); + /// _PyWeakref_ClearRef: unlink without releasing the callback. + fn clear_ref(&self) { + let _lock = weakref_lock::lock(self.referent_addr); + // SAFETY: the original referent stripe is held. + unsafe { self.clear_ref_lock_held() }; + } - // Double-check under lock + /// Unlink without releasing the callback. + /// + /// # Safety + /// The original referent stripe must be held. + unsafe fn clear_ref_lock_held(&self) { let obj_ptr = self.wr_object.load(Ordering::Relaxed); if obj_ptr.is_null() { - return; // Cleared between our check and lock acquisition + return; } let obj = unsafe { &*obj_ptr }; @@ -1092,16 +1123,16 @@ impl PyWeak { // SAFETY: the callback is the only owned Python edge; wr_object is borrowed. unsafe impl Traverse for PyWeak { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - // A stopped thread may hold this lock. Missing a busy edge can - // postpone collection, but must not deadlock the collector. - if let Some(callback) = self.callback.try_read() { - callback.traverse(tracer_fn); - } + // Stripe critical sections never suspend or run Python, so a stopped + // thread cannot own this lock. Also synchronize gc.get_referents(), + // which traverses while other threads are running. + let _lock = weakref_lock::lock(self.referent_addr); + // SAFETY: the original referent stripe protects the callback. + unsafe { &*self.callback.get() }.traverse(tracer_fn); } fn clear(&mut self, out: &mut Vec) { - self.clear_ref(); - let callback = self.callback.write().take(); + let callback = Self::clear(self); if let Some(callback) = callback { out.push(callback); } @@ -1113,7 +1144,7 @@ impl Drop for PyWeak { fn drop(&mut self) { // we do NOT have actual exclusive access! let me: &Self = self; - me.clear_ref(); + drop(me.clear()); } } From 51e0ffe296fa86448e411725cafd082a42b8a331 Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 17 Oct 2026 16:31:59 +0900 Subject: [PATCH 4/5] Defer weakref GC tracking until referent publication Keep freshly allocated weakrefs out of GC's object lists until they are linked and their referent is published. Release the referent stripe before tracking, since tracking can collect and invoke finalizers. Generic reuse losers remain untracked. Exercise the allocator boundary directly for reference, proxy, callable proxy and subclass payloads, and ensure completed constructors track their objects. This prevents observers from retaining a half-built dead weakref that later becomes live. Assisted-by: OpenAI Codex:model version unavailable --- crates/vm/src/builtins/weakref.rs | 3 ++ crates/vm/src/object/core.rs | 82 +++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/builtins/weakref.rs b/crates/vm/src/builtins/weakref.rs index 135db6866cb..c934dc85413 100644 --- a/crates/vm/src/builtins/weakref.rs +++ b/crates/vm/src/builtins/weakref.rs @@ -24,6 +24,9 @@ pub struct WeakNewArgs { } impl PyPayload for PyWeak { + // WeakRefList::add tracks only after linking and publishing the referent. + const NEW_REF_UNTRACKED: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.weakref_type diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 2511b680dc6..dbd84ccf288 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -738,9 +738,9 @@ impl WeakRefList { } } - // Allocate OUTSIDE the stripe lock. PyRef::new_ref may trigger - // maybe_collect → GC → WeakRefList::clear on another object that - // hashes to the same stripe, which would deadlock on the spinlock. + // Allocate untracked: gc.get_objects() must not expose this node + // before its referent is published. Tracking below can trigger GC, + // so it must also happen outside the stripe lock. let weak_payload = PyWeak { pointers: Pointers::new(), wr_object: Radium::new(ptr::null_mut()), @@ -793,6 +793,12 @@ impl WeakRefList { weak.wr_object .store(obj as *const PyObject as *mut PyObject, Ordering::Release); + drop(_lock); + // SAFETY: this fresh allocation opted out of automatic tracking. + // Our strong reference keeps it alive, and it cannot enter a GC + // candidate list before tracking, so its gc_bits are still zero. + unsafe { crate::gc_state::track_new_object(NonNull::from(weak.as_object())) }; + weak } @@ -3635,6 +3641,76 @@ mod tests { drop(obj); } + #[test] + fn weakref_allocation_defers_gc_tracking_until_publication() { + crate::Interpreter::without_stdlib(Default::default()).enter(|vm| { + let target = vm.ctx.new_class( + None, + "WeakrefTarget", + vm.ctx.types.object_type.to_owned(), + Default::default(), + ); + let subclass = vm.ctx.new_class( + None, + "WeakrefSubclass", + vm.ctx.types.weakref_type.to_owned(), + Default::default(), + ); + for class in [ + vm.ctx.types.weakref_type.to_owned(), + vm.ctx.types.weakproxy_type.to_owned(), + vm.ctx.types.weakcallableproxy_type.to_owned(), + subclass, + ] { + let dict = class + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + .then(|| vm.ctx.new_dict()); + // Exercise the allocator boundary directly: an uninitialized + // weakref must not be published through GC generation lists. + let unpublished = PyRef::new_ref( + PyWeak { + pointers: Pointers::new(), + wr_object: Radium::new(ptr::null_mut()), + referent_addr: target.as_object() as *const PyObject as usize, + callback: UnsafeCell::new(None), + hash: Radium::new(crate::common::hash::SENTINEL), + }, + class.clone(), + dict, + ); + assert_eq!(unpublished.as_object().0.gc_bits.load(Ordering::Relaxed), 0); + assert!(!unpublished.as_object().is_gc_tracked()); + assert!( + !vm.state + .gc + .get_objects(None) + .iter() + .any(|object| object.is(unpublished.as_object())) + ); + drop(unpublished); + + let published = target + .as_object() + .downgrade_with_typ(None, class, vm) + .unwrap(); + assert!(published.as_object().is_gc_tracked()); + assert!( + vm.state + .gc + .get_objects(None) + .iter() + .any(|object| object.is(published.as_object())) + ); + assert_eq!( + published.wr_object.load(Ordering::Acquire), + target.as_object() as *const PyObject as *mut PyObject, + ); + } + }); + } + /// A weakref node stays linked into its target's list until its own /// `Drop` unlinks it, and `WeakRefList::add` reads the class off every /// node it walks looking for a proxy to reuse. A node that lost its class From 5cbebf40c018a8ff8d694f44df124eea83f4b28f Mon Sep 17 00:00:00 2001 From: youknowdot Date: Sat, 17 Oct 2026 22:45:22 +0900 Subject: [PATCH 5/5] Retain the original weakref stripe lock like CPython Store a safe handle to the selected static weakref stripe instead of retaining and rehashing the referent address. Keep the same lock after referent clearing, matching CPython free-threaded weakrefs_lock. The nonthreaded handle is zero-sized and introduces no locking. Preserve lock selection, acquire/release ordering, fork reset, deferred GC publication, and callback-owner release outside the stripe. No new locks or unsafe code are needed. Assisted-by: OpenAI Codex:model version unavailable --- crates/vm/src/object/core.rs | 78 +++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index dbd84ccf288..d93cefbebcf 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -593,29 +593,46 @@ mod weakref_lock { static LOCKS: [AtomicU8; NUM_WEAKREF_LOCKS] = [const { AtomicU8::new(0) }; NUM_WEAKREF_LOCKS]; + #[derive(Debug)] + pub(super) struct WeakrefLock { + lock: &'static AtomicU8, + } + pub(super) struct WeakrefLockGuard { - idx: usize, + lock: &'static AtomicU8, } impl Drop for WeakrefLockGuard { fn drop(&mut self) { - LOCKS[self.idx].store(0, Ordering::Release); + self.lock.store(0, Ordering::Release); } } - pub(super) fn lock(addr: usize) -> WeakrefLockGuard { - let idx = (addr >> 4) % NUM_WEAKREF_LOCKS; - loop { - if LOCKS[idx] - .compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed) - .is_ok() - { - return WeakrefLockGuard { idx }; + impl WeakrefLock { + pub(super) fn lock(&self) -> WeakrefLockGuard { + loop { + if self + .lock + .compare_exchange_weak(0, 1, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return WeakrefLockGuard { lock: self.lock }; + } + core::hint::spin_loop(); } - core::hint::spin_loop(); } } + pub(super) fn select(addr: usize) -> WeakrefLock { + WeakrefLock { + lock: &LOCKS[(addr >> 4) % NUM_WEAKREF_LOCKS], + } + } + + pub(super) fn lock(addr: usize) -> WeakrefLockGuard { + select(addr).lock() + } + /// Reset all weakref stripe locks after fork in child process. /// Locks held by parent threads would cause infinite spin in the child. #[cfg(all(unix, feature = "host_env"))] @@ -628,14 +645,27 @@ mod weakref_lock { #[cfg(not(feature = "threading"))] mod weakref_lock { + #[derive(Debug)] + pub(super) struct WeakrefLock; + pub(super) struct WeakrefLockGuard; impl Drop for WeakrefLockGuard { fn drop(&mut self) {} } - pub(super) fn lock(_addr: usize) -> WeakrefLockGuard { - WeakrefLockGuard + impl WeakrefLock { + pub(super) fn lock(&self) -> WeakrefLockGuard { + WeakrefLockGuard + } + } + + pub(super) fn select(_addr: usize) -> WeakrefLock { + WeakrefLock + } + + pub(super) fn lock(addr: usize) -> WeakrefLockGuard { + select(addr).lock() } } @@ -744,7 +774,7 @@ impl WeakRefList { let weak_payload = PyWeak { pointers: Pointers::new(), wr_object: Radium::new(ptr::null_mut()), - referent_addr: obj as *const PyObject as usize, + weakrefs_lock: weakref_lock::select(obj as *const PyObject as usize), callback: UnsafeCell::new(callback), hash: Radium::new(crate::common::hash::SENTINEL), }; @@ -1022,10 +1052,10 @@ pub struct PyWeak { /// Direct pointer to the referent object, null when dead. /// Equivalent to wr_object in PyWeakReference. wr_object: PyAtomic<*mut PyObject>, - /// Immutable key for the original referent stripe, like CPython's - /// weakrefs_lock. Used only for locking, never dereferenced, so the same - /// stripe protects the callback even after wr_object becomes null. - referent_addr: usize, + /// Handle to the original static referent stripe, like CPython's + /// weakrefs_lock. The same stripe protects the callback even after + /// wr_object becomes null and the referent is freed. + weakrefs_lock: weakref_lock::WeakrefLock, /// Owned callback edge, protected by the original referent stripe. /// GC can retain this edge after unlinking the weakref. callback: UnsafeCell>, @@ -1072,7 +1102,7 @@ impl PyWeak { /// Get the owned callback, which GC can retain after clearing the referent. pub(crate) fn get_callback(&self) -> Option { - let _lock = weakref_lock::lock(self.referent_addr); + let _lock = self.weakrefs_lock.lock(); // SAFETY: the original referent stripe protects the callback. unsafe { &*self.callback.get() }.clone() } @@ -1080,7 +1110,7 @@ impl PyWeak { /// Unlink and steal the callback under the original referent stripe. /// Return ownership so callers release it only after unlocking. fn clear(&self) -> Option { - let _lock = weakref_lock::lock(self.referent_addr); + let _lock = self.weakrefs_lock.lock(); // SAFETY: the original referent stripe protects both fields, including // callbacks retained by GC after the weakref was already unlinked. unsafe { @@ -1091,7 +1121,7 @@ impl PyWeak { /// _PyWeakref_ClearRef: unlink without releasing the callback. fn clear_ref(&self) { - let _lock = weakref_lock::lock(self.referent_addr); + let _lock = self.weakrefs_lock.lock(); // SAFETY: the original referent stripe is held. unsafe { self.clear_ref_lock_held() }; } @@ -1132,7 +1162,7 @@ unsafe impl Traverse for PyWeak { // Stripe critical sections never suspend or run Python, so a stopped // thread cannot own this lock. Also synchronize gc.get_referents(), // which traverses while other threads are running. - let _lock = weakref_lock::lock(self.referent_addr); + let _lock = self.weakrefs_lock.lock(); // SAFETY: the original referent stripe protects the callback. unsafe { &*self.callback.get() }.traverse(tracer_fn); } @@ -3673,7 +3703,9 @@ mod tests { PyWeak { pointers: Pointers::new(), wr_object: Radium::new(ptr::null_mut()), - referent_addr: target.as_object() as *const PyObject as usize, + weakrefs_lock: weakref_lock::select( + target.as_object() as *const PyObject as usize + ), callback: UnsafeCell::new(None), hash: Radium::new(crate::common::hash::SENTINEL), },