diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 996b094377a..0e1d8ccf44a 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", "Buffer", + "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -823,6 +823,7 @@ def __eq__(self, other): __reversed__ = None +Mapping.register(frozendict) Mapping.register(mappingproxy) Mapping.register(framelocalsproxy) @@ -1170,3 +1171,13 @@ def __iadd__(self, values): MutableSequence.register(list) MutableSequence.register(bytearray) + +_deprecated_ByteString = globals().pop("ByteString") + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("collections.abc.ByteString", remove=(3, 17)) + globals()["ByteString"] = _deprecated_ByteString + return _deprecated_ByteString + raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}") diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 0aeca1ed7e5..5dbcac19e7a 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -32,6 +32,8 @@ _sys.modules['collections.abc'] = _collections_abc abc = _collections_abc +lazy from copy import copy as _copy +lazy from heapq import nlargest as _nlargest from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -59,8 +61,6 @@ except ImportError: pass -heapq = None # Lazily imported - ################################################################################ ### OrderedDict @@ -328,14 +328,14 @@ def __ior__(self, other): return self def __or__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(self) new.update(other) return new def __ror__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(other) new.update(self) @@ -637,12 +637,7 @@ def most_common(self, n=None): if n is None: return sorted(self.items(), key=_itemgetter(1), reverse=True) - # Lazy import to speedup Python startup time - global heapq - if heapq is None: - import heapq - - return heapq.nlargest(n, self.items(), key=_itemgetter(1)) + return _nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): '''Iterator over elements repeating each as many times as its count. @@ -779,23 +774,27 @@ def __repr__(self): # When the multiplicities are all zero or one, multiset operations # are guaranteed to be equivalent to the corresponding operations # for regular sets. + # # Given counter multisets such as: # cp = Counter(a=1, b=0, c=1) # cq = Counter(c=1, d=0, e=1) + # # The corresponding regular sets would be: # sp = {'a', 'c'} # sq = {'c', 'e'} + # # All of the following relations would hold: - # set(cp + cq) == sp | sq - # set(cp - cq) == sp - sq - # set(cp | cq) == sp | sq - # set(cp & cq) == sp & sq # (cp == cq) == (sp == sq) # (cp != cq) == (sp != sq) # (cp <= cq) == (sp <= sq) # (cp < cq) == (sp < sq) # (cp >= cq) == (sp >= sq) # (cp > cq) == (sp > sq) + # set(cp + cq) == sp | sq + # set(cp - cq) == sp - sq + # set(cp | cq) == sp | sq + # set(cp & cq) == sp & sq + # set(cp ^ cq) == sp ^ sq def __eq__(self, other): 'True if all counts agree. Missing counts are treated as zero.' @@ -908,6 +907,33 @@ def __and__(self, other): result[elem] = newcount return result + def __xor__(self, other): + '''Symmetric difference. Absolute value of count differences. + + The symmetric difference p ^ q is equivalent to: + + (p - q) | (q - p). + + For each element, symmetric difference gives the same result as: + + max(p[elem], q[elem]) - min(p[elem], q[elem]) + + >>> Counter(a=5, b=3, c=2, d=2) ^ Counter(a=1, b=3, c=5, e=1) + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + if not isinstance(other, Counter): + return NotImplemented + result = Counter() + for elem, count in self.items(): + newcount = abs(count - other[elem]) + if newcount: + result[elem] = newcount + for elem, count in other.items(): + if elem not in self and count: + result[elem] = abs(count) + return result + def __pos__(self): 'Adds an empty counter, effectively stripping negative and zero counts' result = Counter() @@ -990,6 +1016,22 @@ def __iand__(self, other): self[elem] = other_count return self._keep_positive() + def __ixor__(self, other): + '''Inplace symmetric difference. Absolute value of count differences. + + >>> c = Counter(a=5, b=3, c=2, d=2) + >>> c ^= Counter(a=1, b=3, c=5, e=1) + >>> c + Counter({'a': 4, 'c': 3, 'd': 2, 'e': 1}) + + ''' + for elem, count in self.items(): + self[elem] = abs(count - other[elem]) + for elem, count in other.items(): + if elem not in self: + self[elem] = abs(count) + return self._keep_positive() + ######################################################################## ### ChainMap @@ -1177,14 +1219,14 @@ def __repr__(self): def __or__(self, other): if isinstance(other, UserDict): return self.__class__(self.data | other.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(self.data | other) return NotImplemented def __ror__(self, other): if isinstance(other, UserDict): return self.__class__(other.data | self.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(other | self.data) return NotImplemented @@ -1205,16 +1247,29 @@ def __copy__(self): def copy(self): if self.__class__ is UserDict: return UserDict(self.data.copy()) - import copy data = self.data try: self.data = {} - c = copy.copy(self) + c = _copy(self) finally: self.data = data c.update(self) return c + + # This method has a default implementation in MutableMapping, but dict's + # equivalent is last-in, first-out instead of first-in, first-out. + def popitem(self): + """Remove and return a (key, value) pair as a 2-tuple. + + Removes pairs in the same order as the wrapped mapping's popitem() + method. For dict objects (the default), that order is last-in, + first-out (LIFO). + Raises KeyError if the UserDict is empty. + """ + return self.data.popitem() + + @classmethod def fromkeys(cls, iterable, value=None): d = cls() diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 5b646fabca0..efc02bfa924 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,11 +1,17 @@ """Utilities for with-statement contexts. See PEP 343.""" + import abc import os import sys import _collections_abc from collections import deque from functools import wraps -from types import MethodType, GenericAlias +lazy from inspect import ( + isasyncgenfunction as _isasyncgenfunction, + iscoroutinefunction as _iscoroutinefunction, + isgeneratorfunction as _isgeneratorfunction, +) +from types import GenericAlias __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", @@ -79,11 +85,37 @@ def _recreate_cm(self): return self def __call__(self, func): - @wraps(func) - def inner(*args, **kwds): - with self._recreate_cm(): - return func(*args, **kwds) - return inner + wrapper = wraps(func) + if _isasyncgenfunction(func): + + async def asyncgen_inner(*args, **kwds): + with self._recreate_cm(): + async with aclosing(func(*args, **kwds)) as gen: + async for value in gen: + yield value + + return wrapper(asyncgen_inner) + elif _iscoroutinefunction(func): + + async def async_inner(*args, **kwds): + with self._recreate_cm(): + return await func(*args, **kwds) + + return wrapper(async_inner) + elif _isgeneratorfunction(func): + + def gen_inner(*args, **kwds): + with self._recreate_cm(), closing(func(*args, **kwds)) as gen: + return (yield from gen) + + return wrapper(gen_inner) + else: + + def inner(*args, **kwds): + with self._recreate_cm(): + return func(*args, **kwds) + + return wrapper(inner) class AsyncContextDecorator(object): @@ -95,11 +127,41 @@ def _recreate_cm(self): return self def __call__(self, func): - @wraps(func) - async def inner(*args, **kwds): - async with self._recreate_cm(): - return await func(*args, **kwds) - return inner + wrapper = wraps(func) + if _isasyncgenfunction(func): + + async def asyncgen_inner(*args, **kwds): + async with ( + self._recreate_cm(), + aclosing(func(*args, **kwds)) as gen + ): + async for value in gen: + yield value + + return wrapper(asyncgen_inner) + elif _iscoroutinefunction(func): + + async def async_inner(*args, **kwds): + async with self._recreate_cm(): + return await func(*args, **kwds) + + return wrapper(async_inner) + elif _isgeneratorfunction(func): + + async def gen_inner(*args, **kwds): + async with self._recreate_cm(): + with closing(func(*args, **kwds)) as gen: + for value in gen: + yield value + + return wrapper(gen_inner) + else: + + async def inner(*args, **kwds): + async with self._recreate_cm(): + return func(*args, **kwds) + + return wrapper(inner) class _GeneratorContextManagerBase: @@ -469,13 +531,23 @@ def __exit__(self, exctype, excinst, exctb): return False +def _lookup_special(obj, name, default): + # Follow the standard lookup behaviour for special methods. + from inspect import getattr_static, _descriptor_get + cls = type(obj) + try: + descr = getattr_static(cls, name) + except AttributeError: + return default + return _descriptor_get(descr, obj) + + +_sentinel = ['SENTINEL'] + + class _BaseExitStack: """A base class for ExitStack and AsyncExitStack.""" - @staticmethod - def _create_exit_wrapper(cm, cm_exit): - return MethodType(cm_exit, cm) - @staticmethod def _create_cb_wrapper(callback, /, *args, **kwds): def _exit_wrapper(exc_type, exc, tb): @@ -499,17 +571,8 @@ def push(self, exit): Also accepts any object with an __exit__ method (registering a call to the method instead of the object itself). """ - # We use an unbound method rather than a bound method to follow - # the standard lookup behaviour for special methods. - _cb_type = type(exit) - - try: - exit_method = _cb_type.__exit__ - except AttributeError: - # Not a context manager, so assume it's a callable. - self._push_exit_callback(exit) - else: - self._push_cm_exit(exit, exit_method) + exit_method = _lookup_special(exit, '__exit__', exit) + self._push_exit_callback(exit_method) return exit # Allow use as a decorator. def enter_context(self, cm): @@ -518,17 +581,18 @@ def enter_context(self, cm): If successful, also pushes its __exit__ method as a callback and returns the result of the __enter__ method. """ - # We look up the special methods on the type to match the with - # statement. - cls = type(cm) - try: - _enter = cls.__enter__ - _exit = cls.__exit__ - except AttributeError: + _enter = _lookup_special(cm, '__enter__', _sentinel) + if _enter is _sentinel: + cls = type(cm) raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " - f"not support the context manager protocol") from None - result = _enter(cm) - self._push_cm_exit(cm, _exit) + f"not support the context manager protocol") + _exit = _lookup_special(cm, '__exit__', _sentinel) + if _exit is _sentinel: + cls = type(cm) + raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " + f"not support the context manager protocol") + result = _enter() + self._push_exit_callback(_exit) return result def callback(self, callback, /, *args, **kwds): @@ -544,11 +608,6 @@ def callback(self, callback, /, *args, **kwds): self._push_exit_callback(_exit_wrapper) return callback # Allow use as a decorator - def _push_cm_exit(self, cm, cm_exit): - """Helper to correctly register callbacks to __exit__ methods.""" - _exit_wrapper = self._create_exit_wrapper(cm, cm_exit) - self._push_exit_callback(_exit_wrapper, True) - def _push_exit_callback(self, callback, is_sync=True): self._exit_callbacks.append((is_sync, callback)) @@ -641,10 +700,6 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): # connection later in the list raise an exception. """ - @staticmethod - def _create_async_exit_wrapper(cm, cm_exit): - return MethodType(cm_exit, cm) - @staticmethod def _create_async_cb_wrapper(callback, /, *args, **kwds): async def _exit_wrapper(exc_type, exc, tb): @@ -657,16 +712,18 @@ async def enter_async_context(self, cm): If successful, also pushes its __aexit__ method as a callback and returns the result of the __aenter__ method. """ - cls = type(cm) - try: - _enter = cls.__aenter__ - _exit = cls.__aexit__ - except AttributeError: + _enter = _lookup_special(cm, '__aenter__', _sentinel) + if _enter is _sentinel: + cls = type(cm) raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " - f"not support the asynchronous context manager protocol" - ) from None - result = await _enter(cm) - self._push_async_cm_exit(cm, _exit) + f"not support the asynchronous context manager protocol") + _exit = _lookup_special(cm, '__aexit__', _sentinel) + if _exit is _sentinel: + cls = type(cm) + raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does " + f"not support the asynchronous context manager protocol") + result = await _enter() + self._push_exit_callback(_exit, False) return result def push_async_exit(self, exit): @@ -677,14 +734,8 @@ def push_async_exit(self, exit): Also accepts any object with an __aexit__ method (registering a call to the method instead of the object itself). """ - _cb_type = type(exit) - try: - exit_method = _cb_type.__aexit__ - except AttributeError: - # Not an async context manager, so assume it's a coroutine function - self._push_exit_callback(exit, False) - else: - self._push_async_cm_exit(exit, exit_method) + exit_method = _lookup_special(exit, '__aexit__', exit) + self._push_exit_callback(exit_method, False) return exit # Allow use as a decorator def push_async_callback(self, callback, /, *args, **kwds): @@ -704,12 +755,6 @@ async def aclose(self): """Immediately unwind the context stack.""" await self.__aexit__(None, None, None) - def _push_async_cm_exit(self, cm, cm_exit): - """Helper to correctly register coroutine function to __aexit__ - method.""" - _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit) - self._push_exit_callback(_exit_wrapper, False) - async def __aenter__(self): return self diff --git a/Lib/difflib.py b/Lib/difflib.py index aef2b6f0fd8..95ba8fd782c 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -33,6 +33,7 @@ from heapq import nlargest as _nlargest from collections import namedtuple as _namedtuple from types import GenericAlias +lazy from _colorize import can_colorize, get_theme Match = _namedtuple('Match', 'a b size') @@ -637,15 +638,15 @@ def quick_ratio(self): # avail[x] is the number of times x appears in 'b' less the # number of times we've seen it in 'a' so far ... kinda avail = {} - availhas, matches = avail.__contains__, 0 + matches = 0 for elt in self.a: - if availhas(elt): + if elt in avail: numb = avail[elt] else: numb = fullbcount.get(elt, 0) avail[elt] = numb - 1 if numb > 0: - matches = matches + 1 + matches += 1 return _calculate_ratio(matches, len(self.a) + len(self.b)) def real_quick_ratio(self): @@ -701,10 +702,12 @@ def get_close_matches(word, possibilities, n=3, cutoff=0.6): s.set_seq2(word) for x in possibilities: s.set_seq1(x) - if s.real_quick_ratio() >= cutoff and \ - s.quick_ratio() >= cutoff and \ - s.ratio() >= cutoff: - result.append((s.ratio(), x)) + if s.real_quick_ratio() < cutoff or s.quick_ratio() < cutoff: + continue + + ratio = s.ratio() + if ratio >= cutoff: + result.append((ratio, x)) # Move the best scorers to head of list result = _nlargest(n, result) @@ -939,10 +942,12 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): cruncher.set_seq1(a[i]) # Ordering by cheapest to most expensive ratio is very # valuable, most often getting out early. - if (crqr() > best_ratio - and cqr() > best_ratio - and cr() > best_ratio): - best_i, best_j, best_ratio = i, j, cr() + if crqr() <= best_ratio or cqr() <= best_ratio: + continue + + ratio = cr() + if ratio > best_ratio: + best_i, best_j, best_ratio = i, j, ratio if best_i is None: # found nothing to synch on yet - move to next j @@ -1094,7 +1099,7 @@ def _format_range_unified(start, stop): return '{},{}'.format(beginning, length) def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', - tofiledate='', n=3, lineterm='\n'): + tofiledate='', n=3, lineterm='\n', *, color=False): r""" Compare two sequences of lines; generate the delta as a unified diff. @@ -1111,6 +1116,10 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', For inputs that do not have trailing newlines, set the lineterm argument to "" so that the output will be uniformly newline free. + Set 'color' to True to enable output in color, similar to + 'git diff --color'. Even if enabled, it can be + controlled using environment variables such as 'NO_COLOR'. + The unidiff format normally has a header for filenames and modification times. Any or all of these may be specified using strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. @@ -1134,6 +1143,11 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', four """ + if color and can_colorize(): + t = get_theme(force_color=True).difflib + else: + t = get_theme(force_no_color=True).difflib + _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) started = False for group in SequenceMatcher(None,a,b).get_grouped_opcodes(n): @@ -1141,25 +1155,25 @@ def unified_diff(a, b, fromfile='', tofile='', fromfiledate='', started = True fromdate = '\t{}'.format(fromfiledate) if fromfiledate else '' todate = '\t{}'.format(tofiledate) if tofiledate else '' - yield '--- {}{}{}'.format(fromfile, fromdate, lineterm) - yield '+++ {}{}{}'.format(tofile, todate, lineterm) + yield f'{t.header}--- {fromfile}{fromdate}{lineterm}{t.reset}' + yield f'{t.header}+++ {tofile}{todate}{lineterm}{t.reset}' first, last = group[0], group[-1] file1_range = _format_range_unified(first[1], last[2]) file2_range = _format_range_unified(first[3], last[4]) - yield '@@ -{} +{} @@{}'.format(file1_range, file2_range, lineterm) + yield f'{t.hunk}@@ -{file1_range} +{file2_range} @@{lineterm}{t.reset}' for tag, i1, i2, j1, j2 in group: if tag == 'equal': for line in a[i1:i2]: - yield ' ' + line + yield f'{t.context} {line}{t.reset}' continue if tag in {'replace', 'delete'}: for line in a[i1:i2]: - yield '-' + line + yield f'{t.removed}-{line}{t.reset}' if tag in {'replace', 'insert'}: for line in b[j1:j2]: - yield '+' + line + yield f'{t.added}+{line}{t.reset}' ######################################################################## @@ -1615,16 +1629,13 @@ def _line_pair_iterator(): _file_template = """ - - - - + + - - - @@ -1636,13 +1647,36 @@ def _line_pair_iterator(): _styles = """ :root {color-scheme: light dark} - table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium} - .diff_header {background-color:#e0e0e0} - td.diff_header {text-align:right} - .diff_next {background-color:#c0c0c0} + table.diff { + font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; + border: medium; + } + .diff_header { + background-color: #e0e0e0; + font-weight: bold; + } + td.diff_header { + text-align: right; + padding: 0 8px; + } + .diff_next { + background-color: #c0c0c0; + padding: 4px 0; + } .diff_add {background-color:palegreen} .diff_chg {background-color:#ffff77} .diff_sub {background-color:#ffaaaa} + table.diff[summary="Legends"] { + margin-top: 20px; + border: 1px solid #ccc; + } + table.diff[summary="Legends"] th { + background-color: #e0e0e0; + padding: 4px 8px; + } + table.diff[summary="Legends"] td { + padding: 4px 8px; + } @media (prefers-color-scheme: dark) { .diff_header {background-color:#666} @@ -1650,6 +1684,8 @@ def _line_pair_iterator(): .diff_add {background-color:darkgreen} .diff_chg {background-color:#847415} .diff_sub {background-color:darkred} + table.diff[summary="Legends"] {border-color:#555} + table.diff[summary="Legends"] th{background-color:#666} }""" _table_template = """ @@ -1692,7 +1728,7 @@ class HtmlDiff(object): make_table -- generates HTML for a single side by side table make_file -- generates complete HTML file with a single side by side table - See tools/scripts/diff.py for an example usage of this class. + See Doc/includes/diff.py for an example usage of this class. """ _file_template = _file_template @@ -1892,8 +1928,11 @@ def _format_line(self,side,flag,linenum,text): # make space non-breakable so they don't get compressed or line wrapped text = text.replace(' ',' ').rstrip() - return '%s%s' \ - % (id,linenum,text) + # add a class to the td tag if there is a difference on the line + css_class = ' class="diff_changed" ' if flag else ' ' + + return f'{linenum}' \ + + f'{text}' def _make_prefix(self): """Create unique anchor prefixes""" diff --git a/Lib/doctest.py b/Lib/doctest.py index a66888d8fc9..7311ce4c926 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -102,15 +102,14 @@ def _test(): import re import sys import traceback +import types import unittest -from io import StringIO, IncrementalNewlineDecoder +from io import StringIO, TextIOWrapper, BytesIO from collections import namedtuple -import _colorize # Used in doctests -from _colorize import ANSIColors, can_colorize +lazy import _colorize # Used in doctests +lazy from _colorize import ANSIColors, can_colorize -__unittest = True - class TestResults(namedtuple('TestResults', 'failed attempted')): def __new__(cls, failed, attempted, *, skipped=0): results = super().__new__(cls, failed, attempted) @@ -238,10 +237,6 @@ def _normalize_module(module, depth=2): else: raise TypeError("Expected a module, string, or None") -def _newline_convert(data): - # The IO module provides a handy decoder for universal newline conversion - return IncrementalNewlineDecoder(None, True).decode(data, True) - def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -253,10 +248,9 @@ def _load_testfile(filename, package, module_relative, encoding): pass if hasattr(loader, 'get_data'): file_contents = loader.get_data(filename) - file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + return TextIOWrapper(BytesIO(file_contents), encoding=encoding, newline=None).read(), filename with open(filename, encoding=encoding) as f: return f.read(), filename @@ -388,7 +382,7 @@ def __init__(self, out): self.__out = out self.__debugger_used = False # do not play signal games in the pdb - pdb.Pdb.__init__(self, stdout=out, nosigint=True) + super().__init__(stdout=out, nosigint=True) # still use input() to get user input self.use_rawinput = 1 @@ -928,7 +922,7 @@ def find(self, obj, name=None, module=None, globs=None, extraglobs=None): # given object's docstring. try: file = inspect.getsourcefile(obj) - except TypeError: + except (TypeError, OSError): source_lines = None else: if not file: @@ -1173,6 +1167,32 @@ def _find_lineno(self, obj, source_lines): if pat.match(source_lines[lineno]): return lineno + # Handle __test__ string doctests formatted as triple-quoted + # strings. Find a non-blank line in the test string and match it + # in the source, verifying subsequent lines also match to handle + # duplicate lines. + if isinstance(obj, str) and source_lines is not None: + obj_lines = obj.splitlines(keepends=True) + # Skip the first line (may be on same line as opening quotes) + # and any blank lines to find a meaningful line to match. + start_index = 1 + while (start_index < len(obj_lines) + and not obj_lines[start_index].strip()): + start_index += 1 + if start_index < len(obj_lines): + target_line = obj_lines[start_index] + for lineno, source_line in enumerate(source_lines): + if source_line == target_line: + # Verify subsequent lines also match + for i in range(start_index + 1, len(obj_lines) - 1): + source_idx = lineno + i - start_index + if source_idx >= len(source_lines): + break + if obj_lines[i] != source_lines[source_idx]: + break + else: + return lineno - start_index + # We couldn't find the line number. return None @@ -1283,6 +1303,11 @@ def __init__(self, checker=None, verbose=None, optionflags=0): # Reporting methods #///////////////////////////////////////////////////////////////// + def report_skip(self, out, test, example): + """ + Report that the given example was skipped. + """ + def report_start(self, out, test, example): """ Report that the test runner is about to process the given @@ -1380,6 +1405,8 @@ def __run(self, test, compileflags, out): # If 'SKIP' is set, then skip this example. if self.optionflags & SKIP: + if not quiet: + self.report_skip(out, test, example) skips += 1 continue @@ -2277,12 +2304,63 @@ def set_unittest_reportflags(flags): return old +class _DocTestCaseRunner(DocTestRunner): + + def __init__(self, *args, test_case, test_result, **kwargs): + super().__init__(*args, **kwargs) + self._test_case = test_case + self._test_result = test_result + self._examplenum = 0 + + def _subTest(self): + subtest = unittest.case._SubTest(self._test_case, str(self._examplenum), {}) + self._examplenum += 1 + return subtest + + def report_skip(self, out, test, example): + unittest.case._addSkip(self._test_result, self._subTest(), '') + + def report_success(self, out, test, example, got): + self._test_result.addSubTest(self._test_case, self._subTest(), None) + + def report_unexpected_exception(self, out, test, example, exc_info): + tb = self._add_traceback(exc_info[2], test, example) + exc_info = (*exc_info[:2], tb) + self._test_result.addSubTest(self._test_case, self._subTest(), exc_info) + + def report_failure(self, out, test, example, got): + msg = ('Failed example:\n' + _indent(example.source) + + self._checker.output_difference(example, got, self.optionflags).rstrip('\n')) + exc = self._test_case.failureException(msg) + tb = self._add_traceback(None, test, example) + exc_info = (type(exc), exc, tb) + self._test_result.addSubTest(self._test_case, self._subTest(), exc_info) + + def _add_traceback(self, traceback, test, example): + if test.lineno is None or example.lineno is None: + lineno = None + else: + lineno = test.lineno + example.lineno + 1 + return types.SimpleNamespace( + tb_frame = types.SimpleNamespace( + f_globals=test.globs, + f_code=types.SimpleNamespace( + co_filename=test.filename, + co_name=test.name, + ), + ), + tb_next = traceback, + tb_lasti = -1, + tb_lineno = lineno, + ) + + class DocTestCase(unittest.TestCase): def __init__(self, test, optionflags=0, setUp=None, tearDown=None, checker=None): - unittest.TestCase.__init__(self) + super().__init__() self._dt_optionflags = optionflags self._dt_checker = checker self._dt_test = test @@ -2306,30 +2384,28 @@ def tearDown(self): test.globs.clear() test.globs.update(self._dt_globs) + def run(self, result=None): + self._test_result = result + return super().run(result) + def runTest(self): test = self._dt_test - old = sys.stdout - new = StringIO() optionflags = self._dt_optionflags + result = self._test_result if not (optionflags & REPORTING_FLAGS): # The option flags don't include any reporting flags, # so add the default reporting flags optionflags |= _unittest_reportflags + if getattr(result, 'failfast', False): + optionflags |= FAIL_FAST - runner = DocTestRunner(optionflags=optionflags, - checker=self._dt_checker, verbose=False) - - try: - runner.DIVIDER = "-"*70 - results = runner.run(test, out=new.write, clear_globs=False) - if results.skipped == results.attempted: - raise unittest.SkipTest("all examples were skipped") - finally: - sys.stdout = old - - if results.failed: - raise self.failureException(self.format_failure(new.getvalue().rstrip('\n'))) + runner = _DocTestCaseRunner(optionflags=optionflags, + checker=self._dt_checker, verbose=False, + test_case=self, test_result=result) + results = runner.run(test, clear_globs=False) + if results.skipped == results.attempted: + raise unittest.SkipTest("all examples were skipped") def format_failure(self, err): test = self._dt_test @@ -2444,7 +2520,7 @@ def shortDescription(self): class SkipDocTestCase(DocTestCase): def __init__(self, module): self.module = module - DocTestCase.__init__(self, None) + super().__init__(None) def setUp(self): self.skipTest("DocTestSuite will not work with -O2 and above") @@ -2875,7 +2951,7 @@ def get(self): def _test(): import argparse - parser = argparse.ArgumentParser(description="doctest runner", color=True) + parser = argparse.ArgumentParser(description="doctest runner") parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print very verbose output for all tests') parser.add_argument('-o', '--option', action='append', @@ -2885,8 +2961,8 @@ def _test(): ' than once to apply multiple options')) parser.add_argument('-f', '--fail-fast', action='store_true', help=('stop running tests after first failure (this' - ' is a shorthand for -o FAIL_FAST, and is' - ' in addition to any other -o options)')) + ' is a shorthand for `-o FAIL_FAST`, and is' + ' in addition to any other `-o` options)')) parser.add_argument('file', nargs='+', help='file containing the tests to run') args = parser.parse_args() diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 2c404f6d80b..64c035307e6 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -15,73 +15,73 @@ Run Python regression tests. If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). +the pattern `test_*` in the `Lib/test` subdirectory and runs +them in alphabetical order (but see `-M` and `-u`, below, for exceptions). For more rigorous testing, it is useful to use the following command line: -python -E -Wd -m test [options] [test_name1 ...] +`python -E -Wd -m test [options] [test_name1 ...]` """ EPILOG = """\ Additional option details: --r randomizes test execution order. You can use --randseed=int to provide an -int seed value for the randomizer. The randseed value will be used +`-r` randomizes test execution order. You can use `--randseed=int` to provide an +int seed value for the randomizer. The `randseed` value will be used to set seeds for all random usages in tests -(including randomizing the tests order if -r is set). +(including randomizing the tests order if `-r` is set). By default we always set random seed, but do not randomize test order. --s On the first invocation of regrtest using -s, the first test file found +`-s` On the first invocation of regrtest using `-s`, the first test file found or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it +the next test is recorded in a file named `pynexttest`. If run from the +Python build directory, `pynexttest` is located in the 'build' subdirectory, +otherwise it is located in `tempfile.gettempdir()`. On subsequent runs, +the test in `pynexttest` is run, and the next test is written to `pynexttest`. +When the last test has been run, `pynexttest` is deleted. In this way it is possible to single step through the test files. This is useful when doing memory analysis on the Python interpreter, which process tends to consume too many resources to run the full regression test non-stop. --S is used to resume running tests after an interrupted run. It will -maintain the order a standard run (i.e. it assumes -r is not used). +`-S` is used to resume running tests after an interrupted run. It will +maintain the order a standard run (i.e. it assumes `-r` is not used). This is useful after the tests have prematurely stopped for some external reason and you want to resume the run from where you left off rather -than starting from the beginning. Note: this is different from --prioritize. +than starting from the beginning. Note: this is different from `--prioritize`. ---prioritize is used to influence the order of selected tests, such that +`--prioritize` is used to influence the order of selected tests, such that the tests listed as an argument are executed first. This is especially -useful when combined with -j and -r to pin the longest-running tests -to start at the beginning of a test run. Pass --prioritize=test_a,test_b -to make test_a run first, followed by test_b, and then the other tests. -If test_a wasn't selected for execution by regular means, --prioritize will +useful when combined with `-j` and `-r` to pin the longest-running tests +to start at the beginning of a test run. Pass `--prioritize=test_a,test_b` +to make `test_a` run first, followed by `test_b`, and then the other tests. +If test_a wasn't selected for execution by regular means, `--prioritize` will not make it execute. --f reads the names of tests from the file given as f's argument, one +`-f` reads the names of tests from the file given as `f`'s argument, one or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for +lines beginning with `#` are ignored. This is especially useful for whittling down failures involving interactions among tests. --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other +`-L` causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on macOS and presumably on some other FreeBSD-derived systems. --R runs each test several times and examines sys.gettotalrefcount() to +`-R` runs each test several times and examines `sys.gettotalrefcount()` to see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. +be of the form `stab:run:fname` where `stab` is the number of times the +test is run to let gettotalrefcount settle down, `run` is the number +of times further it is run and `fname` is the name of the file the +reports are written to. These parameters all have defaults (`5`, `4` and +`"reflog.txt"` respectively), and the minimal invocation is `-R :`. --M runs tests that require an exorbitant amount of memory. These tests +`-M` runs tests that require an exorbitant amount of memory. These tests typically try to ascertain containers keep working when containing more than 2 billion objects, which only works on 64-bit systems. There are also some tests that try to exhaust the address space of the process, which only makes sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines how much memory the -tests will limit themselves to (but they may go slightly over.) The number +which is a string in the form of `'2.5Gb'`, determines how much memory the +tests will limit themselves to (but they may go slightly over). The number shouldn't be more memory than the machine has (including swap memory). You should also keep in mind that swap memory is generally much, much slower than RAM, and setting memlimit to all available RAM or higher will heavily @@ -90,7 +90,7 @@ to use more than memlimit memory will be skipped. The big-memory tests generally run very, very long. --u is used to specify which special resource intensive tests to run, +`-u` is used to specify which special resource intensive tests to run, such as those requiring large file support or network connectivity. The argument is a comma-separated list of words indicating the resources to test. Currently only the following are defined: @@ -137,16 +137,16 @@ wantobjects - Allows to run Tkinter tests with the specified value of tkinter.wantobjects. -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. +To enable all resources except one, use `-uall,-`. For +example, to run all the tests except for the `gui` tests, give the +option `-uall,-gui`. ---matchfile filters tests using a text file, one pattern per line. +`--matchfile` filters tests using a text file, one pattern per line. Pattern examples: -- test method: test_stat_attributes -- test class: FileTests -- test identifier: test_os.FileTests.test_stat_attributes +- test method: `test_stat_attributes` +- test class: `FileTests` +- test identifier: `test_os.FileTests.test_stat_attributes` """ @@ -245,7 +245,7 @@ def _create_parser(): 'buildbot workers') group.add_argument('--timeout', metavar='TIMEOUT', help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'more than `TIMEOUT` seconds; disabled if `TIMEOUT` ' 'is negative or equals to zero') group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' @@ -261,11 +261,11 @@ def _create_parser(): group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') + help='run tests in verbose mode with output to `stdout`') group.add_argument('-w', '--rerun', action='store_true', help='re-run failed tests in verbose mode') group.add_argument('--verbose2', action='store_true', dest='rerun', - help='deprecated alias to --rerun') + help='deprecated alias to `--rerun`') group.add_argument('-W', '--verbose3', action='store_true', help='display test output on failure') group.add_argument('-q', '--quiet', action='store_true', @@ -295,22 +295,22 @@ def _create_parser(): more_details) group.add_argument('-m', '--match', metavar='PAT', dest='match_tests', action=FilterAction, const=True, - help='match test cases and methods with glob pattern PAT') + help='match test cases and methods with glob pattern `PAT`') group.add_argument('-i', '--ignore', metavar='PAT', dest='match_tests', action=FilterAction, const=False, - help='ignore test cases and methods with glob pattern PAT') + help='ignore test cases and methods with glob pattern `PAT`') group.add_argument('--matchfile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=True, - help='similar to --match but get patterns from a ' + help='similar to `--match` but get patterns from a ' 'text file, one pattern per line') group.add_argument('--ignorefile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=False, - help='similar to --matchfile but it receives patterns ' + help='similar to `--matchfile` but it receives patterns ' 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') + help='fail as soon as a test fails (only with `-v` or `-W`)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', action='extend', type=resources_list, help='specify which special resource intensive tests ' @@ -325,7 +325,7 @@ def _create_parser(): group = parser.add_argument_group('Special runs') group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + + help='run the `leaks(1)` command just before exit.' + more_details) group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', type=huntrleaks, @@ -333,20 +333,20 @@ def _create_parser(): 'very slow).' + more_details) group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, - help='run PROCESSES processes at once') + help='run `PROCESSES` processes at once') group.add_argument('--single-process', action='store_true', dest='single_process', help='always run all tests sequentially in ' - 'a single process, ignore -jN option, ' + 'a single process, ignore `-jN` option, ' 'and failed tests are also rerun sequentially ' 'in the same process') group.add_argument('--parallel-threads', metavar='PARALLEL_THREADS', type=int, - help='run copies of each test in PARALLEL_THREADS at ' + help='run copies of each test in `PARALLEL_THREADS` at ' 'once') group.add_argument('-T', '--coverage', action='store_true', dest='trace', - help='turn on code coverage tracing using the trace ' + help='turn on code coverage tracing using the `trace` ' 'module') group.add_argument('-D', '--coverdir', metavar='DIR', type=relative_filename, @@ -356,18 +356,18 @@ def _create_parser(): help='put coverage files alongside modules') group.add_argument('-t', '--threshold', metavar='THRESHOLD', type=int, - help='call gc.set_threshold(THRESHOLD)') + help='call `gc.set_threshold(THRESHOLD)`') group.add_argument('-n', '--nowindows', action='store_true', help='suppress error message boxes on Windows') group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' - 'error happens; imply --failfast') + 'error happens; imply `--failfast`') group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") group.add_argument('--list-cases', action='store_true', - help='only write the name of test cases that will be run' - ' , don\'t execute them') + help='only write the name of test cases that will be run, ' + 'don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', @@ -390,9 +390,11 @@ def _create_parser(): group.add_argument('--tempdir', metavar='PATH', help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', - help='remove old test_python_* directories') + help='remove old `test_python_*` directories') group.add_argument('--bisect', action='store_true', - help='if some tests fail, run test.bisect_cmd on them') + help='if some tests fail, run `test.bisect_cmd` on them') + group.add_argument('--pythoninfo', action='store_true', + help="run `python -m test.pythoninfo` before tests") group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index d8b9605ea49..ac82541059c 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -26,7 +26,7 @@ strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, - is_cross_compiled, get_host_runner, + is_cross_compiled, get_host_runner, display_title, EXIT_TIMEOUT) @@ -126,6 +126,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir self._tmp_dir: StrPath | None = ns.tempdir + self.pythoninfo: bool = ns.pythoninfo # Randomize self.randomize: bool = ns.randomize @@ -322,9 +323,7 @@ def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: title = f"Bisect {test}" if progress: title = f"{title} ({progress})" - print(title) - print("#" * len(title)) - print() + display_title(title) cmd = runtests.create_python_cmd() cmd.extend([ @@ -345,9 +344,7 @@ def _run_bisect(self, runtests: RunTests, test: str, progress: str) -> bool: exitcode = proc.returncode title = f"{title}: exit code {exitcode}" - print(title) - print("#" * len(title)) - print(flush=True) + display_title(title) if exitcode: print(f"Bisect failed with exit code {exitcode}") @@ -752,6 +749,15 @@ def tmp_dir(self) -> StrPath: ) return self._tmp_dir + def run_pythoninfo(self): + from test import pythoninfo + try: + pythoninfo.main() + except SystemExit: + # Ignore non-zero exit code on purpose + pass + print() + def main(self, tests: TestList | None = None) -> NoReturn: if self.want_add_python_opts: self._add_python_opts() @@ -765,6 +771,9 @@ def main(self, tests: TestList | None = None) -> NoReturn: if self.want_wait: input("Press any key to continue...") + if self.pythoninfo: + self.run_pythoninfo() + setup_test_dir(self.test_dir) selected, tests = self.find_tests(tests) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 15caba75acf..99e47cfabf2 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -118,7 +118,8 @@ def printlist(x, width=70, indent=4, file=None): blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks), + initial_indent=blanks, subsequent_indent=blanks, + break_long_words=False, break_on_hyphens=False), file=file) @@ -294,6 +295,25 @@ def clear_caches(): else: importlib_metadata.FastPath.__new__.cache_clear() + try: + encodings = sys.modules['encodings'] + except KeyError: + pass + else: + encodings._cache.clear() + + try: + codecs = sys.modules['codecs'] + except KeyError: + pass + else: + # There's no direct API to clear the codecs search cache, but + # `unregister` clears it implicitly. + def noop_search_function(name): + return None + codecs.register(noop_search_function) + codecs.unregister(noop_search_function) + def get_build_info(): # Get most important configure and build options as a list of strings. @@ -727,3 +747,9 @@ def _sanitize_xml_replace(regs): def sanitize_xml(text: str) -> str: return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) + + +def display_title(title): + print(title) + print("#" * len(title)) + print(flush=True) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 19b336ba96f..a870b3e5243 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -8,6 +8,12 @@ import warnings +MS_WINDOWS = (sys.platform == "win32") +APPLE = (sys.platform in ("darwin", "ios", "tvos", "watchos")) + +COMMAND_TIMEOUT = 60.0 + + def normalize_text(text): if text is None: return None @@ -16,6 +22,25 @@ def normalize_text(text): return text.strip() +def first_line(text): + # Get the first line. Return text unchanged if it's empty. + lines = text.splitlines() + if lines: + return lines[0] + else: + # text is an empty string + return text + + +def read_first_line(filename): + # Get the first line of a text file and strip trailing spaces + try: + with open(filename, encoding="utf-8") as fp: + return fp.readline().rstrip() + except OSError: + return '' + + class PythonInfo: def __init__(self): self.info = {} @@ -146,6 +171,8 @@ def collect_sys(info_add): text = 'No (sys.getobjects() missing)' info_add('build.Py_TRACE_REFS', text) + info_add('sys.is_remote_debug_enabled', sys.is_remote_debug_enabled()) + def collect_platform(info_add): import platform @@ -279,9 +306,11 @@ def format_groups(groups): "BUILDPYTHON", "CC", "CFLAGS", + "CI", "COLUMNS", "COMPUTERNAME", "COMSPEC", + "CONTAINER", "CPP", "CPPFLAGS", "DISPLAY", @@ -289,11 +318,14 @@ def format_groups(groups): "DISTUTILS_USE_SDK", "DYLD_LIBRARY_PATH", "ENSUREPIP_OPTIONS", + "FORCE_COLOR", + "GITHUB_ACTIONS", "HISTORY_FILE", "HOME", "HOMEDRIVE", "HOMEPATH", "IDLESTARTUP", + "IMAGE_OS_VERSION", "IPHONEOS_DEPLOYMENT_TARGET", "LANG", "LDFLAGS", @@ -305,11 +337,13 @@ def format_groups(groups): "MAKEFLAGS", "MIXERDEV", "MSSDK", + "NO_COLOR", "PATH", "PATHEXT", "PIP_CONFIG_FILE", "PLAT", "POSIXLY_CORRECT", + "PYTHON_COLORS", "PY_SAX_PARSER", "ProgramFiles", "ProgramFiles(x86)", @@ -416,24 +450,53 @@ def format_attr(attr, value): info_add('readline.library', 'GNU readline') -def collect_gdb(info_add): +def run_command(cmd, check=True, **kwargs): import subprocess + from test.support import has_subprocess_support + + if not has_subprocess_support: + # subprocess is not supported by the current platform + return '' + timeout = COMMAND_TIMEOUT + + cmd_str = ' '.join(cmd) try: - proc = subprocess.Popen(["gdb", "-nx", "--version"], + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - version = proc.communicate()[0] - if proc.returncode: - # ignore gdb failure: test_gdb will log the error - return - except OSError: - return + stderr=subprocess.DEVNULL, + text=True, + **kwargs) + with proc: + try: + stdout = proc.communicate(timeout=timeout)[0] + except: + proc.kill() + proc.communicate() + raise + + if check and proc.returncode: + print(f"Command {cmd_str} failed with exit code {proc.returncode}") + return '' - # Only keep the first line - version = version.splitlines()[0] - info_add('gdb_version', version) + # Strip trailing spaces and newlines + stdout = stdout.rstrip() + return stdout + except FileNotFoundError: + return '' + except OSError as exc: + print(f"Command {cmd_str} failed with: {exc!r}") + return '' + except subprocess.TimeoutExpired: + print(f"Command {cmd_str}: timeout!") + return '' + + +def collect_gdb(info_add): + version = run_command(["gdb", "-nx", "--version"]) + if version: + # Only keep the first line + info_add('gdb_version', first_line(version)) def collect_tkinter(info_add): @@ -507,6 +570,7 @@ def collect_sysconfig(info_add): for name in ( 'ABIFLAGS', + 'ALT_SOABI', 'ANDROID_API_LEVEL', 'CC', 'CCSHARED', @@ -522,15 +586,19 @@ def collect_sysconfig(info_add): 'PY_CFLAGS', 'PY_CFLAGS_NODIST', 'PY_CORE_LDFLAGS', + 'PY_CORE_EXE_LDFLAGS', 'PY_LDFLAGS', 'PY_LDFLAGS_NODIST', 'PY_STDMODULE_CFLAGS', 'Py_DEBUG', 'Py_ENABLE_SHARED', 'Py_GIL_DISABLED', + 'Py_REMOTE_DEBUG', 'SHELL', 'SOABI', + 'SOABI_PLATFORM', 'TEST_MODULES', + 'VAPTH', 'abs_builddir', 'abs_srcdir', 'prefix', @@ -554,7 +622,6 @@ def collect_sysconfig(info_add): for name in ( 'WITH_DOC_STRINGS', 'WITH_DTRACE', - 'WITH_FREELISTS', 'WITH_MIMALLOC', 'WITH_PYMALLOC', 'WITH_VALGRIND', @@ -652,10 +719,20 @@ def collect_zlib(info_add): except ImportError: return - attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION') + attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION', 'ZLIBNG_VERSION') copy_attributes(info_add, zlib, 'zlib.%s', attributes) +def collect_zstd(info_add): + try: + import _zstd + except ImportError: + return + + attributes = ('zstd_version',) + copy_attributes(info_add, _zstd, 'zstd.%s', attributes) + + def collect_expat(info_add): try: from xml.parsers import expat @@ -685,7 +762,6 @@ def collect_testcapi(info_add): for name in ( 'LONG_MAX', # always 32-bit on Windows, 64-bit on 64-bit Unix 'PY_SSIZE_T_MAX', - 'Py_C_RECURSION_LIMIT', 'SIZEOF_TIME_T', # 32-bit or 64-bit depending on the platform 'SIZEOF_WCHAR_T', # 16-bit or 32-bit depending on the platform ): @@ -734,6 +810,10 @@ def collect_test_socket(info_add): if name.startswith('HAVE_')] copy_attributes(info_add, test_socket, 'test_socket.%s', attributes) + # Get IOCTL_VM_SOCKETS_GET_LOCAL_CID of /dev/vsock + cid = test_socket.get_cid() + info_add('test_socket.get_cid', cid) + def collect_support(info_add): try: @@ -751,6 +831,7 @@ def collect_support(info_add): 'is_emscripten', 'is_jython', 'is_wasi', + 'is_wasm32', ) copy_attributes(info_add, support, 'support.%s', attributes) @@ -813,7 +894,6 @@ def collect_support_threading_helper(info_add): def collect_cc(info_add): - import subprocess import sysconfig CC = sysconfig.get_config_var('CC') @@ -826,23 +906,17 @@ def collect_cc(info_add): except ImportError: args = CC.split() args.append('--version') - try: - proc = subprocess.Popen(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) - except OSError: + + stdout = run_command(args) + if not stdout: # Cannot run the compiler, for example when Python has been # cross-compiled and installed on the target platform where the # compiler is missing. - return - - stdout = proc.communicate()[0] - if proc.returncode: + # # CC --version failed: ignore error return - text = stdout.splitlines()[0] + text = first_line(stdout) text = normalize_text(text) info_add('CC.version', text) @@ -875,8 +949,48 @@ def collect_subprocess(info_add): copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',)) +def winreg_query(path): + try: + import winreg + except ImportError: + return None + + key, path = path.split('\\', 1) + sub_key, value = path.rsplit('\\', 1) + if key == "HKEY_LOCAL_MACHINE": + key = winreg.HKEY_LOCAL_MACHINE + else: + raise ValueError(f"unknown key {key!r}") + + try: + access = winreg.KEY_READ | winreg.KEY_WOW64_64KEY + with winreg.OpenKey(key, sub_key, access=access) as key_handle: + result, _ = winreg.QueryValueEx(key_handle, value) + return result + except OSError: + return None + + +def wmi_query(query): + try: + import _wmi + except ImportError: + return {} + + try: + data = _wmi.exec_query(query) + except OSError: + return {} + + dict_data = {} + for item in data.split("\0"): + key, _, value = item.partition("=") + dict_data[key] = value + return dict_data + + def collect_windows(info_add): - if sys.platform != "win32": + if not MS_WINDOWS: # Code specific to Windows return @@ -909,71 +1023,39 @@ def collect_windows(info_add): try: import _winapi - dll_path = _winapi.GetModuleFileName(sys.dllhandle) - info_add('windows.dll_path', dll_path) - except (ImportError, AttributeError): - pass - - # windows.version_caption: "wmic os get Caption,Version /value" command - import subprocess - try: - # When wmic.exe output is redirected to a pipe, - # it uses the OEM code page - proc = subprocess.Popen(["wmic", "os", "get", "Caption,Version", "/value"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="oem", - text=True) - output, stderr = proc.communicate() - if proc.returncode: - output = "" - except OSError: + except ImportError: pass else: - for line in output.splitlines(): - line = line.strip() - if line.startswith('Caption='): - line = line.removeprefix('Caption=').strip() - if line: - info_add('windows.version_caption', line) - elif line.startswith('Version='): - line = line.removeprefix('Version=').strip() - if line: - info_add('windows.version', line) + try: + dll_path = _winapi.GetModuleFileName(sys.dllhandle) + info_add('windows.dll_path', dll_path) + except AttributeError: + pass + + call_func(info_add, 'windows.ansi_code_page', _winapi, 'GetACP') + call_func(info_add, 'windows.oem_code_page', _winapi, 'GetOEMCP') + + # Get operating system caption and version using WMI + data = wmi_query("SELECT Caption, Version FROM Win32_OperatingSystem") + caption = data.get('Caption', '') + if caption: + info_add('windows.version_caption', caption) + version = data.get('Version', '') + if version: + info_add('windows.version', version) # windows.ver: "ver" command - try: - proc = subprocess.Popen(["ver"], shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True) - output = proc.communicate()[0] - if proc.returncode == 0xc0000142: - return - if proc.returncode: - output = "" - except OSError: - return - else: - output = output.strip() - line = output.splitlines()[0] - if line: - info_add('windows.ver', line) + output = run_command(["ver"], shell=True) + # "ver" output starts with an empty line: remove it + output = output.strip() + if output: + info_add('windows.ver', first_line(output)) # windows.developer_mode: get AllowDevelopmentWithoutDevLicense registry - import winreg - try: - key = winreg.OpenKey( - winreg.HKEY_LOCAL_MACHINE, - r"SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock") - subkey = "AllowDevelopmentWithoutDevLicense" - try: - value, value_type = winreg.QueryValueEx(key, subkey) - finally: - winreg.CloseKey(key) - except OSError: - pass - else: + value = winreg_query(r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows" + r"\CurrentVersion\AppModelUnlock" + r"\AllowDevelopmentWithoutDevLicense") + if value is not None: info_add('windows.developer_mode', "enabled" if value else "disabled") @@ -986,14 +1068,9 @@ def collect_fips(info_add): if _hashlib is not None: call_func(info_add, 'fips.openssl_fips_mode', _hashlib, 'get_fips_mode') - try: - with open("/proc/sys/crypto/fips_enabled", encoding="utf-8") as fp: - line = fp.readline().rstrip() - - if line: - info_add('fips.linux_crypto_fips_enabled', line) - except OSError: - pass + fips_enabled = read_first_line("/proc/sys/crypto/fips_enabled") + if fips_enabled: + info_add('fips.linux_crypto_fips_enabled', fips_enabled) def collect_tempfile(info_add): @@ -1011,6 +1088,242 @@ def collect_libregrtest_utils(info_add): info_add('libregrtests.build_info', ' '.join(utils.get_build_info())) +def uptime_boottime(): + # Use CLOCK_BOOTTIME + import time + try: + return time.clock_gettime(time.CLOCK_BOOTTIME) + except (AttributeError, OSError): + return None + + +def uptime_linux(): + # Parse the first member of /proc/uptime + line = read_first_line("/proc/uptime") + if not line: + return + try: + parts = line.split() + if not parts: + return + return float(parts[0]) + except ValueError: + return + + +def uptime_bsd(): + # Get sysctlbyname("kern.boottime") + try: + import _testcapi + except ImportError: + return None + try: + return _testcapi.uptime_bsd() + except (AttributeError, OSError): + return None + + +def uptime_windows(): + try: + import _winapi + except ImportError: + return None + else: + return _winapi.GetTickCount64() / 1000. + + +def get_uptime(): + for func in (uptime_boottime, uptime_linux, uptime_bsd, uptime_windows): + uptime = func() + if uptime is not None: + return uptime + return None + + +def get_machine_id(): + if MS_WINDOWS: + machine_guid = winreg_query(r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft" + r"\Cryptography\MachineGuid") + if machine_guid: + return machine_guid + + for filename in ( + # https://www.freedesktop.org/software/systemd/man/latest/machine-id.html + "/etc/machine-id", + # BSD + "/etc/hostid", + ): + machine_id = read_first_line(filename) + if machine_id: + return machine_id + + return None + + +def detect_virt_windows(info_add): + # On Windows, use WMI to detect the virtualization. + # + # Microsoft Hyper-V: + # - Win32_Bios.Version = 'VRTUAL - 12001807' + # - Win32_Bios.Manufacturer = 'American Megatrends Inc.' + # - Win32_ComputerSystem.Model = 'Virtual Machine' + # - Win32_ComputerSystem.Manufacturer = 'Microsoft Corporation' + # + # VMware: + # - Win32_ComputerSystem.Model = 'VMware' + # - Win32_ComputerSystem.Manufacturer = 'VMWare' (uppercase W in Ware) + # - Win32_Bios.SerialNumber starts with 'VMware-' + # + # QEMU: + # - Win32_ComputerSystem.Manufacturer = 'QEMU' + # - Win32_ComputerSystem.Model = 'Standard PC (Q35 + ICH9, 2009)' + # - Win32_Bios.Version = 'BOCHS - 1' + # - Win32_Bios.Manufacturer = 'EDK II' + # + # Parallels: + # - Win32_Bios.Version = 'PARALLELS' + # + # VirtualBox: + # - Win32_Bios.Version = 'VBOX' + # - Win32_ComputerSystem.Model = 'VirtualBox' + # - Win32_ComputerSystem.Manufacturer = 'innotek GmbH' + # + # Amazon EC2: + # - Win32_Bios.Version = 'AMAZON - 1' + # - Win32_Bios.Manufacturer = 'Amazon EC2' + # - Win32_ComputerSystem.Model = 'm7i.4xlarge' + # - Win32_ComputerSystem.Manufacturer = 'Amazon EC2' + + KNOWN_VIRT = ( + 'Amazon EC2', + 'QEMU', + 'VMware', + 'VirtualBox', + 'Xen', + 'oVirt', + ) + KNOWN_BIOS_VERSIONS = { + 'PARALLELS': 'Parallels', + 'VBOX': 'VirtualBox', + } + + computer = wmi_query('SELECT Model, Manufacturer FROM Win32_ComputerSystem') + computer_model = computer.get('Model', '') + computer_manufacturer = computer.get('Manufacturer', '') + if computer_manufacturer == 'Amazon EC2': + # Log the VM model (ex: 'm7i.4xlarge') + info_add('system.computer.model', computer_model) + return computer_manufacturer + if computer_model in KNOWN_VIRT: + return computer_model + if computer_manufacturer in KNOWN_VIRT: + return computer_manufacturer + + bios = wmi_query('SELECT Version, Manufacturer FROM Win32_Bios') + + bios_version = bios.get('Version', '') + if bios_version in KNOWN_VIRT: + return bios_version + if (bios_version.startswith('VRTUAL - ') + and computer_manufacturer == 'Microsoft Corporation'): + return 'Microsoft Hyper-V' + try: + return KNOWN_BIOS_VERSIONS[bios_version] + except KeyError: + pass + + bios_manufacturer = bios.get('Manufacturer', '') + if bios_manufacturer in KNOWN_VIRT: + return bios_manufacturer + + # Log the values to update the code if a new VM is discovered + if computer_model: + info_add('system.computer.model', computer_model) + if computer_manufacturer: + info_add('system.computer.manufacturer', computer_manufacturer) + if bios_version: + info_add('system.bios.version', bios_version) + if bios_manufacturer: + info_add('system.bios.manufacturer', bios_manufacturer) + + +def detect_virt(info_add): + if MS_WINDOWS: + return detect_virt_windows(info_add) + + # Run systemd-detect-virt command + virt = run_command(["systemd-detect-virt"], check=False) + if virt and virt != "none": + return virt + + # Check if the process in running in a container + import os.path + if os.path.exists('/.dockerenv'): + return 'docker' + if os.path.exists('/run/.containerenv'): + return 'podman' + + container = read_first_line('/run/systemd/container') + if container: + return container + + if APPLE: + hv_vmm_present = run_command(['sysctl', '-n', 'kern.hv_vmm_present']) + if hv_vmm_present == '1': + return 'run in a VM (kern.hv_vmm_present is 1)' + + # Other ways to check if running in a container: + # * Parse /proc/1/mounts or /proc/1/mountinfo (check "/" filesystem). + # * Parse /proc/1/cgroup. + # * Parse the first line of /proc/1/sched (check process name is different + # than "init" and "systemd"). + # * Check / inode. + # * On systems using SELinux (Fedora/CentOS/RHEL), check for "container_t" + # label, for example of /proc/1/attr/current. + # * Check for "container" variable in /proc/1/environ + # (only root can read this file). + # * Check for "container" environment variable. + # * Set a specific env var when creating the container image. + # * Run virt-what, need to install the script, and must be run as root. + # * Check for "GITHUB_ACTIONS" environmant variable (GitHub Action). + + +def collect_system(info_add): + boot_id = read_first_line("/proc/sys/kernel/random/boot_id") + if boot_id: + info_add('system.boot_id', boot_id) + + machine_id = get_machine_id() + if machine_id: + info_add('system.machine_id', machine_id) + + uptime = get_uptime() + if uptime is not None: + # truncate microseconds + uptime = int(uptime) + try: + import datetime + uptime = str(datetime.timedelta(seconds=uptime)) + except ImportError: + uptime = f'{uptime} sec' + info_add('system.uptime', uptime) + + virt = detect_virt(info_add) + if virt: + info_add('system.virt', virt) + + if APPLE: + hardware = run_command(['sysctl', '-n', 'hw.model']) + if hardware: + info_add('system.hardware', hardware) + + +def collect_importlib(info_add): + import importlib.machinery + info_add('importlib.extension_suffixes', + importlib.machinery.EXTENSION_SUFFIXES) + + def collect_info(info): error = False info_add = info.add @@ -1050,7 +1363,10 @@ def collect_info(info): collect_tkinter, collect_windows, collect_zlib, + collect_zstd, collect_libregrtest_utils, + collect_system, + collect_importlib, # Collecting from tests should be last as they have side effects. collect_test_socket, @@ -1073,9 +1389,9 @@ def collect_info(info): def dump_info(info, file=None): - title = "Python debug information" + title = "Python build information" print(title) - print("=" * len(title)) + print("#" * len(title)) print() infos = info.get_infos() diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index c1dadc4e274..b1b2dd2ca5c 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -12,6 +12,7 @@ import string import sys from test import support +from test.support.import_helper import import_fresh_module import types import unittest @@ -26,7 +27,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer +from collections.abc import Buffer class TestUserObjects(unittest.TestCase): @@ -1935,6 +1936,14 @@ def assert_index_same(seq1, seq2, index_args): nativeseq, seqseq, (letter, start, stop)) def test_ByteString(self): + previous_sys_modules = sys.modules.copy() + self.addCleanup(sys.modules.update, previous_sys_modules) + + for module in "collections", "_collections_abc", "collections.abc": + sys.modules.pop(module, None) + + with self.assertWarns(DeprecationWarning): + from collections.abc import ByteString for sample in [bytes, bytearray]: with self.assertWarns(DeprecationWarning): self.assertIsInstance(sample(), ByteString) @@ -1956,6 +1965,14 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass + def test_ByteString_attribute_access(self): + collections_abc = import_fresh_module( + "collections.abc", + fresh=("collections", "_collections_abc") + ) + with self.assertWarns(DeprecationWarning): + collections_abc.ByteString + def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -2175,6 +2192,7 @@ def correctly_ordered(seq): self.assertTrue(correctly_ordered(p - q)) self.assertTrue(correctly_ordered(p | q)) self.assertTrue(correctly_ordered(p & q)) + self.assertTrue(correctly_ordered(p ^ q)) p, q = Counter(ps), Counter(qs) p += q @@ -2192,6 +2210,10 @@ def correctly_ordered(seq): p &= q self.assertTrue(correctly_ordered(p)) + p, q = Counter(ps), Counter(qs) + p ^= q + self.assertTrue(correctly_ordered(p)) + p, q = Counter(ps), Counter(qs) p.update(q) self.assertTrue(correctly_ordered(p)) @@ -2274,6 +2296,7 @@ def test_multiset_operations(self): (Counter.__sub__, lambda x, y: max(0, x-y)), (Counter.__or__, lambda x, y: max(0,x,y)), (Counter.__and__, lambda x, y: max(0, min(x,y))), + (Counter.__xor__, lambda x, y: max(0, max(x,y) - min(x,y))), ]: result = counterop(p, q) for x in elements: @@ -2291,6 +2314,7 @@ def test_multiset_operations(self): (Counter.__sub__, set.__sub__), (Counter.__or__, set.__or__), (Counter.__and__, set.__and__), + (Counter.__xor__, set.__xor__), ]: counter_result = counterop(p, q) set_result = setop(set(p.elements()), set(q.elements())) @@ -2309,6 +2333,7 @@ def test_inplace_operations(self): (Counter.__isub__, Counter.__sub__), (Counter.__ior__, Counter.__or__), (Counter.__iand__, Counter.__and__), + (Counter.__ixor__, Counter.__xor__), ]: c = p.copy() c_id = id(c) @@ -2384,6 +2409,7 @@ def test_multiset_operations_equivalent_to_set_operations(self): self.assertEqual(set(cp - cq), sp - sq) self.assertEqual(set(cp | cq), sp | sq) self.assertEqual(set(cp & cq), sp & sq) + self.assertEqual(set(cp ^ cq), sp ^ sq) self.assertEqual(cp == cq, sp == sq) self.assertEqual(cp != cq, sp != sq) self.assertEqual(cp <= cq, sp <= sq) @@ -2397,6 +2423,8 @@ def test_eq(self): def test_le(self): self.assertTrue(Counter(a=3, b=2, c=0) <= Counter('ababa')) + self.assertTrue(Counter() <= Counter(c=1)) + self.assertFalse(Counter() <= Counter(c=-1)) self.assertFalse(Counter(a=3, b=2) <= Counter('babab')) def test_lt(self): @@ -2405,12 +2433,48 @@ def test_lt(self): def test_ge(self): self.assertTrue(Counter(a=2, b=1, c=0) >= Counter('aab')) + self.assertTrue(Counter() >= Counter(c=-1)) + self.assertFalse(Counter() >= Counter(c=1)) self.assertFalse(Counter(a=3, b=2, c=0) >= Counter('aabd')) def test_gt(self): self.assertTrue(Counter(a=3, b=2, c=0) > Counter('aab')) self.assertFalse(Counter(a=2, b=1, c=0) > Counter('aab')) + def test_symmetric_difference(self): + population = (-4, -3, -2, -1, 0, 1, 2, 3, 4) + + for a, b1, b2, c in product(population, repeat=4): + p = Counter(a=a, b=b1) + q = Counter(b=b2, c=c) + r = p ^ q + + # Elementwise invariants + for k in ('a', 'b', 'c'): + self.assertEqual(r[k], max(p[k], q[k]) - min(p[k], q[k])) + self.assertEqual(r[k], abs(p[k] - q[k])) + + # Invariant for all positive, negative, and zero counts + self.assertEqual(r, (p - q) | (q - p)) + + # Invariant for non-negative counts + if a >= 0 and b1 >= 0 and b2 >= 0 and c >= 0: + self.assertEqual(r, (p | q) - (p & q)) + + # Zeros and negatives eliminated + self.assertTrue(all(value > 0 for value in r.values())) + + # Output preserves input order: p first and then q + keys = list(p) + list(q) + indices = [keys.index(k) for k in r] + self.assertEqual(indices, sorted(indices)) + + # Inplace operation matches binary operation + pp = Counter(p) + qq = Counter(q) + pp ^= qq + self.assertEqual(pp, r) + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(collections)) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 6a3329fa5aa..e291f814edb 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -680,6 +680,154 @@ def test(x): self.assertEqual(state, [1, 'something else', 999]) + def test_contextmanager_decorate_generator_function(self): + @contextmanager + def woohoo(y): + state.append(y) + yield + state.append(999) + + state = [] + @woohoo(1) + def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + return "result" + + gen = test("something") + for _ in gen: + self.assertEqual(state, [1, "something"]) + self.assertEqual(state, [1, "something", "second item", 999]) + + # The wrapped generator's return value is preserved. + state = [] + gen = test("something") + with self.assertRaises(StopIteration) as cm: + while True: + next(gen) + self.assertEqual(cm.exception.value, "result") + + + def test_contextmanager_decorate_generator_function_exception(self): + @contextmanager + def woohoo(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @woohoo() + def test(): + state.append("body") + yield + raise ZeroDivisionError + + with self.assertRaises(ZeroDivisionError): + for _ in test(): + pass + self.assertEqual(state, ["enter", "body", "exit"]) + + + def test_contextmanager_decorate_generator_function_early_stop(self): + @contextmanager + def woohoo(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @woohoo() + def test(): + try: + yield 1 + yield 2 + finally: + state.append("inner closed") + + gen = test() + self.assertEqual(next(gen), 1) + gen.close() + # The inner generator is closed before the context manager exits. + self.assertEqual(state, ["enter", "inner closed", "exit"]) + + + def test_contextmanager_decorate_generator_function_send_throw(self): + @contextmanager + def woohoo(): + yield + + @woohoo() + def test(): + received = yield "first" + state.append(("received", received)) + try: + yield "second" + except ValueError as exc: + state.append(("caught", type(exc))) + yield "after throw" + + # .send() and .throw() are forwarded to the wrapped generator. + state = [] + gen = test() + self.assertEqual(next(gen), "first") + self.assertEqual(gen.send("VALUE"), "second") + self.assertEqual(gen.throw(ValueError), "after throw") + gen.close() + self.assertEqual( + state, [("received", "VALUE"), ("caught", ValueError)] + ) + + + def test_contextmanager_decorate_coroutine_function(self): + @contextmanager + def woohoo(y): + state.append(y) + yield + state.append(999) + + state = [] + @woohoo(1) + async def test(x): + self.assertEqual(state, [1]) + state.append(x) + + coro = test("something") + with self.assertRaises(StopIteration): + coro.send(None) + + self.assertEqual(state, [1, "something", 999]) + + + def test_contextmanager_decorate_asyncgen_function(self): + @contextmanager + def woohoo(y): + state.append(y) + yield + state.append(999) + + state = [] + @woohoo(1) + async def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + + agen = test("something") + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + with self.assertRaises(StopAsyncIteration): + agen.asend(None).send(None) + + self.assertEqual(state, [1, "something", "second item", 999]) + + class TestBaseExitStack: exit_stack = None @@ -788,6 +936,75 @@ def _exit(): result.append(2) self.assertEqual(result, [1, 2, 3, 4]) + def test_enter_context_classmethod(self): + class TestCM: + @classmethod + def __enter__(cls): + result.append(('enter', cls)) + @classmethod + def __exit__(cls, *exc_details): + result.append(('exit', cls, *exc_details)) + + cm = TestCM() + result = [] + with self.exit_stack() as stack: + stack.enter_context(cm) + self.assertEqual(result, [('enter', TestCM)]) + self.assertEqual(result, [('enter', TestCM), + ('exit', TestCM, None, None, None)]) + + result = [] + with self.exit_stack() as stack: + stack.push(cm) + self.assertEqual(result, []) + self.assertEqual(result, [('exit', TestCM, None, None, None)]) + + def test_enter_context_staticmethod(self): + class TestCM: + @staticmethod + def __enter__(): + result.append('enter') + @staticmethod + def __exit__(*exc_details): + result.append(('exit', *exc_details)) + + cm = TestCM() + result = [] + with self.exit_stack() as stack: + stack.enter_context(cm) + self.assertEqual(result, ['enter']) + self.assertEqual(result, ['enter', ('exit', None, None, None)]) + + result = [] + with self.exit_stack() as stack: + stack.push(cm) + self.assertEqual(result, []) + self.assertEqual(result, [('exit', None, None, None)]) + + def test_enter_context_slots(self): + class TestCM: + __slots__ = ('__enter__', '__exit__') + def __init__(self): + def enter(): + result.append('enter') + def exit(*exc_details): + result.append(('exit', *exc_details)) + self.__enter__ = enter + self.__exit__ = exit + + cm = TestCM() + result = [] + with self.exit_stack() as stack: + stack.enter_context(cm) + self.assertEqual(result, ['enter']) + self.assertEqual(result, ['enter', ('exit', None, None, None)]) + + result = [] + with self.exit_stack() as stack: + stack.push(cm) + self.assertEqual(result, []) + self.assertEqual(result, [('exit', None, None, None)]) + def test_enter_context_errors(self): class LacksEnterAndExit: pass diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index dcd00720379..95bdfdb3d9d 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -402,6 +402,144 @@ async def test(): await test() self.assertFalse(entered) + @_async_test + async def test_decorator_decorate_sync_function(self): + @asynccontextmanager + async def context(): + state.append(1) + yield + state.append(999) + + state = [] + @context() + def test(x): + self.assertEqual(state, [1]) + state.append(x) + + await test("something") + self.assertEqual(state, [1, "something", 999]) + + @_async_test + async def test_decorator_decorate_generator_function(self): + @asynccontextmanager + async def context(): + state.append(1) + yield + state.append(999) + + state = [] + @context() + def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + + async for _ in test("something"): + self.assertEqual(state, [1, "something"]) + self.assertEqual(state, [1, "something", "second item", 999]) + + @_async_test + async def test_decorator_decorate_asyncgen_function(self): + @asynccontextmanager + async def context(): + state.append(1) + yield + state.append(999) + + state = [] + @context() + async def test(x): + self.assertEqual(state, [1]) + state.append(x) + yield + state.append("second item") + + async for _ in test("something"): + self.assertEqual(state, [1, "something"]) + self.assertEqual(state, [1, "something", "second item", 999]) + + @_async_test + async def test_decorator_decorate_asyncgen_function_exception(self): + @asynccontextmanager + async def context(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @context() + async def test(): + state.append("body") + yield + raise ZeroDivisionError + + with self.assertRaises(ZeroDivisionError): + async for _ in test(): + pass + self.assertEqual(state, ["enter", "body", "exit"]) + + @_async_test + async def test_decorator_decorate_asyncgen_function_early_stop(self): + @asynccontextmanager + async def context(): + state.append("enter") + try: + yield + finally: + state.append("exit") + + state = [] + @context() + async def test(): + try: + yield 1 + yield 2 + finally: + state.append("inner closed") + + agen = test() + async for value in agen: + self.assertEqual(value, 1) + break + await agen.aclose() + # The inner async generator is closed before the context + # manager exits. + self.assertEqual(state, ["enter", "inner closed", "exit"]) + + @_async_test + async def test_decorator_decorate_asyncgen_function_asend_athrow(self): + @asynccontextmanager + async def context(): + yield + + @context() + async def test(): + try: + received = yield "first" + state.append(("received", received)) + yield "second" + except ValueError: + state.append("inner saw ValueError") + raise + finally: + state.append("inner closed") + + # asend() values and athrow() exceptions are not forwarded to the + # wrapped generator (a documented limitation). + state = [] + agen = test() + self.assertEqual(await agen.__anext__(), "first") + self.assertEqual(await agen.asend("VALUE"), "second") + # The inner generator received None, not "VALUE". + self.assertEqual(state, [("received", None)]) + with self.assertRaises(ValueError): + await agen.athrow(ValueError) + # The inner generator was closed, not thrown into. + self.assertEqual(state, [("received", None), "inner closed"]) + @_async_test async def test_decorator_with_exception(self): entered = False @@ -641,6 +779,78 @@ async def _exit(): self.assertEqual(result, [1, 2, 3, 4]) + @_async_test + async def test_enter_async_context_classmethod(self): + class TestCM: + @classmethod + async def __aenter__(cls): + result.append(('enter', cls)) + @classmethod + async def __aexit__(cls, *exc_details): + result.append(('exit', cls, *exc_details)) + + cm = TestCM() + result = [] + async with self.exit_stack() as stack: + await stack.enter_async_context(cm) + self.assertEqual(result, [('enter', TestCM)]) + self.assertEqual(result, [('enter', TestCM), + ('exit', TestCM, None, None, None)]) + + result = [] + async with self.exit_stack() as stack: + stack.push_async_exit(cm) + self.assertEqual(result, []) + self.assertEqual(result, [('exit', TestCM, None, None, None)]) + + @_async_test + async def test_enter_async_context_staticmethod(self): + class TestCM: + @staticmethod + async def __aenter__(): + result.append('enter') + @staticmethod + async def __aexit__(*exc_details): + result.append(('exit', *exc_details)) + + cm = TestCM() + result = [] + async with self.exit_stack() as stack: + await stack.enter_async_context(cm) + self.assertEqual(result, ['enter']) + self.assertEqual(result, ['enter', ('exit', None, None, None)]) + + result = [] + async with self.exit_stack() as stack: + stack.push_async_exit(cm) + self.assertEqual(result, []) + self.assertEqual(result, [('exit', None, None, None)]) + + @_async_test + async def test_enter_async_context_slots(self): + class TestCM: + __slots__ = ('__aenter__', '__aexit__') + def __init__(self): + async def enter(): + result.append('enter') + async def exit(*exc_details): + result.append(('exit', *exc_details)) + self.__aenter__ = enter + self.__aexit__ = exit + + cm = TestCM() + result = [] + async with self.exit_stack() as stack: + await stack.enter_async_context(cm) + self.assertEqual(result, ['enter']) + self.assertEqual(result, ['enter', ('exit', None, None, None)]) + + result = [] + async with self.exit_stack() as stack: + stack.push_async_exit(cm) + self.assertEqual(result, []) + self.assertEqual(result, [('exit', None, None, None)]) + @_async_test async def test_enter_async_context_errors(self): class LacksEnterAndExit: diff --git a/Lib/test/test_defaultdict.py b/Lib/test/test_defaultdict.py index a193eb10f16..cc78f01e3e2 100644 --- a/Lib/test/test_defaultdict.py +++ b/Lib/test/test_defaultdict.py @@ -186,6 +186,18 @@ def test_union(self): with self.assertRaises(TypeError): i |= None + # frozendict + i_fd = i | frozendict(s) + self.assertIs(type(i_fd), defaultdict) + self.assertIs(i_fd.default_factory, int) + self.assertDictEqual(i_fd, {1: "one", 2: 2, 0: "zero"}) + self.assertEqual(list(i_fd), [1, 2, 0]) + + fd_i = frozendict(s) | i + self.assertIs(type(fd_i), frozendict) + self.assertEqual(fd_i, {1: "one", 2: 2, 0: "zero"}) + self.assertEqual(list(fd_i), [0, 1, 2]) + def test_factory_conflict_with_set_value(self): key = "conflict_test" count = 0 diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 518b8c5278e..4f99b7c91c6 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -1,5 +1,7 @@ import difflib -from test.support import findfile +from test import support +from test.support import findfile, force_colorized +from test.support.import_helper import ensure_lazy_imports import unittest import doctest import sys @@ -265,21 +267,21 @@ def test_make_file_default_charset(self): html_diff = difflib.HtmlDiff() output = html_diff.make_file(patch914575_from1.splitlines(), patch914575_to1.splitlines()) - self.assertIn('content="text/html; charset=utf-8"', output) + self.assertIn('charset="utf-8"', output) def test_make_file_iso88591_charset(self): html_diff = difflib.HtmlDiff() output = html_diff.make_file(patch914575_from1.splitlines(), patch914575_to1.splitlines(), charset='iso-8859-1') - self.assertIn('content="text/html; charset=iso-8859-1"', output) + self.assertIn('charset="iso-8859-1"', output) def test_make_file_usascii_charset_with_nonascii_input(self): html_diff = difflib.HtmlDiff() output = html_diff.make_file(patch914575_nonascii_from1.splitlines(), patch914575_nonascii_to1.splitlines(), charset='us-ascii') - self.assertIn('content="text/html; charset=us-ascii"', output) + self.assertIn('charset="us-ascii"', output) self.assertIn('ımplıcıt', output) def test_strip_trailing_newlines_before_diff(self): @@ -397,6 +399,22 @@ def test_range_format_context(self): self.assertEqual(fmt(3,6), '4,6') self.assertEqual(fmt(0,0), '0') + @force_colorized + def test_unified_diff_colored_output(self): + args = [['one', 'three'], ['two', 'three'], 'Original', 'Current', + '2005-01-26 23:30:50', '2010-04-02 10:20:52'] + actual = list(difflib.unified_diff(*args, lineterm='', color=True)) + + expect = [ + "\033[1m--- Original\t2005-01-26 23:30:50\033[0m", + "\033[1m+++ Current\t2010-04-02 10:20:52\033[0m", + "\033[36m@@ -1,2 +1,2 @@\033[0m", + "\033[31m-one\033[0m", + "\033[32m+two\033[0m", + "\033[0m three\033[0m", + ] + self.assertEqual(expect, actual) + class TestBytes(unittest.TestCase): # don't really care about the content of the output, just the fact @@ -651,6 +669,12 @@ def setUpModule(): difflib.HtmlDiff._default_prefix = 0 +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("difflib", {"_colorize"}) + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(difflib)) return tests diff --git a/Lib/test/test_difflib_expect.html b/Lib/test/test_difflib_expect.html index 9f33a9e9c9c..240e2c33368 100644 --- a/Lib/test/test_difflib_expect.html +++ b/Lib/test/test_difflib_expect.html @@ -1,22 +1,42 @@ - - - - + + - - - @@ -37,11 +59,11 @@
from
to f1f1 - n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. - 3   2. Explicit is better than implicit. - 4   3. Simple is better than complex.3   3.   Simple is better than complex. - 5   4. Complex is better than complicated.4   4. Complicated is better than complex. - 5   5. Flat is better than nested. + n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. + 3   2. Explicit is better than implicit. + 4   3. Simple is better than complex.3   3.   Simple is better than complex. + 5   4. Complex is better than complicated.4   4. Complicated is better than complex. + 5   5. Flat is better than nested. 61236123 71237123 81238123 @@ -53,11 +75,11 @@ 1412314123 1512315123 1616 - n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. - 18   2. Explicit is better than implicit. - 19   3. Simple is better than complex.18   3.   Simple is better than complex. - 20   4. Complex is better than complicated.19   4. Complicated is better than complex. - 20   5. Flat is better than nested. + n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. + 18   2. Explicit is better than implicit. + 19   3. Simple is better than complex.18   3.   Simple is better than complex. + 20   4. Complex is better than complicated.19   4. Complicated is better than complex. + 20   5. Flat is better than nested. 2112321123 2212322123 2312323123 @@ -69,11 +91,11 @@ 2912329123 3012330123 3131 - t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. - 33   2. Explicit is better than implicit. - 34   3. Simple is better than complex.33   3.   Simple is better than complex. - 35   4. Complex is better than complicated.34   4. Complicated is better than complex. - 35   5. Flat is better than nested. + t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. + 33   2. Explicit is better than implicit. + 34   3. Simple is better than complex.33   3.   Simple is better than complex. + 35   4. Complex is better than complicated.34   4. Complicated is better than complex. + 35   5. Flat is better than nested. 3612336123 3712337123 3812338123 @@ -111,11 +133,11 @@

Context (first diff within numlines=5(default))


from
to f1f1 - n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. - 3   2. Explicit is better than implicit. - 4   3. Simple is better than complex.3   3.   Simple is better than complex. - 5   4. Complex is better than complicated.4   4. Complicated is better than complex. - 5   5. Flat is better than nested. + n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. + 3   2. Explicit is better than implicit. + 4   3. Simple is better than complex.3   3.   Simple is better than complex. + 5   4. Complex is better than complicated.4   4. Complicated is better than complex. + 5   5. Flat is better than nested. 61236123 71237123 81238123 @@ -128,11 +150,11 @@

Context (first diff within numlines=5(default))

1412314123 1512315123 1616 - n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. - 18   2. Explicit is better than implicit. - 19   3. Simple is better than complex.18   3.   Simple is better than complex. - 20   4. Complex is better than complicated.19   4. Complicated is better than complex. - 20   5. Flat is better than nested. + n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. + 18   2. Explicit is better than implicit. + 19   3. Simple is better than complex.18   3.   Simple is better than complex. + 20   4. Complex is better than complicated.19   4. Complicated is better than complex. + 20   5. Flat is better than nested. 2112321123 2212322123 2312323123 @@ -145,11 +167,11 @@

Context (first diff within numlines=5(default))

2912329123 3012330123 3131 - t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. - 33   2. Explicit is better than implicit. - 34   3. Simple is better than complex.33   3.   Simple is better than complex. - 35   4. Complex is better than complicated.34   4. Complicated is better than complex. - 35   5. Flat is better than nested. + t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. + 33   2. Explicit is better than implicit. + 34   3. Simple is better than complex.33   3.   Simple is better than complex. + 35   4. Complex is better than complicated.34   4. Complicated is better than complex. + 35   5. Flat is better than nested. 3612336123 3712337123 3812338123 @@ -170,11 +192,11 @@

Context (first diff after numlines=5(default))

94569456 1045610456 1111 - n12   1. Beautiful is beTTer than ugly.n12   1. Beautiful is better than ugly. - 13   2. Explicit is better than implicit. - 14   3. Simple is better than complex.13   3.   Simple is better than complex. - 15   4. Complex is better than complicated.14   4. Complicated is better than complex. - 15   5. Flat is better than nested. + n12   1. Beautiful is beTTer than ugly.n12   1. Beautiful is better than ugly. + 13   2. Explicit is better than implicit. + 14   3. Simple is better than complex.13   3.   Simple is better than complex. + 15   4. Complex is better than complicated.14   4. Complicated is better than complex. + 15   5. Flat is better than nested. 1612316123 1712317123 1812318123 @@ -187,11 +209,11 @@

Context (first diff after numlines=5(default))

2412324123 2512325123 2626 - n27   1. Beautiful is beTTer than ugly.n27   1. Beautiful is better than ugly. - 28   2. Explicit is better than implicit. - 29   3. Simple is better than complex.28   3.   Simple is better than complex. - 30   4. Complex is better than complicated.29   4. Complicated is better than complex. - 30   5. Flat is better than nested. + n27   1. Beautiful is beTTer than ugly.n27   1. Beautiful is better than ugly. + 28   2. Explicit is better than implicit. + 29   3. Simple is better than complex.28   3.   Simple is better than complex. + 30   4. Complex is better than complicated.29   4. Complicated is better than complex. + 30   5. Flat is better than nested. 3112331123 3212332123 3312333123 @@ -204,11 +226,11 @@

Context (first diff after numlines=5(default))

3912339123 4012340123 4141 - t42   1. Beautiful is beTTer than ugly.t42   1. Beautiful is better than ugly. - 43   2. Explicit is better than implicit. - 44   3. Simple is better than complex.43   3.   Simple is better than complex. - 45   4. Complex is better than complicated.44   4. Complicated is better than complex. - 45   5. Flat is better than nested. + t42   1. Beautiful is beTTer than ugly.t42   1. Beautiful is better than ugly. + 43   2. Explicit is better than implicit. + 44   3. Simple is better than complex.43   3.   Simple is better than complex. + 45   4. Complex is better than complicated.44   4. Complicated is better than complex. + 45   5. Flat is better than nested. 4612346123 4712347123 4812348123 @@ -225,11 +247,11 @@

Context (numlines=6)


from
to f1f1 - n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. - 3   2. Explicit is better than implicit. - 4   3. Simple is better than complex.3   3.   Simple is better than complex. - 5   4. Complex is better than complicated.4   4. Complicated is better than complex. - 5   5. Flat is better than nested. + n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. + 3   2. Explicit is better than implicit. + 4   3. Simple is better than complex.3   3.   Simple is better than complex. + 5   4. Complex is better than complicated.4   4. Complicated is better than complex. + 5   5. Flat is better than nested. 61236123 71237123 81238123 @@ -241,11 +263,11 @@

Context (numlines=6)

1412314123 1512315123 1616 - n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. - 18   2. Explicit is better than implicit. - 19   3. Simple is better than complex.18   3.   Simple is better than complex. - 20   4. Complex is better than complicated.19   4. Complicated is better than complex. - 20   5. Flat is better than nested. + n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. + 18   2. Explicit is better than implicit. + 19   3. Simple is better than complex.18   3.   Simple is better than complex. + 20   4. Complex is better than complicated.19   4. Complicated is better than complex. + 20   5. Flat is better than nested. 2112321123 2212322123 2312323123 @@ -257,11 +279,11 @@

Context (numlines=6)

2912329123 3012330123 3131 - t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. - 33   2. Explicit is better than implicit. - 34   3. Simple is better than complex.33   3.   Simple is better than complex. - 35   4. Complex is better than complicated.34   4. Complicated is better than complex. - 35   5. Flat is better than nested. + t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. + 33   2. Explicit is better than implicit. + 34   3. Simple is better than complex.33   3.   Simple is better than complex. + 35   4. Complex is better than complicated.34   4. Complicated is better than complex. + 35   5. Flat is better than nested. 3612336123 3712337123 3812338123 @@ -278,25 +300,25 @@

Context (numlines=0)


from
to - n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. - 3   2. Explicit is better than implicit. - 4   3. Simple is better than complex.3   3.   Simple is better than complex. - 5   4. Complex is better than complicated.4   4. Complicated is better than complex. - 5   5. Flat is better than nested. + n2   1. Beautiful is beTTer than ugly.n2   1. Beautiful is better than ugly. + 3   2. Explicit is better than implicit. + 4   3. Simple is better than complex.3   3.   Simple is better than complex. + 5   4. Complex is better than complicated.4   4. Complicated is better than complex. + 5   5. Flat is better than nested. - n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. - 18   2. Explicit is better than implicit. - 19   3. Simple is better than complex.18   3.   Simple is better than complex. - 20   4. Complex is better than complicated.19   4. Complicated is better than complex. - 20   5. Flat is better than nested. + n17   1. Beautiful is beTTer than ugly.n17   1. Beautiful is better than ugly. + 18   2. Explicit is better than implicit. + 19   3. Simple is better than complex.18   3.   Simple is better than complex. + 20   4. Complex is better than complicated.19   4. Complicated is better than complex. + 20   5. Flat is better than nested. - t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. - 33   2. Explicit is better than implicit. - 34   3. Simple is better than complex.33   3.   Simple is better than complex. - 35   4. Complex is better than complicated.34   4. Complicated is better than complex. - 35   5. Flat is better than nested. + t32   1. Beautiful is beTTer than ugly.t32   1. Beautiful is better than ugly. + 33   2. Explicit is better than implicit. + 34   3. Simple is better than complex.33   3.   Simple is better than complex. + 35   4. Complex is better than complicated.34   4. Complicated is better than complex. + 35   5. Flat is better than nested.

Same Context

@@ -396,11 +418,11 @@

tabsize=2

f1f1 - t2    Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss] - 3      Line 2: preceded by from:[sstt] to:[sssst]3      Line 2: preceded by from:[sstt] to:[sssst] - 4      Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss] - 5Line 4:   has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after : - 6Line 5: has from:[t] to:[ss] at end 6Line 5: has from:[t] to:[ss] at end + t2    Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss] + 3      Line 2: preceded by from:[sstt] to:[sssst]3      Line 2: preceded by from:[sstt] to:[sssst] + 4      Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss] + 5Line 4:   has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after : + 6Line 5: has from:[t] to:[ss] at end 6Line 5: has from:[t] to:[ss] at end

tabsize=default

@@ -412,11 +434,11 @@

tabsize=default

f1f1 - t2                Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss] - 3                Line 2: preceded by from:[sstt] to:[sssst]3        Line 2: preceded by from:[sstt] to:[sssst] - 4                Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss] - 5Line 4:         has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after : - 6Line 5: has from:[t] to:[ss] at end     6Line 5: has from:[t] to:[ss] at end + t2                Line 1: preceded by from:[tt] to:[ssss]t2    Line 1: preceded by from:[tt] to:[ssss] + 3                Line 2: preceded by from:[sstt] to:[sssst]3        Line 2: preceded by from:[sstt] to:[sssst] + 4                Line 3: preceded by from:[sstst] to:[ssssss]4      Line 3: preceded by from:[sstst] to:[ssssss] + 5Line 4:         has from:[sst] to:[sss] after :5Line 4:   has from:[sst] to:[sss] after : + 6Line 5: has from:[t] to:[ss] at end     6Line 5: has from:[t] to:[ss] at end

Context (wrapcolumn=14,numlines=0)

@@ -427,31 +449,31 @@

Context (wrapcolumn=14,numlines=0)

- n4line 2n4line 2    adde -  >d + n4line 2n4line 2    adde +  >d - n6line 4   changn6line 4   chanG - >ed>Ed - 7line 5   chang7line 5a  chanG - >ed>ed - 8line 6   chang8line 6a  chang - >ed>Ed + n6line 4   changn6line 4   chanG + >ed>Ed + 7line 5   chang7line 5a  chanG + >ed>ed + 8line 6   chang8line 6a  chang + >ed>Ed - n10line 8  subtran10line 8 - >cted  + n10line 8  subtran10line 8 + >cted  - t1212345678901234t121234567890 - >56789012345689  - >012345  - 13short line13another long l -  >ine that needs -  > to be wrapped - 14just fits in!!14just fitS in!! - 15just fits in t15just fits in t - >wo lines yup!!>wo lineS yup!! + t1212345678901234t121234567890 + >56789012345689  + >012345  + 13short line13another long l +  >ine that needs +  > to be wrapped + 14just fits in!!14just fitS in!! + 15just fits in t15just fits in t + >wo lines yup!!>wo lineS yup!!

wrapcolumn=14,splitlines()

@@ -467,28 +489,28 @@

wrapcolumn=14,splitlines()

>56789012345689>56789012345689 >012345>012345 3line 13line 1 - n4line 2n4line 2    adde -  >d + n4line 2n4line 2    adde +  >d 5line 35line 3 - n6line 4   changn6line 4   chanG - >ed>Ed - 7line 5   chang7line 5a  chanG - >ed>ed - 8line 6   chang8line 6a  chang - >ed>Ed + n6line 4   changn6line 4   chanG + >ed>Ed + 7line 5   chang7line 5a  chanG + >ed>ed + 8line 6   chang8line 6a  chang + >ed>Ed 9line 79line 7 - n10line 8  subtran10line 8 - >cted  + n10line 8  subtran10line 8 + >cted  11line 911line 9 - t1212345678901234t121234567890 - >56789012345689  - >012345  - 13short line13another long l -  >ine that needs -  > to be wrapped - 14just fits in!!14just fitS in!! - 15just fits in t15just fits in t - >wo lines yup!!>wo lineS yup!! + t1212345678901234t121234567890 + >56789012345689  + >012345  + 13short line13another long l +  >ine that needs +  > to be wrapped + 14just fits in!!14just fitS in!! + 15just fits in t15just fits in t + >wo lines yup!!>wo lineS yup!! 16the end16the end @@ -505,28 +527,28 @@

wrapcolumn=14,splitlines(True)

>56789012345689>56789012345689 >012345>012345 3line 13line 1 - n4line 2n4line 2    adde -  >d + n4line 2n4line 2    adde +  >d 5line 35line 3 - n6line 4   changn6line 4   chanG - >ed>Ed - 7line 5   chang7line 5a  chanG - >ed>ed - 8line 6   chang8line 6a  chang - >ed>Ed + n6line 4   changn6line 4   chanG + >ed>Ed + 7line 5   chang7line 5a  chanG + >ed>ed + 8line 6   chang8line 6a  chang + >ed>Ed 9line 79line 7 - n10line 8  subtran10line 8 - >cted  + n10line 8  subtran10line 8 + >cted  11line 911line 9 - t1212345678901234t121234567890 - >56789012345689  - >012345  - 13short line13another long l -  >ine that needs -  > to be wrapped - 14just fits in!!14just fitS in!! - 15just fits in t15just fits in t - >wo lines yup!!>wo lineS yup!! + t1212345678901234t121234567890 + >56789012345689  + >012345  + 13short line13another long l +  >ine that needs +  > to be wrapped + 14just fits in!!14just fitS in!! + 15just fits in t15just fits in t + >wo lines yup!!>wo lineS yup!! 16the end16the end diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index b9f67f07ebc..b125693ab08 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -833,6 +833,118 @@ def test_empty_namespace_package(self): self.assertEqual(len(include_empty_finder.find(mod)), 1) self.assertEqual(len(exclude_empty_finder.find(mod)), 0) + def test_lineno_of_test_dict_strings(self): + """Test line numbers are found for __test__ dict strings.""" + module_content = '''\ +"""Module docstring.""" + +def dummy_function(): + """Dummy function docstring.""" + pass + +__test__ = { + 'test_string': """ + This is a test string. + >>> 1 + 1 + 2 + """, +} +''' + with tempfile.TemporaryDirectory() as tmpdir: + module_path = os.path.join(tmpdir, 'test_module_lineno.py') + with open(module_path, 'w') as f: + f.write(module_content) + + sys.path.insert(0, tmpdir) + try: + import test_module_lineno + finder = doctest.DocTestFinder() + tests = finder.find(test_module_lineno) + + test_dict_test = None + for test in tests: + if '__test__' in test.name: + test_dict_test = test + break + + self.assertIsNotNone( + test_dict_test, + "__test__ dict test not found" + ) + # gh-69113: line number should not be None for __test__ strings + self.assertIsNotNone( + test_dict_test.lineno, + "Line number should not be None for __test__ dict strings" + ) + self.assertGreater( + test_dict_test.lineno, + 0, + "Line number should be positive" + ) + finally: + if 'test_module_lineno' in sys.modules: + del sys.modules['test_module_lineno'] + sys.path.pop(0) + + def test_lineno_multiline_matching(self): + """Test multi-line matching when no unique line exists.""" + # gh-69113: test that line numbers are found even when lines + # appear multiple times (e.g., ">>> x = 1" in both test entries) + module_content = '''\ +"""Module docstring.""" + +__test__ = { + 'test_one': """ + >>> x = 1 + >>> x + 1 + """, + 'test_two': """ + >>> x = 1 + >>> x + 2 + """, +} +''' + with tempfile.TemporaryDirectory() as tmpdir: + module_path = os.path.join(tmpdir, 'test_module_multiline.py') + with open(module_path, 'w') as f: + f.write(module_content) + + sys.path.insert(0, tmpdir) + try: + import test_module_multiline + finder = doctest.DocTestFinder() + tests = finder.find(test_module_multiline) + + test_one = None + test_two = None + for test in tests: + if 'test_one' in test.name: + test_one = test + elif 'test_two' in test.name: + test_two = test + + self.assertIsNotNone(test_one, "test_one not found") + self.assertIsNotNone(test_two, "test_two not found") + self.assertIsNotNone( + test_one.lineno, + "Line number should not be None for test_one" + ) + self.assertIsNotNone( + test_two.lineno, + "Line number should not be None for test_two" + ) + self.assertNotEqual( + test_one.lineno, + test_two.lineno, + "test_one and test_two should have different line numbers" + ) + finally: + if 'test_module_multiline' in sys.modules: + del sys.modules['test_module_multiline'] + sys.path.pop(0) + def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2273,20 +2385,22 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> result = suite.run(unittest.TestResult()) >>> result - + >>> for tst, _ in result.failures: ... print(tst) - bad (test.test_doctest.sample_doctest.__test__) - foo (test.test_doctest.sample_doctest) - test_silly_setup (test.test_doctest.sample_doctest) - y_is_one (test.test_doctest.sample_doctest) + bad (test.test_doctest.sample_doctest.__test__) [0] + foo (test.test_doctest.sample_doctest) [0] + >>> for tst, _ in result.errors: + ... print(tst) + test_silly_setup (test.test_doctest.sample_doctest) [1] + y_is_one (test.test_doctest.sample_doctest) [0] We can also supply the module by name: >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') >>> result = suite.run(unittest.TestResult()) >>> result - + The module need not contain any doctest examples: @@ -2308,21 +2422,26 @@ def test_DocTestSuite(): >>> result >>> len(result.skipped) - 2 + 7 >>> for tst, _ in result.skipped: ... print(tst) + double_skip (test.test_doctest.sample_doctest_skip) [0] + double_skip (test.test_doctest.sample_doctest_skip) [1] double_skip (test.test_doctest.sample_doctest_skip) + partial_skip_fail (test.test_doctest.sample_doctest_skip) [0] + partial_skip_pass (test.test_doctest.sample_doctest_skip) [0] + single_skip (test.test_doctest.sample_doctest_skip) [0] single_skip (test.test_doctest.sample_doctest_skip) >>> for tst, _ in result.failures: ... print(tst) - no_skip_fail (test.test_doctest.sample_doctest_skip) - partial_skip_fail (test.test_doctest.sample_doctest_skip) + no_skip_fail (test.test_doctest.sample_doctest_skip) [0] + partial_skip_fail (test.test_doctest.sample_doctest_skip) [1] We can use the current module: >>> suite = test.test_doctest.sample_doctest.test_suite() >>> suite.run(unittest.TestResult()) - + We can also provide a DocTestFinder: @@ -2330,7 +2449,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... test_finder=finder) >>> suite.run(unittest.TestResult()) - + The DocTestFinder need not return any tests: @@ -2346,7 +2465,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', globs={}) >>> suite.run(unittest.TestResult()) - + Alternatively, we can provide extra globals. Here we'll make an error go away by providing an extra global variable: @@ -2354,7 +2473,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... extraglobs={'y': 1}) >>> suite.run(unittest.TestResult()) - + You can pass option flags. Here we'll cause an extra error by disabling the blank-line feature: @@ -2362,7 +2481,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> suite.run(unittest.TestResult()) - + You can supply setUp and tearDown functions: @@ -2379,7 +2498,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) - + But the tearDown restores sanity: @@ -2397,7 +2516,7 @@ def test_DocTestSuite(): >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest', setUp=setUp) >>> suite.run(unittest.TestResult()) - + Here, we didn't need to use a tearDown function because we modified the test globals, which are a copy of the @@ -2413,115 +2532,99 @@ def test_DocTestSuite_errors(): >>> suite = doctest.DocTestSuite(mod) >>> result = suite.run(unittest.TestResult()) >>> result - + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors - File "...sample_doctest_errors.py", line 0, in sample_doctest_errors - - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors - Failed example: + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + >...>> 2 + 2 + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors - Failed example: - 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero >>> print(result.failures[1][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.__test__.bad - File "...sample_doctest_errors.py", line unknown line number, in bad - - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad - Failed example: + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 37, in test.test_doctest.sample_doctest_errors.__test__.bad + >...>> 2 + 2 + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad - Failed example: - 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero >>> print(result.failures[2][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.errors - File "...sample_doctest_errors.py", line 14, in errors - - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors - Failed example: + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + >...>> 2 + 2 + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors - Failed example: + + >>> print(result.errors[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + >...>> 1/0 + File "", line 1, in 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors - Failed example: + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 39, in test.test_doctest.sample_doctest_errors.__test__.bad + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + >...>> 1/0 + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[3][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + >...>> f() + File "", line 1, in f() - Exception raised: - Traceback (most recent call last): - File "", line 1, in - f() - ~^^ - File "", line 2, in f - 2 + '2' - ~~^~~~~ - TypeError: ... - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors - Failed example: - g() - Exception raised: - Traceback (most recent call last): - File "", line 1, in - g() - ~^^ - File "...sample_doctest_errors.py", line 12, in g - [][0] # line 12 - ~~^^^ - IndexError: list index out of range + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... - >>> print(result.failures[3][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.syntax_error - File "...sample_doctest_errors.py", line 29, in syntax_error + >>> print(result.errors[4][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + >...>> g() + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range - ---------------------------------------------------------------------- - File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error - Failed example: + >>> print(result.errors[5][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + >...>> 2+*3 + File "", line 1 2+*3 - Exception raised: - File "", line 1 - 2+*3 - ^ - SyntaxError: invalid syntax + ^ + SyntaxError: invalid syntax """ @@ -2536,7 +2639,7 @@ def test_DocFileSuite(): ... 'test_doctest2.txt', ... 'test_doctest4.txt') >>> suite.run(unittest.TestResult()) - + The test files are looked for in the directory containing the calling module. A package keyword argument can be provided to @@ -2548,14 +2651,14 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... package='test.test_doctest') >>> suite.run(unittest.TestResult()) - + '/' should be used as a path separator. It will be converted to a native separator at run time: >>> suite = doctest.DocFileSuite('../test_doctest/test_doctest.txt') >>> suite.run(unittest.TestResult()) - + If DocFileSuite is used from an interactive session, then files are resolved relative to the directory of sys.argv[0]: @@ -2581,7 +2684,7 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite(test_file, module_relative=False) >>> suite.run(unittest.TestResult()) - + It is an error to specify `package` when `module_relative=False`: @@ -2599,12 +2702,15 @@ def test_DocFileSuite(): ... 'test_doctest_skip2.txt') >>> result = suite.run(unittest.TestResult()) >>> result - + >>> len(result.skipped) - 1 + 4 >>> for tst, _ in result.skipped: # doctest: +ELLIPSIS ... print('=', tst) + = ...test_doctest_skip.txt [0] + = ...test_doctest_skip.txt [1] = ...test_doctest_skip.txt + = ...test_doctest_skip2.txt [0] You can specify initial global variables: @@ -2613,7 +2719,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... globs={'favorite_color': 'blue'}) >>> suite.run(unittest.TestResult()) - + In this case, we supplied a missing favorite color. You can provide doctest options: @@ -2624,7 +2730,7 @@ def test_DocFileSuite(): ... optionflags=doctest.DONT_ACCEPT_BLANKLINE, ... globs={'favorite_color': 'blue'}) >>> suite.run(unittest.TestResult()) - + And, you can provide setUp and tearDown functions: @@ -2643,7 +2749,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... setUp=setUp, tearDown=tearDown) >>> suite.run(unittest.TestResult()) - + But the tearDown restores sanity: @@ -2685,7 +2791,7 @@ def test_DocFileSuite(): ... 'test_doctest4.txt', ... encoding='utf-8') >>> suite.run(unittest.TestResult()) - + """ def test_DocFileSuite_errors(): @@ -2695,52 +2801,49 @@ def test_DocFileSuite_errors(): >>> suite = doctest.DocFileSuite('test_doctest_errors.txt') >>> result = suite.run(unittest.TestResult()) >>> result - + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest_errors.txt - File "...test_doctest_errors.txt", line 0 - - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt - Failed example: + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + >...>> 2 + 2 + AssertionError: Failed example: 2 + 2 Expected: 5 Got: 4 - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt - Failed example: + + >>> print(result.errors[0][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + >...>> 1/0 + File "", line 1, in 1/0 - Exception raised: - Traceback (most recent call last): - File "", line 1, in - 1/0 - ~^~ - ZeroDivisionError: division by zero - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt - Failed example: + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.errors[1][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + >...>> f() + File "", line 1, in f() - Exception raised: - Traceback (most recent call last): - File "", line 1, in - f() - ~^^ - File "", line 2, in f - 2 + '2' - ~~^~~~~ - TypeError: ... - ---------------------------------------------------------------------- - File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt - Failed example: + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + + >>> print(result.errors[2][1]) # doctest: +ELLIPSIS + Traceback (most recent call last): + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + >...>> 2+*3 + File "", line 1 2+*3 - Exception raised: - File "", line 1 - 2+*3 - ^ - SyntaxError: invalid syntax + ^ + SyntaxError: invalid syntax + """ def test_trailing_space_in_test(): @@ -2811,16 +2914,25 @@ def test_unittest_reportflags(): >>> import unittest >>> result = suite.run(unittest.TestResult()) >>> result - + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest.txt - ... - Failed example: - favorite_color - ... - Failed example: + Traceback (most recent call last): + File ... + >...>> if 1: + AssertionError: Failed example: if 1: - ... + print('a') + print() + print('b') + Expected: + a + + b + Got: + a + + b + Note that we see both failures displayed. @@ -2829,18 +2941,8 @@ def test_unittest_reportflags(): Now, when we run the test: - >>> result = suite.run(unittest.TestResult()) - >>> result - - >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest.txt - ... - Failed example: - favorite_color - Exception raised: - ... - NameError: name 'favorite_color' is not defined - + >>> suite.run(unittest.TestResult()) + We get only the first failure. @@ -2850,22 +2952,20 @@ def test_unittest_reportflags(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... optionflags=doctest.DONT_ACCEPT_BLANKLINE | doctest.REPORT_NDIFF) - Then the default eporting options are ignored: + Then the default reporting options are ignored: >>> result = suite.run(unittest.TestResult()) >>> result - + *NOTE*: These doctest are intentionally not placed in raw string to depict the trailing whitespace using `\x20` in the diff below. >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - AssertionError: Failed doctest test for test_doctest.txt - ... - Failed example: - favorite_color - ... - Failed example: + Traceback ... + File ... + >...>> if 1: + AssertionError: Failed example: if 1: print('a') print() @@ -3270,7 +3370,7 @@ def test_testmod_errors(): r""" ~^~ ZeroDivisionError: division by zero ********************************************************************** - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line 37, in test.test_doctest.sample_doctest_errors.__test__.bad Failed example: 2 + 2 Expected: @@ -3278,7 +3378,7 @@ def test_testmod_errors(): r""" Got: 4 ********************************************************************** - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line 39, in test.test_doctest.sample_doctest_errors.__test__.bad Failed example: 1/0 Exception raised: @@ -3673,9 +3773,9 @@ def test_run_doctestsuite_multiple_times(): >>> import test.test_doctest.sample_doctest >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) >>> suite.run(unittest.TestResult()) - + >>> suite.run(unittest.TestResult()) - + """ @@ -3855,8 +3955,7 @@ def test_syntax_error_with_incorrect_expected_note(): def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(doctest)) - from test.support.rustpython import DocTestChecker # TODO: RUSTPYTHON - tests.addTest(doctest.DocTestSuite(checker=DocTestChecker())) # TODO: RUSTPYTHON + tests.addTest(doctest.DocTestSuite()) return tests diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 04942effa1f..ac24110cf63 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -74,6 +74,9 @@ def test_update(self): od.update(dict(pairs)) self.assertEqual(sorted(od.items()), pairs) # dict input od = OrderedDict() + od.update(frozendict(pairs)) + self.assertEqual(sorted(od.items()), pairs) # frozendict input + od = OrderedDict() od.update(**dict(pairs)) self.assertEqual(sorted(od.items()), pairs) # kwds input od = OrderedDict() @@ -288,9 +291,11 @@ def test_equality(self): pairs = pairs[2:] + pairs[:2] od2 = OrderedDict(pairs) self.assertNotEqual(od1, od2) # different order implies inequality - # comparison to regular dict is not order sensitive + # comparison to regular (frozen)dict is not order sensitive self.assertEqual(od1, dict(od2)) self.assertEqual(dict(od2), od1) + self.assertEqual(od1, frozendict(od2)) + self.assertEqual(frozendict(od1), od2) # different length implied inequality self.assertNotEqual(od1, OrderedDict(pairs[:-1])) @@ -698,6 +703,7 @@ def test_merge_operator(self): d |= list(b.items()) expected = OrderedDict({0: 0, 1: 1, 2: 2, 3: 3}) self.assertEqual(a | dict(b), expected) + self.assertEqual(a | frozendict(b), expected) self.assertEqual(a | b, expected) self.assertEqual(c, expected) self.assertEqual(d, expected) @@ -706,12 +712,17 @@ def test_merge_operator(self): c |= a expected = OrderedDict({1: 1, 2: 1, 3: 3, 0: 0}) self.assertEqual(dict(b) | a, expected) + self.assertEqual(frozendict(b) | a, expected) + self.assertEqual(a.__ror__(frozendict(b)), expected) self.assertEqual(b | a, expected) self.assertEqual(c, expected) self.assertIs(type(a | b), OrderedDict) self.assertIs(type(dict(a) | b), OrderedDict) + self.assertIs(type(frozendict(a) | b), frozendict) + self.assertIs(type(b.__ror__(frozendict(a))), OrderedDict) self.assertIs(type(a | dict(b)), OrderedDict) + self.assertIs(type(a | frozendict(b)), OrderedDict) expected = a.copy() a |= () diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 6f465b73f4b..65f56b5c523 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -166,21 +166,20 @@ def test_randomize(self): ns = self.parse_args([opt]) self.assertTrue(ns.randomize) - with os_helper.EnvironmentVarGuard() as env: - # with SOURCE_DATE_EPOCH - env['SOURCE_DATE_EPOCH'] = '1697839080' - ns = self.parse_args(['--randomize']) - regrtest = main.Regrtest(ns) - self.assertFalse(regrtest.randomize) - self.assertIsInstance(regrtest.random_seed, str) - self.assertEqual(regrtest.random_seed, '1697839080') - - # without SOURCE_DATE_EPOCH - del env['SOURCE_DATE_EPOCH'] - ns = self.parse_args(['--randomize']) - regrtest = main.Regrtest(ns) - self.assertTrue(regrtest.randomize) - self.assertIsInstance(regrtest.random_seed, int) + @os_helper.with_source_date_epoch(epoch=1697839080) + def test_randomize_with_source_date_epoch(self): + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertFalse(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, str) + self.assertEqual(regrtest.random_seed, '1697839080') + + @os_helper.without_source_date_epoch + def test_randomize_without_source_date_epoch(self): + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertTrue(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, int) def test_no_randomize(self): ns = self.parse_args([]) @@ -571,6 +570,13 @@ def test_single_process(self): self.assertEqual(regrtest.num_workers, 0) self.assertTrue(regrtest.single_process) + def test_pythoninfo(self): + ns = self.parse_args([]) + self.assertFalse(ns.pythoninfo) + + ns = self.parse_args(['--pythoninfo']) + self.assertTrue(ns.pythoninfo) + @dataclasses.dataclass(slots=True) class Rerun: @@ -2141,7 +2147,7 @@ def load_tests(loader, tests, pattern): self.check_executed_tests(output, [testname], failed=[testname], parallel=True, - stats=TestStats(1, 1, 0)) + stats=TestStats(1, 2, 1)) def _check_random_seed(self, run_workers: bool): # gh-109276: When -r/--randomize is used, random.seed() is called @@ -2428,6 +2434,11 @@ def test_pgo_exclude(self): self.assertNotIn('test_re', tests) self.assertEqual(len(tests), len(pgo_tests) - 1) + def test_pythoninfo(self): + testname = self.create_test() + output = self.run_tests('--pythoninfo', testname) + self.assertIn("Python build information", output) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 1dec947ea76..df95e558949 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -406,11 +406,12 @@ def testAssertWarnsRegex(self): # test warning raised but with wrong message def raise_wrong_message(): warnings.warn('foo') - self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'regex'), - raise_wrong_message, - ['^"regex" does not match "foo"$', '^oops$', - '^"regex" does not match "foo"$', - '^"regex" does not match "foo" : oops$']) + with self.assertWarnsRegex(UserWarning, 'foo'): + self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'regex'), + raise_wrong_message, + ['^"regex" does not match "foo"$', '^oops$', + '^"regex" does not match "foo"$', + '^"regex" does not match "foo" : oops$']) if __name__ == "__main__": diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index d66cab146af..83b42918e1e 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1631,11 +1631,11 @@ def testAssertRaisesRegexNoExceptionType(self): self.assertRaisesRegex((ValueError, object), 'expect') def testAssertWarnsCallable(self): - def _runtime_warn(): - warnings.warn("foo", RuntimeWarning) + def _runtime_warn(categories=(RuntimeWarning,)): + for category in categories: + warnings.warn("foo", category) # Success when the right warning is triggered, even several times - self.assertWarns(RuntimeWarning, _runtime_warn) - self.assertWarns(RuntimeWarning, _runtime_warn) + self.assertWarns(RuntimeWarning, _runtime_warn, (RuntimeWarning, RuntimeWarning)) # A tuple of warning classes is accepted self.assertWarns((DeprecationWarning, RuntimeWarning), _runtime_warn) # *args and **kwargs also work @@ -1648,22 +1648,35 @@ def _runtime_warn(): with self.assertRaises(TypeError): self.assertWarns(RuntimeWarning, None) # Failure when another warning is triggered - with warnings.catch_warnings(): + with warnings.catch_warnings(record=True) as log: # Force default filter (in case tests are run with -We) warnings.simplefilter("default", RuntimeWarning) with self.assertRaises(self.failureException): - self.assertWarns(DeprecationWarning, _runtime_warn) + self.assertWarns(DeprecationWarning, _runtime_warn, + (RuntimeWarning, RuntimeWarning)) + self.assertEqual(len(log), 1, log) + self.assertIsInstance(log[0].message, RuntimeWarning) # Filters for other warnings are not modified with warnings.catch_warnings(): warnings.simplefilter("error", RuntimeWarning) with self.assertRaises(RuntimeWarning): self.assertWarns(DeprecationWarning, _runtime_warn) + # Warnings that do not match the category are not swallowed. + with self.assertWarns(RuntimeWarning): + with self.assertRaises(self.failureException): + self.assertWarns(DeprecationWarning, _runtime_warn) + with self.assertWarns(RuntimeWarning): + self.assertWarns(DeprecationWarning, _runtime_warn, + (RuntimeWarning, DeprecationWarning)) + with self.assertWarns(RuntimeWarning): + self.assertWarns(DeprecationWarning, _runtime_warn, + (DeprecationWarning, RuntimeWarning)) def testAssertWarnsContext(self): # Believe it or not, it is preferable to duplicate all tests above, # to make sure the __warningregistry__ $@ is circumvented correctly. - def _runtime_warn(): - warnings.warn("foo", RuntimeWarning) + def _runtime_warn(category=RuntimeWarning): + warnings.warn("foo", category) _runtime_warn_lineno = inspect.getsourcelines(_runtime_warn)[1] with self.assertWarns(RuntimeWarning) as cm: _runtime_warn() @@ -1694,18 +1707,58 @@ def _runtime_warn(): with self.assertWarns(RuntimeWarning, foobar=42): pass # Failure when another warning is triggered - with warnings.catch_warnings(): + with warnings.catch_warnings(record=True) as log: # Force default filter (in case tests are run with -We) warnings.simplefilter("default", RuntimeWarning) with self.assertRaises(self.failureException): with self.assertWarns(DeprecationWarning): _runtime_warn() + _runtime_warn() + self.assertEqual(len(log), 1, log) + self.assertIsInstance(log[0].message, RuntimeWarning) + with warnings.catch_warnings(record=True) as log: + # Force default filter (in case tests are run with -We) + warnings.simplefilter("error", RuntimeWarning) + warnings.filterwarnings("default", category=RuntimeWarning, + module=__name__) + with self.assertRaises(self.failureException): + with self.assertWarns(DeprecationWarning): + _runtime_warn() + _runtime_warn() + self.assertEqual(len(log), 1, log) + self.assertIsInstance(log[0].message, RuntimeWarning) # Filters for other warnings are not modified with warnings.catch_warnings(): warnings.simplefilter("error", RuntimeWarning) with self.assertRaises(RuntimeWarning): with self.assertWarns(DeprecationWarning): _runtime_warn() + # Warnings that do not match the category are not swallowed. + with self.assertWarns(RuntimeWarning): + with self.assertRaises(self.failureException): + with self.assertWarns(DeprecationWarning): + _runtime_warn() + with self.assertWarns(RuntimeWarning): + with self.assertWarns(DeprecationWarning): + _runtime_warn() + _runtime_warn(DeprecationWarning) + with self.assertWarns(RuntimeWarning): + with self.assertWarns(DeprecationWarning): + _runtime_warn(DeprecationWarning) + _runtime_warn() + # Filters by module name work for other warnings. + with warnings.catch_warnings(record=True) as log: + warnings.filterwarnings("error", category=RuntimeWarning) + warnings.filterwarnings("default", category=RuntimeWarning, + module=re.escape(__name__)) + warnings.filterwarnings("error", category=RuntimeWarning, + module='test_case') + with self.assertWarns(DeprecationWarning): + _runtime_warn(DeprecationWarning) + _runtime_warn() + _runtime_warn() + self.assertEqual(len(log), 1, log) + self.assertIsInstance(log[0].message, RuntimeWarning) def testAssertWarnsNoExceptionType(self): with self.assertRaises(TypeError): @@ -1722,8 +1775,9 @@ def testAssertWarnsNoExceptionType(self): self.assertWarns((UserWarning, Exception)) def testAssertWarnsRegexCallable(self): - def _runtime_warn(msg): - warnings.warn(msg, RuntimeWarning) + def _runtime_warn(*msgs): + for msg in msgs: + warnings.warn(msg, RuntimeWarning) self.assertWarnsRegex(RuntimeWarning, "o+", _runtime_warn, "foox") # Failure when no warning is triggered @@ -1734,16 +1788,26 @@ def _runtime_warn(msg): with self.assertRaises(TypeError): self.assertWarnsRegex(RuntimeWarning, "o+", None) # Failure when another warning is triggered - with warnings.catch_warnings(): + with warnings.catch_warnings(record=True) as log: # Force default filter (in case tests are run with -We) warnings.simplefilter("default", RuntimeWarning) with self.assertRaises(self.failureException): self.assertWarnsRegex(DeprecationWarning, "o+", - _runtime_warn, "foox") - # Failure when message doesn't match - with self.assertRaises(self.failureException): + _runtime_warn, "foox", "foox") + self.assertEqual(len(log), 1, log) + self.assertIsInstance(log[0].message, RuntimeWarning) + # Failure when message doesn't match. + # Warnings that do not match the regex are not swallowed. + with self.assertWarnsRegex(RuntimeWarning, "ar"): + with self.assertRaises(self.failureException): + self.assertWarnsRegex(RuntimeWarning, "o+", + _runtime_warn, "barz") + with self.assertWarnsRegex(RuntimeWarning, "ar"): + self.assertWarnsRegex(RuntimeWarning, "o+", + _runtime_warn, "barz", "foox") + with self.assertWarnsRegex(RuntimeWarning, "ar"): self.assertWarnsRegex(RuntimeWarning, "o+", - _runtime_warn, "barz") + _runtime_warn, "foox", "barz") # A little trickier: we ask RuntimeWarnings to be raised, and then # check for some of them. It is implementation-defined whether # non-matching RuntimeWarnings are simply re-raised, or produce a @@ -1778,16 +1842,29 @@ def _runtime_warn(msg): with self.assertWarnsRegex(RuntimeWarning, 'o+', foobar=42): pass # Failure when another warning is triggered - with warnings.catch_warnings(): + with warnings.catch_warnings(record=True) as log: # Force default filter (in case tests are run with -We) warnings.simplefilter("default", RuntimeWarning) with self.assertRaises(self.failureException): with self.assertWarnsRegex(DeprecationWarning, "o+"): _runtime_warn("foox") - # Failure when message doesn't match - with self.assertRaises(self.failureException): + _runtime_warn("foox") + self.assertEqual(len(log), 1, log) + self.assertIsInstance(log[0].message, RuntimeWarning) + # Failure when message doesn't match. + # Warnings that do not match the regex are not swallowed. + with self.assertWarnsRegex(RuntimeWarning, "ar"): + with self.assertRaises(self.failureException): + with self.assertWarnsRegex(RuntimeWarning, "o+"): + _runtime_warn("barz") + with self.assertWarnsRegex(RuntimeWarning, "ar"): with self.assertWarnsRegex(RuntimeWarning, "o+"): _runtime_warn("barz") + _runtime_warn("foox") + with self.assertWarnsRegex(RuntimeWarning, "ar"): + with self.assertWarnsRegex(RuntimeWarning, "o+"): + _runtime_warn("foox") + _runtime_warn("barz") # A little trickier: we ask RuntimeWarnings to be raised, and then # check for some of them. It is implementation-defined whether # non-matching RuntimeWarnings are simply re-raised, or produce a @@ -1797,6 +1874,19 @@ def _runtime_warn(msg): with self.assertRaises((RuntimeWarning, self.failureException)): with self.assertWarnsRegex(RuntimeWarning, "o+"): _runtime_warn("barz") + # Filters by module name work for warnings with other message. + with warnings.catch_warnings(record=True) as log: + warnings.filterwarnings("error", category=RuntimeWarning) + warnings.filterwarnings("default", category=RuntimeWarning, + module=re.escape(__name__)) + warnings.filterwarnings("error", category=RuntimeWarning, + module='test_case') + with self.assertWarnsRegex(RuntimeWarning, "ar"): + _runtime_warn("bar") + _runtime_warn("foox") + _runtime_warn("foox") + self.assertEqual(len(log), 1, log) + self.assertIsInstance(log[0].message, RuntimeWarning) def testAssertWarnsRegexNoExceptionType(self): with self.assertRaises(TypeError): @@ -1920,6 +2010,22 @@ def testAssertLogsUnexpectedException(self): with self.assertLogs(): raise ZeroDivisionError("Unexpected") + def testAssertLogsWithFormatter(self): + # Check alternative formats will be respected + format = "[No.1: the larch] %(levelname)s:%(name)s:%(message)s" + formatter = logging.Formatter(format) + with self.assertNoStderr(): + with self.assertLogs() as cm: + log_foo.info("1") + log_foobar.debug("2") + self.assertEqual(cm.output, ["INFO:foo:1"]) + self.assertLogRecords(cm.records, [{'name': 'foo'}]) + with self.assertLogs(formatter=formatter) as cm: + log_foo.info("1") + log_foobar.debug("2") + self.assertEqual(cm.output, ["[No.1: the larch] INFO:foo:1"]) + self.assertLogRecords(cm.records, [{'name': 'foo'}]) + def testAssertNoLogsDefault(self): with self.assertRaises(self.failureException) as cm: with self.assertNoLogs(): diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index cdff6d1a20c..f4e50a3dccb 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,3 +1,4 @@ +import abc import functools import sys import types @@ -98,6 +99,22 @@ def test_loadTestsFromTestCase__from_FunctionTestCase(self): self.assertIsInstance(suite, loader.suiteClass) self.assertEqual(list(suite), []) + # "Do not load any tests from a TestCase-derived class that is an abstract + # base class." + def test_loadTestsFromTestCase__from_abc_TestCase(self): + class FooBase(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test(self): ... + class Foo(FooBase): + def test(self): pass + + empty_suite = unittest.TestSuite() + + loader = unittest.TestLoader() + suite = loader.loadTestsFromTestCase(Foo) + self.assertEqual(loader.loadTestsFromTestCase(FooBase), empty_suite) + self.assertEqual(list(suite), [Foo('test')]) + ################################################################ ### /Tests for TestLoader.loadTestsFromTestCase @@ -252,6 +269,24 @@ def load_tests(loader, tests, pattern): self.assertRaisesRegex(TypeError, "some failure", test.m) + # Check that loadTestsFromModule skips abstract base classes derived from + # TestCase, which can't be instantiated. + def test_loadTestsFromModule__skip_abc_TestCase(self): + m = types.ModuleType('m') + class MyTestCaseBase(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test(self): + ... + class MyTestCase(MyTestCaseBase): + def test(self): + pass + m.testcase_1 = MyTestCaseBase + m.testcase_2 = MyTestCase + loader = unittest.TestLoader() + suite = loader.loadTestsFromModule(m) + expected = [loader.suiteClass([MyTestCase('test')])] + self.assertEqual(list(suite), expected) + ################################################################ ### /Tests for TestLoader.loadTestsFromModule() @@ -1052,6 +1087,22 @@ def test_loadTestsFromNames__module_not_loaded(self): if module_name in sys.modules: del sys.modules[module_name] + # "The specifier should not refer to a test method in a TestCase-derived + # subclass that is an abstract base class" + def test_loadTestsFromNames__testmethod_in_abc_TestCase(self): + m = types.ModuleType('m') + class Foo(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test_1(self): ... + def test_2(self): pass + m.Foo = Foo + + loader = unittest.TestLoader() + for name in 'Foo.test_1', 'Foo.test_2': + with self.subTest(name=name), self.assertRaisesRegex(TypeError, + "Cannot instantiate abstract test case Foo"): + loader.loadTestsFromNames([name], m) + ################################################################ ### /Tests for TestLoader.loadTestsFromNames() diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 9ac4c52449c..3f44e617303 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -1282,14 +1282,22 @@ def setUpModule(): suite(result) expected_out = '\nStdout:\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 1) + self.assertEqual(len(result.errors), 2) description = 'tearDownModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferSetUpModule_DoModuleCleanups(self): with captured_stdout() as stdout: result = unittest.TestResult() @@ -1313,22 +1321,34 @@ def setUpModule(): suite(result) expected_out = '\nStdout:\nset up module\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 2) + self.assertEqual(len(result.errors), 3) description = 'setUpModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ValueError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn('\nStdout:\nset up module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] self.assertIn(expected_out, formatted_exc) self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ZeroDivisionError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertIn(expected_out, formatted_exc) + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferTearDownModule_DoModuleCleanups(self): with captured_stdout() as stdout: result = unittest.TestResult() @@ -1355,21 +1375,32 @@ def tearDownModule(): suite(result) expected_out = '\nStdout:\ntear down module\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 2) + self.assertEqual(len(result.errors), 3) description = 'tearDownModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ValueError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn('\nStdout:\ntear down module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ZeroDivisionError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 4d3cfd60b8d..a47e2ebb59d 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -13,6 +13,7 @@ LoggingResult, ResultWithNoStartTestRunStopTestRun, ) +from test.support.testcase import ExceptionIsLikeMixin def resultFactory(*_): @@ -604,7 +605,7 @@ class EmptyTest(unittest.TestCase): @support.force_not_colorized_test_class -class TestModuleCleanUp(unittest.TestCase): +class TestModuleCleanUp(ExceptionIsLikeMixin, unittest.TestCase): def test_add_and_do_ModuleCleanup(self): module_cleanups = [] @@ -646,11 +647,50 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(CustomError) as e: + with self.assertRaises(Exception) as e: unittest.case.doModuleCleanups() - self.assertEqual(str(e.exception), 'CleanUpExc') + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', + [CustomError('CleanUpExc')])) self.assertEqual(unittest.case._module_cleanups, []) + def test_doModuleCleanup_with_multiple_errors_in_addModuleCleanup(self): + def module_cleanup_bad1(): + raise TypeError('CleanUpExc1') + + def module_cleanup_bad2(): + raise ValueError('CleanUpExc2') + + class Module: + unittest.addModuleCleanup(module_cleanup_bad1) + unittest.addModuleCleanup(module_cleanup_bad2) + with self.assertRaises(ExceptionGroup) as e: + unittest.case.doModuleCleanups() + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ])) + + def test_doModuleCleanup_with_exception_group_in_addModuleCleanup(self): + def module_cleanup_bad(): + raise ExceptionGroup('CleanUpExc', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ]) + + class Module: + unittest.addModuleCleanup(module_cleanup_bad) + with self.assertRaises(ExceptionGroup) as e: + unittest.case.doModuleCleanups() + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', [ + ExceptionGroup('CleanUpExc', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ]), + ])) + def test_addModuleCleanup_arg_errors(self): cleanups = [] def cleanup(*args, **kwargs): @@ -871,9 +911,11 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(CustomError) as cm: + with self.assertRaises(Exception) as cm: suite.debug() - self.assertEqual(str(cm.exception), 'CleanUpExc') + self.assertExceptionIsLike(cm.exception, + ExceptionGroup('module cleanup failed', + [CustomError('CleanUpExc')])) self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) self.assertEqual(unittest.case._module_cleanups, []) diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py index d590a333930..09ce09b91b7 100644 --- a/Lib/test/test_unittest/test_util.py +++ b/Lib/test/test_unittest/test_util.py @@ -26,6 +26,39 @@ def test_sorted_list_difference(self): self.assertEqual(sorted_list_difference([2], [1, 1]), ([2], [1])) self.assertEqual(sorted_list_difference([1, 2], [1, 1]), ([2], [])) + def test_sorted_list_difference_tail_deduplication(self): + # Tail deduplication when one list is exhausted before the other. + # These exercise the except-IndexError path in sorted_list_difference. + self.assertEqual(sorted_list_difference([], [0, 0]), ([], [0])) + self.assertEqual(sorted_list_difference([0, 0], []), ([0], [])) + self.assertEqual(sorted_list_difference([], [1, 1, 2, 2]), ([], [1, 2])) + self.assertEqual(sorted_list_difference([1, 1, 2, 2], []), ([1, 2], [])) + # One list exhausts mid-way, leaving duplicated tail in the other. + self.assertEqual(sorted_list_difference([1], [1, 2, 2, 3, 3]), ([], [2, 3])) + self.assertEqual(sorted_list_difference([1, 2, 2, 3, 3], [1]), ([2, 3], [])) + + def test_sorted_list_difference_strings(self): + self.assertEqual( + sorted_list_difference(['a', 'b'], ['b', 'c']), + (['a'], ['c'])) + self.assertEqual( + sorted_list_difference([], ['a', 'a', 'b']), + ([], ['a', 'b'])) + self.assertEqual( + sorted_list_difference(['a', 'a', 'b'], []), + (['a', 'b'], [])) + + def test_sorted_list_difference_unhashable(self): + self.assertEqual( + sorted_list_difference([[1], [2]], [[2], [3]]), + ([[1]], [[3]])) + self.assertEqual( + sorted_list_difference([], [[0], [0]]), + ([], [[0]])) + self.assertEqual( + sorted_list_difference([[0], [0]], []), + ([[0]], [])) + def test_unorderable_list_difference(self): self.assertEqual(unorderable_list_difference([], []), ([], [])) self.assertEqual(unorderable_list_difference([1, 2], []), ([2, 1], [])) diff --git a/Lib/unittest/_log.py b/Lib/unittest/_log.py index 94868e5bb95..3d69385ea24 100644 --- a/Lib/unittest/_log.py +++ b/Lib/unittest/_log.py @@ -30,7 +30,7 @@ class _AssertLogsContext(_BaseTestCaseContext): LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" - def __init__(self, test_case, logger_name, level, no_logs): + def __init__(self, test_case, logger_name, level, no_logs, formatter=None): _BaseTestCaseContext.__init__(self, test_case) self.logger_name = logger_name if level: @@ -39,13 +39,14 @@ def __init__(self, test_case, logger_name, level, no_logs): self.level = logging.INFO self.msg = None self.no_logs = no_logs + self.formatter = formatter def __enter__(self): if isinstance(self.logger_name, logging.Logger): logger = self.logger = self.logger_name else: logger = self.logger = logging.getLogger(self.logger_name) - formatter = logging.Formatter(self.LOGGING_FORMAT) + formatter = self.formatter or logging.Formatter(self.LOGGING_FORMAT) handler = _CapturingHandler() handler.setLevel(self.level) handler.setFormatter(formatter) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 884fc1b21f6..a392238c85a 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -149,9 +149,7 @@ def doModuleCleanups(): except Exception as exc: exceptions.append(exc) if exceptions: - # Swallows all but first exception. If a multi-exception handler - # gets written we should use that here instead. - raise exceptions[0] + raise ExceptionGroup('module cleanup failed', exceptions) def skip(reason): @@ -303,7 +301,7 @@ def __enter__(self): v.__warningregistry__ = {} self.warnings_manager = warnings.catch_warnings(record=True) self.warnings = self.warnings_manager.__enter__() - warnings.simplefilter("always", self.expected) + warnings.simplefilter("always") return self def __exit__(self, exc_type, exc_value, tb): @@ -316,19 +314,44 @@ def __exit__(self, exc_type, exc_value, tb): except AttributeError: exc_name = str(self.expected) first_matching = None + matched = False + non_matching_warnings = [] for m in self.warnings: w = m.message if not isinstance(w, self.expected): + non_matching_warnings.append(m) continue if first_matching is None: first_matching = w if (self.expected_regex is not None and not self.expected_regex.search(str(w))): + non_matching_warnings.append(m) continue + if matched: + continue + matched = True # store warning for later retrieval self.warning = w self.filename = m.filename self.lineno = m.lineno + for m in non_matching_warnings: + module = m.module + module_globals = None + registry = None + if module is not None: + try: + module_globals = vars(sys.modules[module]) + except (KeyError, TypeError): + # module == "" or sys.modules[module] is None + pass + else: + registry = module_globals.setdefault("__warningregistry__", {}) + warnings.warn_explicit(m.message, m.category, m.filename, m.lineno, + module=module, + registry=registry, + module_globals=module_globals, + source=m.source) + if matched: return # Now we simply try to choose a helpful failure message if first_matching is not None: @@ -340,7 +363,6 @@ def __exit__(self, exc_type, exc_value, tb): else: self._raiseFailure("{} not triggered".format(exc_name)) - class _AssertNotWarnsContext(_AssertWarnsContext): def __exit__(self, exc_type, exc_value, tb): @@ -851,7 +873,7 @@ def _assertNotWarns(self, expected_warning, *args, **kwargs): context = _AssertNotWarnsContext(expected_warning, self) return context.handle('_assertNotWarns', args, kwargs) - def assertLogs(self, logger=None, level=None): + def assertLogs(self, logger=None, level=None, formatter=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to INFO and *logger* defaults to the root logger. @@ -863,6 +885,8 @@ def assertLogs(self, logger=None, level=None): `records` attribute will be a list of the corresponding LogRecord objects. + Optionally supply `formatter` to control how messages are formatted. + Example:: with self.assertLogs('foo', level='INFO') as cm: @@ -873,7 +897,7 @@ def assertLogs(self, logger=None, level=None): """ # Lazy import to avoid importing logging if it is not needed. from ._log import _AssertLogsContext - return _AssertLogsContext(self, logger, level, no_logs=False) + return _AssertLogsContext(self, logger, level, no_logs=False, formatter=formatter) def assertNoLogs(self, logger=None, level=None): """ Fail unless no log messages of level *level* or higher are emitted diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index a52950dad22..697520246f0 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -1,5 +1,6 @@ """Loading unittests.""" +import inspect import os import re import sys @@ -84,8 +85,10 @@ def loadTestsFromTestCase(self, testCaseClass): raise TypeError("Test cases should not be derived from " "TestSuite. Maybe you meant to derive from " "TestCase?") - if testCaseClass in (case.TestCase, case.FunctionTestCase): - # We don't load any tests from base types that should not be loaded. + if (testCaseClass in (case.TestCase, case.FunctionTestCase) or + inspect.isabstract(testCaseClass)): + # We don't load any tests from base types that should not be loaded, + # and abstract base classes that can't be instantiated testCaseNames = [] else: testCaseNames = self.getTestCaseNames(testCaseClass) @@ -103,6 +106,7 @@ def loadTestsFromModule(self, module, *, pattern=None): isinstance(obj, type) and issubclass(obj, case.TestCase) and obj not in (case.TestCase, case.FunctionTestCase) + and not inspect.isabstract(obj) ): tests.append(self.loadTestsFromTestCase(obj)) @@ -181,6 +185,9 @@ def loadTestsFromName(self, name, module=None): elif (isinstance(obj, types.FunctionType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): + if inspect.isabstract(parent): + raise TypeError( + "Cannot instantiate abstract test case %s" % parent.__name__) name = parts[-1] inst = parent(name) # static methods follow a different path diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index be99d93c78c..6eeebf9657a 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -172,7 +172,7 @@ def _getParentArgParser(self): help='Show local variables in tracebacks') parser.add_argument('--durations', dest='durations', type=int, default=None, metavar="N", - help='Show the N slowest test cases (N=0 for all)') + help='Show the `N` slowest test cases (`N=0` for all)') if self.failfast is None: parser.add_argument('-f', '--failfast', dest='failfast', action='store_true', @@ -181,12 +181,12 @@ def _getParentArgParser(self): if self.catchbreak is None: parser.add_argument('-c', '--catch', dest='catchbreak', action='store_true', - help='Catch Ctrl-C and display results so far') + help='Catch `Ctrl-C` and display results so far') self.catchbreak = False if self.buffer is None: parser.add_argument('-b', '--buffer', dest='buffer', action='store_true', - help='Buffer stdout and stderr during tests') + help='Buffer `stdout` and `stderr` during tests') self.buffer = False if self.testNamePatterns is None: parser.add_argument('-k', dest='testNamePatterns', @@ -197,7 +197,7 @@ def _getParentArgParser(self): return parser def _getMainArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent], color=True) + parser = argparse.ArgumentParser(parents=[parent]) parser.prog = self.progName parser.print_help = self._print_help @@ -208,16 +208,16 @@ def _getMainArgParser(self, parent): return parser def _getDiscoveryArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent], color=True) + parser = argparse.ArgumentParser(parents=[parent]) parser.prog = '%s discover' % self.progName parser.epilog = ('For test discovery all test modules must be ' 'importable from the top level directory of the ' 'project.') parser.add_argument('-s', '--start-directory', dest='start', - help="Directory to start discovery ('.' default)") + help="Directory to start discovery (`.` default)") parser.add_argument('-p', '--pattern', dest='pattern', - help="Pattern to match tests ('test*.py' default)") + help="Pattern to match tests (`test*.py` default)") parser.add_argument('-t', '--top-level-directory', dest='top', help='Top level directory of project (defaults to ' 'start directory)') diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 5f22d91aebd..893fcba968c 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,11 +4,10 @@ import time import warnings -from _colorize import get_theme - from . import result from .case import _SubTest from .signals import registerResult +lazy from _colorize import get_theme __unittest = True diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 6f45b6fe5f6..ae9ca2d615d 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -223,6 +223,11 @@ def _handleModuleFixture(self, test, result): if result._moduleSetUpFailed: try: case.doModuleCleanups() + except ExceptionGroup as eg: + for e in eg.exceptions: + self._createClassOrModuleLevelException(result, e, + 'setUpModule', + currentModule) except Exception as e: self._createClassOrModuleLevelException(result, e, 'setUpModule', @@ -235,15 +240,15 @@ def _createClassOrModuleLevelException(self, result, exc, method_name, errorName = f'{method_name} ({parent})' self._addClassOrModuleLevelException(result, exc, errorName, info) - def _addClassOrModuleLevelException(self, result, exception, errorName, + def _addClassOrModuleLevelException(self, result, exc, errorName, info=None): error = _ErrorHolder(errorName) addSkip = getattr(result, 'addSkip', None) - if addSkip is not None and isinstance(exception, case.SkipTest): - addSkip(error, str(exception)) + if addSkip is not None and isinstance(exc, case.SkipTest): + addSkip(error, str(exc)) else: if not info: - result.addError(error, sys.exc_info()) + result.addError(error, (type(exc), exc, exc.__traceback__)) else: result.addError(error, info) @@ -273,6 +278,13 @@ def _handleModuleTearDown(self, result): previousModule) try: case.doModuleCleanups() + except ExceptionGroup as eg: + if isinstance(result, _DebugResult): + raise + for e in eg.exceptions: + self._createClassOrModuleLevelException(result, e, + 'tearDownModule', + previousModule) except Exception as e: if isinstance(result, _DebugResult): raise diff --git a/Lib/unittest/util.py b/Lib/unittest/util.py index 050eaed0b3f..0681163c979 100644 --- a/Lib/unittest/util.py +++ b/Lib/unittest/util.py @@ -1,7 +1,6 @@ """Various utility functions.""" from collections import namedtuple, Counter -from os.path import commonprefix __unittest = True @@ -21,13 +20,23 @@ def _shorten(s, prefixlen, suffixlen): s = '%s[%d chars]%s' % (s[:prefixlen], skip, s[len(s) - suffixlen:]) return s +def _common_prefix(m): + if not m: + return "" + s1 = min(m) + s2 = max(m) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 + def _common_shorten_repr(*args): args = tuple(map(safe_repr, args)) maxlen = max(map(len, args)) if maxlen <= _MAX_LENGTH: return args - prefix = commonprefix(args) + prefix = _common_prefix(args) prefixlen = len(prefix) common_len = _MAX_LENGTH - \ @@ -54,6 +63,14 @@ def safe_repr(obj, short=False): def strclass(cls): return "%s.%s" % (cls.__module__, cls.__qualname__) +def _dedupe_sorted(lst): + """Remove consecutive duplicate elements from a sorted list.""" + result = [] + for item in lst: + if not result or result[-1] != item: + result.append(item) + return result + def sorted_list_difference(expected, actual): """Finds elements in only one or the other of two, sorted input lists. @@ -89,8 +106,8 @@ def sorted_list_difference(expected, actual): while actual[j] == a: j += 1 except IndexError: - missing.extend(expected[i:]) - unexpected.extend(actual[j:]) + missing.extend(_dedupe_sorted(expected[i:])) + unexpected.extend(_dedupe_sorted(actual[j:])) break return missing, unexpected diff --git a/crates/vm/src/stdlib/_collections.rs b/crates/vm/src/stdlib/_collections.rs index e120e73cbc5..5a11f409d8b 100644 --- a/crates/vm/src/stdlib/_collections.rs +++ b/crates/vm/src/stdlib/_collections.rs @@ -11,8 +11,8 @@ mod _collections { VirtualMachine, atomic_func, builtins::{ IterStatus::{Active, Exhausted}, - PositionIterInternal, PyDict, PyGenericAlias, PyInt, PyList, PyStr, PyTuple, PyType, - PyTypeRef, locked_step, + PositionIterInternal, PyAnyDict, PyDict, PyGenericAlias, PyInt, PyList, PyStr, PyTuple, + PyType, PyTypeRef, locked_step, }, common::lock::{PyMutex, PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}, convert::ToPyObject, @@ -1072,7 +1072,7 @@ mod _collections { rhs.class().name() ))); }; - if !other.fast_isinstance(vm.ctx.types.dict_type) { + if PyAnyDict::from_object(other).is_none() { return not_implemented(); } diff --git a/crates/vm/src/stdlib/_collections/ordered_dict.rs b/crates/vm/src/stdlib/_collections/ordered_dict.rs index 1a2222c7bba..c02887b4389 100644 --- a/crates/vm/src/stdlib/_collections/ordered_dict.rs +++ b/crates/vm/src/stdlib/_collections/ordered_dict.rs @@ -684,9 +684,7 @@ pub(crate) mod ordered_dict { fn as_number() -> &'static PyNumberMethods { static AS_NUMBER: PyNumberMethods = PyNumberMethods { or: Some(|a, b, vm| { - if !a.fast_isinstance(vm.ctx.types.dict_type) - || !b.fast_isinstance(vm.ctx.types.dict_type) - { + if PyAnyDict::from_object(a).is_none() || PyAnyDict::from_object(b).is_none() { return Ok(vm.ctx.not_implemented()); } let cls = if let Some(od) = a.downcast_ref::() {