From e99073911a624df94602930e29823c100fb763ba Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 10 May 2026 13:25:33 +0300 Subject: [PATCH 1/7] Update `_pyrepl` to 3.14.4 --- Lib/_pyrepl/__main__.py | 4 + Lib/_pyrepl/_minimal_curses.py | 68 ----- Lib/_pyrepl/_module_completer.py | 426 ++++++++++++++++++++++++++ Lib/_pyrepl/base_eventqueue.py | 110 +++++++ Lib/_pyrepl/commands.py | 58 ++-- Lib/_pyrepl/completing_reader.py | 6 +- Lib/_pyrepl/console.py | 22 +- Lib/_pyrepl/curses.py | 33 -- Lib/_pyrepl/fancy_termios.py | 44 +-- Lib/_pyrepl/keymap.py | 18 +- Lib/_pyrepl/main.py | 11 +- Lib/_pyrepl/mypy.ini | 3 +- Lib/_pyrepl/reader.py | 224 ++++++-------- Lib/_pyrepl/readline.py | 40 ++- Lib/_pyrepl/simple_interact.py | 36 ++- Lib/_pyrepl/terminfo.py | 488 ++++++++++++++++++++++++++++++ Lib/_pyrepl/trace.py | 27 +- Lib/_pyrepl/types.py | 14 +- Lib/_pyrepl/unix_console.py | 131 +++++--- Lib/_pyrepl/unix_eventqueue.py | 95 +----- Lib/_pyrepl/utils.py | 374 ++++++++++++++++++++++- Lib/_pyrepl/windows_console.py | 256 +++++++++++----- Lib/_pyrepl/windows_eventqueue.py | 42 +++ 23 files changed, 1992 insertions(+), 538 deletions(-) delete mode 100644 Lib/_pyrepl/_minimal_curses.py create mode 100644 Lib/_pyrepl/_module_completer.py create mode 100644 Lib/_pyrepl/base_eventqueue.py delete mode 100644 Lib/_pyrepl/curses.py create mode 100644 Lib/_pyrepl/terminfo.py create mode 100644 Lib/_pyrepl/windows_eventqueue.py diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index 3fa992eee8e..9c66812e13a 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,6 +1,10 @@ # Important: don't add things to this module, as they will end up in the REPL's # default globals. Use _pyrepl.main instead. +# Avoid caching this file by linecache and incorrectly report tracebacks. +# See https://github.com/python/cpython/issues/129098. +__spec__ = __loader__ = None + if __name__ == "__main__": from .main import interactive_console as __pyrepl_interactive_console __pyrepl_interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py deleted file mode 100644 index d884f880f50..00000000000 --- a/Lib/_pyrepl/_minimal_curses.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Minimal '_curses' module, the low-level interface for curses module -which is not meant to be used directly. - -Based on ctypes. It's too incomplete to be really called '_curses', so -to use it, you have to import it and stick it in sys.modules['_curses'] -manually. - -Note that there is also a built-in module _minimal_curses which will -hide this one if compiled in. -""" - -import ctypes -import ctypes.util - - -class error(Exception): - pass - - -def _find_clib() -> str: - trylibs = ["ncursesw", "ncurses", "curses"] - - for lib in trylibs: - path = ctypes.util.find_library(lib) - if path: - return path - raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") - - -_clibpath = _find_clib() -clib = ctypes.cdll.LoadLibrary(_clibpath) - -clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] -clib.setupterm.restype = ctypes.c_int - -clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.c_ssize_t - -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] -clib.tparm.restype = ctypes.c_char_p - -OK = 0 -ERR = -1 - -# ____________________________________________________________ - - -def setupterm(termstr, fd): - err = ctypes.c_int(0) - result = clib.setupterm(termstr, fd, ctypes.byref(err)) - if result == ERR: - raise error("setupterm() failed (err=%d)" % err.value) - - -def tigetstr(cap): - if not isinstance(cap, bytes): - cap = cap.encode("ascii") - result = clib.tigetstr(cap) - if result == ERR: - return None - return ctypes.cast(result, ctypes.c_char_p).value - - -def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): - result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) - if result is None: - raise error("tparm() returned NULL") - return result diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py new file mode 100644 index 00000000000..2098d0a54ab --- /dev/null +++ b/Lib/_pyrepl/_module_completer.py @@ -0,0 +1,426 @@ +from __future__ import annotations + +import importlib +import os +import pkgutil +import sys +import token +import tokenize +from importlib.machinery import FileFinder +from io import StringIO +from contextlib import contextmanager +from dataclasses import dataclass +from itertools import chain +from tokenize import TokenInfo + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any, Iterable, Iterator, Mapping + + +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + + +def make_default_module_completer() -> ModuleCompleter: + # Inside pyrepl, __package__ is set to None by default + return ModuleCompleter(namespace={'__package__': None}) + + +class ModuleCompleter: + """A completer for Python import statements. + + Examples: + - import + - import foo + - import foo. + - import foo as bar, baz + + - from + - from foo + - from foo import + - from foo import bar + - from foo import (bar as baz, qux + """ + + def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: + self.namespace = namespace or {} + self._global_cache: list[pkgutil.ModuleInfo] = [] + self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) + + def get_completions(self, line: str) -> list[str] | None: + """Return the next possible import completions for 'line'.""" + result = ImportParser(line).parse() + if not result: + return None + try: + return self.complete(*result) + except Exception: + # Some unexpected error occurred, make it look like + # no completions are available + return [] + + def complete(self, from_name: str | None, name: str | None) -> list[str]: + if from_name is None: + # import x.y.z + assert name is not None + path, prefix = self.get_path_and_prefix(name) + modules = self.find_modules(path, prefix) + return [self.format_completion(path, module) for module in modules] + + if name is None: + # from x.y.z + path, prefix = self.get_path_and_prefix(from_name) + modules = self.find_modules(path, prefix) + return [self.format_completion(path, module) for module in modules] + + # from x.y import z + return self.find_modules(from_name, name) + + def find_modules(self, path: str, prefix: str) -> list[str]: + """Find all modules under 'path' that start with 'prefix'.""" + modules = self._find_modules(path, prefix) + # Filter out invalid module names + # (for example those containing dashes that cannot be imported with 'import') + return [mod for mod in modules if mod.isidentifier()] + + def _find_modules(self, path: str, prefix: str) -> list[str]: + if not path: + # Top-level import (e.g. `import foo`` or `from foo`)` + builtin_modules = [name for name in sys.builtin_module_names + if self.is_suggestion_match(name, prefix)] + third_party_modules = [module.name for module in self.global_cache + if self.is_suggestion_match(module.name, prefix)] + return sorted(builtin_modules + third_party_modules) + + if path.startswith('.'): + # Convert relative path to absolute path + package = self.namespace.get('__package__', '') + path = self.resolve_relative_name(path, package) # type: ignore[assignment] + if path is None: + return [] + + modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + imported_module = sys.modules.get(path.split('.')[0]) + if imported_module: + # Filter modules to those who name and specs match the + # imported module to avoid invalid suggestions + spec = imported_module.__spec__ + if spec: + modules = [mod for mod in modules + if mod.name == spec.name + and mod.module_finder.find_spec(mod.name, None) == spec] + else: + modules = [] + + is_stdlib_import: bool | None = None + for segment in path.split('.'): + modules = [mod_info for mod_info in modules + if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) + modules = self.iter_submodules(modules) + + module_names = [module.name for module in modules] + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] + + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) + + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: + if prefix: + return module_name.startswith(prefix) + # For consistency with attribute completion, which + # does not suggest private attributes unless requested. + return not module_name.startswith("_") + + def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]: + """Iterate over all submodules of the given parent modules.""" + specs = [info.module_finder.find_spec(info.name, None) + for info in parent_modules if info.ispkg] + search_locations = set(chain.from_iterable( + getattr(spec, 'submodule_search_locations', []) + for spec in specs if spec + )) + return pkgutil.iter_modules(search_locations) + + def get_path_and_prefix(self, dotted_name: str) -> tuple[str, str]: + """ + Split a dotted name into an import path and a + final prefix that is to be completed. + + Examples: + 'foo.bar' -> 'foo', 'bar' + 'foo.' -> 'foo', '' + '.foo' -> '.', 'foo' + """ + if '.' not in dotted_name: + return '', dotted_name + if dotted_name.startswith('.'): + stripped = dotted_name.lstrip('.') + dots = '.' * (len(dotted_name) - len(stripped)) + if '.' not in stripped: + return dots, stripped + path, prefix = stripped.rsplit('.', 1) + return dots + path, prefix + path, prefix = dotted_name.rsplit('.', 1) + return path, prefix + + def format_completion(self, path: str, module: str) -> str: + if path == '' or path.endswith('.'): + return f'{path}{module}' + return f'{path}.{module}' + + def resolve_relative_name(self, name: str, package: str) -> str | None: + """Resolve a relative module name to an absolute name. + + Example: resolve_relative_name('.foo', 'bar') -> 'bar.foo' + """ + # taken from importlib._bootstrap + level = 0 + for character in name: + if character != '.': + break + level += 1 + bits = package.rsplit('.', level - 1) + if len(bits) < level: + return None + base = bits[0] + name = name[level:] + return f'{base}.{name}' if name else base + + @property + def global_cache(self) -> list[pkgutil.ModuleInfo]: + """Global module cache""" + if not self._global_cache or self._curr_sys_path != sys.path: + self._curr_sys_path = sys.path[:] + self._global_cache = list(pkgutil.iter_modules()) + return self._global_cache + + +class ImportParser: + """ + Parses incomplete import statements that are + suitable for autocomplete suggestions. + + Examples: + - import foo -> Result(from_name=None, name='foo') + - import foo. -> Result(from_name=None, name='foo.') + - from foo -> Result(from_name='foo', name=None) + - from foo import bar -> Result(from_name='foo', name='bar') + - from .foo import ( -> Result(from_name='.foo', name='') + + Note that the parser works in reverse order, starting from the + last token in the input string. This makes the parser more robust + when parsing multiple statements. + """ + _ignored_tokens = { + token.INDENT, token.DEDENT, token.COMMENT, + token.NL, token.NEWLINE, token.ENDMARKER + } + _keywords = {'import', 'from', 'as'} + + def __init__(self, code: str) -> None: + self.code = code + tokens = [] + try: + for t in tokenize.generate_tokens(StringIO(code).readline): + if t.type not in self._ignored_tokens: + tokens.append(t) + except tokenize.TokenError as e: + if 'unexpected EOF' not in str(e): + # unexpected EOF is fine, since we're parsing an + # incomplete statement, but other errors are not + # because we may not have all the tokens so it's + # safer to bail out + tokens = [] + except SyntaxError: + tokens = [] + self.tokens = TokenQueue(tokens[::-1]) + + def parse(self) -> tuple[str | None, str | None] | None: + if not (res := self._parse()): + return None + return res.from_name, res.name + + def _parse(self) -> Result | None: + with self.tokens.save_state(): + return self.parse_from_import() + with self.tokens.save_state(): + return self.parse_import() + + def parse_import(self) -> Result: + if self.code.rstrip().endswith('import') and self.code.endswith(' '): + return Result(name='') + if self.tokens.peek_string(','): + name = '' + else: + if self.code.endswith(' '): + raise ParseError('parse_import') + name = self.parse_dotted_name() + if name.startswith('.'): + raise ParseError('parse_import') + while self.tokens.peek_string(','): + self.tokens.pop() + self.parse_dotted_as_name() + if self.tokens.peek_string('import'): + return Result(name=name) + raise ParseError('parse_import') + + def parse_from_import(self) -> Result: + stripped = self.code.rstrip() + if stripped.endswith('import') and self.code.endswith(' '): + return Result(from_name=self.parse_empty_from_import(), name='') + if stripped.endswith('from') and self.code.endswith(' '): + return Result(from_name='') + if self.tokens.peek_string('(') or self.tokens.peek_string(','): + return Result(from_name=self.parse_empty_from_import(), name='') + if self.code.endswith(' '): + raise ParseError('parse_from_import') + name = self.parse_dotted_name() + if '.' in name: + self.tokens.pop_string('from') + return Result(from_name=name) + if self.tokens.peek_string('from'): + return Result(from_name=name) + from_name = self.parse_empty_from_import() + return Result(from_name=from_name, name=name) + + def parse_empty_from_import(self) -> str: + if self.tokens.peek_string(','): + self.tokens.pop() + self.parse_as_names() + if self.tokens.peek_string('('): + self.tokens.pop() + self.tokens.pop_string('import') + return self.parse_from() + + def parse_from(self) -> str: + from_name = self.parse_dotted_name() + self.tokens.pop_string('from') + return from_name + + def parse_dotted_as_name(self) -> str: + self.tokens.pop_name() + if self.tokens.peek_string('as'): + self.tokens.pop() + with self.tokens.save_state(): + return self.parse_dotted_name() + + def parse_dotted_name(self) -> str: + name = [] + if self.tokens.peek_string('.'): + name.append('.') + self.tokens.pop() + if (self.tokens.peek_name() + and (tok := self.tokens.peek()) + and tok.string not in self._keywords): + name.append(self.tokens.pop_name()) + if not name: + raise ParseError('parse_dotted_name') + while self.tokens.peek_string('.'): + name.append('.') + self.tokens.pop() + if (self.tokens.peek_name() + and (tok := self.tokens.peek()) + and tok.string not in self._keywords): + name.append(self.tokens.pop_name()) + else: + break + + while self.tokens.peek_string('.'): + name.append('.') + self.tokens.pop() + return ''.join(name[::-1]) + + def parse_as_names(self) -> None: + self.parse_as_name() + while self.tokens.peek_string(','): + self.tokens.pop() + self.parse_as_name() + + def parse_as_name(self) -> None: + self.tokens.pop_name() + if self.tokens.peek_string('as'): + self.tokens.pop() + self.tokens.pop_name() + + +class ParseError(Exception): + pass + + +@dataclass(frozen=True) +class Result: + from_name: str | None = None + name: str | None = None + + +class TokenQueue: + """Provides helper functions for working with a sequence of tokens.""" + + def __init__(self, tokens: list[TokenInfo]) -> None: + self.tokens: list[TokenInfo] = tokens + self.index: int = 0 + self.stack: list[int] = [] + + @contextmanager + def save_state(self) -> Any: + try: + self.stack.append(self.index) + yield + except ParseError: + self.index = self.stack.pop() + else: + self.stack.pop() + + def __bool__(self) -> bool: + return self.index < len(self.tokens) + + def peek(self) -> TokenInfo | None: + if not self: + return None + return self.tokens[self.index] + + def peek_name(self) -> bool: + if not (tok := self.peek()): + return False + return tok.type == token.NAME + + def pop_name(self) -> str: + tok = self.pop() + if tok.type != token.NAME: + raise ParseError('pop_name') + return tok.string + + def peek_string(self, string: str) -> bool: + if not (tok := self.peek()): + return False + return tok.string == string + + def pop_string(self, string: str) -> str: + tok = self.pop() + if tok.string != string: + raise ParseError('pop_string') + return tok.string + + def pop(self) -> TokenInfo: + if not self: + raise ParseError('pop') + tok = self.tokens[self.index] + self.index += 1 + return tok diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py new file mode 100644 index 00000000000..0589a0f437e --- /dev/null +++ b/Lib/_pyrepl/base_eventqueue.py @@ -0,0 +1,110 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +OS-independent base for an event and VT sequence scanner + +See unix_eventqueue and windows_eventqueue for subclasses. +""" + +from collections import deque + +from . import keymap +from .console import Event +from .trace import trace + +class BaseEventQueue: + def __init__(self, encoding: str, keymap_dict: dict[bytes, str]) -> None: + self.compiled_keymap = keymap.compile_keymap(keymap_dict) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + assert isinstance(char, (int, bytes)) + ord_char = char if isinstance(char, int) else ord(char) + char = ord_char.to_bytes() + self.buf.append(ord_char) + + if char in self.keymap: + if self.keymap is self.compiled_keymap: + # sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, bytes(self.flush_buf()))) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', b'\033')) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, bytes(self.flush_buf()))) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 503ca1da329..10127e58897 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -21,6 +21,7 @@ from __future__ import annotations import os +import time # Categories of actions: # killing @@ -31,6 +32,7 @@ # finishing # [completion] +from .trace import trace # types if False: @@ -368,6 +370,13 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) + if r.paste_mode: + data = "" + ev = r.console.getpending() + data += ev.data + if data: + r.insert(data) + r.last_refresh_cache.invalidated = True class insert_nl(EditCommand): @@ -411,14 +420,17 @@ class delete(EditCommand): def do(self) -> None: r = self.reader b = r.buffer - if ( - r.pos == 0 - and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004" - ): - r.update_screen() - r.console.finish() - raise EOFError + if self.event[-1] == "\004": + if b and b[-1].endswith("\n"): + self.finish = True + elif ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] @@ -437,7 +449,7 @@ def do(self) -> None: import _sitebuiltins with self.reader.suspend(): - self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment] class invalid_key(Command): @@ -456,7 +468,7 @@ def do(self) -> None: class show_history(Command): def do(self) -> None: from .pager import get_pager - from site import gethistoryfile # type: ignore[attr-defined] + from site import gethistoryfile history = os.linesep.join(self.reader.history[:]) self.reader.console.restore() @@ -471,19 +483,23 @@ def do(self) -> None: class paste_mode(Command): - def do(self) -> None: self.reader.paste_mode = not self.reader.paste_mode self.reader.dirty = True -class enable_bracketed_paste(Command): - def do(self) -> None: - self.reader.paste_mode = True - self.reader.in_bracketed_paste = True - -class disable_bracketed_paste(Command): - def do(self) -> None: - self.reader.paste_mode = False - self.reader.in_bracketed_paste = False - self.reader.dirty = True +class perform_bracketed_paste(Command): + def do(self) -> None: + done = "\x1b[201~" + data = "" + start = time.time() + while done not in data: + ev = self.reader.console.getpending() + data += ev.data + trace( + "bracketed pasting of {l} chars done in {s:.2f}s", + l=len(data), + s=time.time() - start, + ) + self.reader.insert(data.replace(done, "")) + self.reader.last_refresh_cache.invalidated = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 9a005281dab..9d2d43be514 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -91,7 +91,7 @@ def build_menu( # D E F B E # G C F # - # "fill" the table with empty words, so we always have the same amout + # "fill" the table with empty words, so we always have the same amount # of rows for each column missing = cols*rows - len(wordlist) wordlist = wordlist + ['']*missing @@ -293,3 +293,7 @@ def get_stem(self) -> str: def get_completions(self, stem: str) -> list[str]: return [] + + def get_line(self) -> str: + """Return the current line until the cursor position.""" + return ''.join(self.buffer[:self.pos]) diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 0d78890b4f4..8956fb1242e 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,11 +19,12 @@ from __future__ import annotations -import _colorize # type: ignore[import-not-found] +import _colorize from abc import ABC, abstractmethod import ast import code +import linecache from dataclasses import dataclass, field import os.path import sys @@ -152,6 +153,8 @@ def repaint(self) -> None: ... class InteractiveColoredConsole(code.InteractiveConsole): + STATEMENT_FAILED = object() + def __init__( self, locals: dict[str, object] | None = None, @@ -159,7 +162,7 @@ def __init__( *, local_exit: bool = False, ) -> None: - super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + super().__init__(locals=locals, filename=filename, local_exit=local_exit) self.can_colorize = _colorize.can_colorize() def showsyntaxerror(self, filename=None, **kwargs): @@ -173,6 +176,16 @@ def _excepthook(self, typ, value, tb): limit=traceback.BUILTIN_EXCEPTION_LIMIT) self.write(''.join(lines)) + def runcode(self, code): + try: + exec(code, self.locals) + except SystemExit: + raise + except BaseException: + self.showtraceback() + return self.STATEMENT_FAILED + return None + def runsource(self, source, filename="", symbol="single"): try: tree = self.compile.compiler( @@ -193,6 +206,7 @@ def runsource(self, source, filename="", symbol="single"): item = wrapper([stmt]) try: code = self.compile.compiler(item, filename, the_symbol) + linecache._register_code(code, source, filename) except SyntaxError as e: if e.args[0] == "'await' outside function": python = os.path.basename(sys.executable) @@ -209,5 +223,7 @@ def runsource(self, source, filename="", symbol="single"): if code is None: return True - self.runcode(code) + result = self.runcode(code) + if result is self.STATEMENT_FAILED: + break return False diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py deleted file mode 100644 index 3a624d9f683..00000000000 --- a/Lib/_pyrepl/curses.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2000-2010 Michael Hudson-Doyle -# Armin Rigo -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -try: - import _curses -except ImportError: - try: - import curses as _curses # type: ignore[no-redef] - except ImportError: - from . import _minimal_curses as _curses # type: ignore[no-redef] - -setupterm = _curses.setupterm -tigetstr = _curses.tigetstr -tparm = _curses.tparm -error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 0468b9a2670..8d5bd183f21 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -20,19 +20,25 @@ import termios +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import cast +else: + cast = lambda typ, val: val + + class TermState: - def __init__(self, tuples): - ( - self.iflag, - self.oflag, - self.cflag, - self.lflag, - self.ispeed, - self.ospeed, - self.cc, - ) = tuples + def __init__(self, attrs: list[int | list[bytes]]) -> None: + self.iflag = cast(int, attrs[0]) + self.oflag = cast(int, attrs[1]) + self.cflag = cast(int, attrs[2]) + self.lflag = cast(int, attrs[3]) + self.ispeed = cast(int, attrs[4]) + self.ospeed = cast(int, attrs[5]) + self.cc = cast(list[bytes], attrs[6]) - def as_list(self): + def as_list(self) -> list[int | list[bytes]]: return [ self.iflag, self.oflag, @@ -45,32 +51,32 @@ def as_list(self): self.cc[:], ] - def copy(self): + def copy(self) -> "TermState": return self.__class__(self.as_list()) -def tcgetattr(fd): +def tcgetattr(fd: int) -> TermState: return TermState(termios.tcgetattr(fd)) -def tcsetattr(fd, when, attrs): +def tcsetattr(fd: int, when: int, attrs: TermState) -> None: termios.tcsetattr(fd, when, attrs.as_list()) class Term(TermState): TS__init__ = TermState.__init__ - def __init__(self, fd=0): + def __init__(self, fd: int = 0) -> None: self.TS__init__(termios.tcgetattr(fd)) self.fd = fd - self.stack = [] + self.stack: list[list[int | list[bytes]]] = [] - def save(self): + def save(self) -> None: self.stack.append(self.as_list()) - def set(self, when=termios.TCSANOW): + def set(self, when: int = termios.TCSANOW) -> None: termios.tcsetattr(self.fd, when, self.as_list()) - def restore(self): + def restore(self) -> None: self.TS__init__(self.stack.pop()) self.set() diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py index 2fb03d19523..d11df4b5164 100644 --- a/Lib/_pyrepl/keymap.py +++ b/Lib/_pyrepl/keymap.py @@ -30,20 +30,20 @@ pyrepl uses its own keyspec format that is meant to be a strict superset of readline's KEYSEQ format. This means that if a spec is found that readline accepts that this doesn't, it should be logged as a bug. Note that this means -we're using the `\\C-o' style of readline's keyspec, not the `Control-o' sort. +we're using the '\\C-o' style of readline's keyspec, not the 'Control-o' sort. The extension to readline is that the sequence \\ denotes the sequence of characters produced by hitting KEY. Examples: -`a' - what you get when you hit the `a' key -`\\EOA' - Escape - O - A (up, on my terminal) -`\\' - the up arrow key -`\\' - ditto (keynames are case-insensitive) -`\\C-o', `\\c-o' - control-o -`\\M-.' - meta-period -`\\E.' - ditto (that's how meta works for pyrepl) -`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' +'a' - what you get when you hit the 'a' key +'\\EOA' - Escape - O - A (up, on my terminal) +'\\' - the up arrow key +'\\' - ditto (keynames are case-insensitive) +'\\C-o', '\\c-o' - control-o +'\\M-.' - meta-period +'\\E.' - ditto (that's how meta works for pyrepl) +'\\', '\\', '\\t', '\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' - all of these are the tab character. """ diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py index a6f824dcc4a..447eb1e551e 100644 --- a/Lib/_pyrepl/main.py +++ b/Lib/_pyrepl/main.py @@ -1,6 +1,7 @@ import errno import os import sys +import types CAN_USE_PYREPL: bool @@ -29,12 +30,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): print(FAIL_REASON, file=sys.stderr) return sys._baserepl() - if mainmodule: - namespace = mainmodule.__dict__ - else: - import __main__ - namespace = __main__.__dict__ - namespace.pop("__pyrepl_interactive_console", None) + if not mainmodule: + mainmodule = types.ModuleType("__main__") + + namespace = mainmodule.__dict__ # sys._baserepl() above does this internally, we do it here startup_path = os.getenv("PYTHONSTARTUP") diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini index 395f5945ab7..9375a55b53c 100644 --- a/Lib/_pyrepl/mypy.ini +++ b/Lib/_pyrepl/mypy.ini @@ -4,8 +4,9 @@ [mypy] files = Lib/_pyrepl +mypy_path = $MYPY_CONFIG_FILE_DIR/../../Misc/mypy explicit_package_bases = True -python_version = 3.12 +python_version = 3.13 platform = linux pretty = True diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index dc26bfd3a34..9ab92f64d1e 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -22,15 +22,13 @@ from __future__ import annotations import sys +import _colorize from contextlib import contextmanager from dataclasses import dataclass, field, fields -import unicodedata -from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] - from . import commands, console, input -from .utils import ANSI_ESCAPE_SEQUENCE, wlen, str_width +from .utils import wlen, unbracket, disp_str, gen_colors, THEME from .trace import trace @@ -39,38 +37,7 @@ from .types import Callback, SimpleContextManager, KeySpec, CommandName -def disp_str(buffer: str) -> tuple[str, list[int]]: - """disp_str(buffer:string) -> (string, [int]) - - Return the string that should be the printed representation of - |buffer| and a list detailing where the characters of |buffer| - get used up. E.g.: - - >>> disp_str(chr(3)) - ('^C', [1, 0]) - - """ - b: list[int] = [] - s: list[str] = [] - for c in buffer: - if c == '\x1a': - s.append(c) - b.append(2) - elif ord(c) < 128: - s.append(c) - b.append(1) - elif unicodedata.category(c).startswith("C"): - c = r"\u%04x" % ord(c) - s.append(c) - b.extend([0] * (len(c) - 1)) - else: - s.append(c) - b.append(str_width(c)) - return "".join(s), b - - -# syntax classes: - +# syntax classes SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) @@ -136,8 +103,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\M-9", "digit-arg"), (r"\M-\n", "accept"), ("\\\\", "self-insert"), - (r"\x1b[200~", "enable_bracketed_paste"), - (r"\x1b[201~", "disable_bracketed_paste"), + (r"\x1b[200~", "perform-bracketed-paste"), (r"\x03", "ctrl-c"), ] + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] @@ -175,20 +141,21 @@ class Reader: Instance variables of note include: * buffer: - A *list* (*not* a string at the moment :-) containing all the - characters that have been entered. + A per-character list containing all the characters that have been + entered. Does not include color information. * console: Hopefully encapsulates the OS dependent stuff. * pos: - A 0-based index into `buffer' for where the insertion point + A 0-based index into 'buffer' for where the insertion point is. * screeninfo: - Ahem. This list contains some info needed to move the - insertion point around reasonably efficiently. + A list of screen position tuples. Each list element is a tuple + representing information on visible line length for a given line. + Allows for efficient skipping of color escape sequences. * cxy, lxy: the position of the insertion point in screen ... * syntax_table: - Dictionary mapping characters to `syntax class'; read the + Dictionary mapping characters to 'syntax class'; read the emacs docs to see what this means :-) * commands: Dictionary mapping command names to command classes. @@ -234,7 +201,6 @@ class Reader: dirty: bool = False finished: bool = False paste_mode: bool = False - in_bracketed_paste: bool = False commands: dict[str, type[Command]] = field(default_factory=make_default_commands) last_command: type[Command] | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) @@ -252,7 +218,6 @@ class Reader: ## cached metadata to speed up screen refreshes @dataclass class RefreshCache: - in_bracketed_paste: bool = False screen: list[str] = field(default_factory=list) screeninfo: list[tuple[int, list[int]]] = field(init=False) line_end_offsets: list[int] = field(default_factory=list) @@ -266,7 +231,6 @@ def update_cache(self, screen: list[str], screeninfo: list[tuple[int, list[int]]], ) -> None: - self.in_bracketed_paste = reader.in_bracketed_paste self.screen = screen.copy() self.screeninfo = screeninfo.copy() self.pos = reader.pos @@ -279,8 +243,7 @@ def valid(self, reader: Reader) -> bool: return False dimensions = reader.console.width, reader.console.height dimensions_changed = dimensions != self.dimensions - paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste - return not (dimensions_changed or paste_changed) + return not dimensions_changed def get_cached_location(self, reader: Reader) -> tuple[int, int]: if self.invalidated: @@ -310,7 +273,7 @@ def __post_init__(self) -> None: self.screeninfo = [(0, [])] self.cxy = self.pos2xy() self.lxy = (self.pos, 0) - self.can_colorize = can_colorize() + self.can_colorize = _colorize.can_colorize() self.last_refresh_cache.screeninfo = self.screeninfo self.last_refresh_cache.pos = self.pos @@ -348,13 +311,17 @@ def calc_screen(self) -> list[str]: prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + if self.can_colorize: + colors = list(gen_colors(self.get_unicode())) + else: + colors = None + trace("colors = {colors}", colors=colors) lines = "".join(self.buffer[offset:]).split("\n") - cursor_found = False lines_beyond_cursor = 0 for ln, line in enumerate(lines, num_common_lines): - ll = len(line) - if 0 <= pos <= ll: + line_len = len(line) + if 0 <= pos <= line_len: self.lxy = pos, ln cursor_found = True elif cursor_found: @@ -368,34 +335,33 @@ def calc_screen(self) -> list[str]: prompt_from_cache = False prompt = "" else: - prompt = self.get_prompt(ln, ll >= pos >= 0) + prompt = self.get_prompt(ln, line_len >= pos >= 0) while "\n" in prompt: pre_prompt, _, prompt = prompt.partition("\n") last_refresh_line_end_offsets.append(offset) screen.append(pre_prompt) screeninfo.append((0, [])) - pos -= ll + 1 - prompt, lp = self.process_prompt(prompt) - l, l2 = disp_str(line) - wrapcount = (wlen(l) + lp) // self.console.width - if wrapcount == 0: - offset += ll + 1 # Takes all of the line plus the newline + pos -= line_len + 1 + prompt, prompt_len = self.process_prompt(prompt) + chars, char_widths = disp_str(line, colors, offset) + wrapcount = (sum(char_widths) + prompt_len) // self.console.width + if wrapcount == 0 or not char_widths: + offset += line_len + 1 # Takes all of the line plus the newline last_refresh_line_end_offsets.append(offset) - screen.append(prompt + l) - screeninfo.append((lp, l2)) + screen.append(prompt + "".join(chars)) + screeninfo.append((prompt_len, char_widths)) else: - i = 0 - while l: - prelen = lp if i == 0 else 0 + pre = prompt + prelen = prompt_len + for wrap in range(wrapcount + 1): index_to_wrap_before = 0 column = 0 - for character_width in l2: - if column + character_width >= self.console.width - prelen: + for char_width in char_widths: + if column + char_width + prelen >= self.console.width: break index_to_wrap_before += 1 - column += character_width - pre = prompt if i == 0 else "" - if len(l) > index_to_wrap_before: + column += char_width + if len(chars) > index_to_wrap_before: offset += index_to_wrap_before post = "\\" after = [1] @@ -404,11 +370,14 @@ def calc_screen(self) -> list[str]: post = "" after = [] last_refresh_line_end_offsets.append(offset) - screen.append(pre + l[:index_to_wrap_before] + post) - screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) - l = l[index_to_wrap_before:] - l2 = l2[index_to_wrap_before:] - i += 1 + render = pre + "".join(chars[:index_to_wrap_before]) + post + render_widths = char_widths[:index_to_wrap_before] + after + screen.append(render) + screeninfo.append((prelen, render_widths)) + chars = chars[index_to_wrap_before:] + char_widths = char_widths[index_to_wrap_before:] + pre = "" + prelen = 0 self.screeninfo = screeninfo self.cxy = self.pos2xy() if self.msg: @@ -421,42 +390,15 @@ def calc_screen(self) -> list[str]: @staticmethod def process_prompt(prompt: str) -> tuple[str, int]: - """Process the prompt. - - This means calculate the length of the prompt. The character \x01 - and \x02 are used to bracket ANSI control sequences and need to be - excluded from the length calculation. So also a copy of the prompt - is returned with these control characters removed.""" + r"""Return a tuple with the prompt string and its visible length. - # The logic below also ignores the length of common escape - # sequences if they were not explicitly within \x01...\x02. - # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) - - # wlen from utils already excludes ANSI_ESCAPE_SEQUENCE chars, - # which breaks the logic below so we redefine it here. - def wlen(s: str) -> int: - return sum(str_width(i) for i in s) - - out_prompt = "" - l = wlen(prompt) - pos = 0 - while True: - s = prompt.find("\x01", pos) - if s == -1: - break - e = prompt.find("\x02", s) - if e == -1: - break - # Found start and end brackets, subtract from string length - l = l - (e - s + 1) - keep = prompt[pos:s] - l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) - out_prompt += keep + prompt[s + 1 : e] - pos = e + 1 - keep = prompt[pos:] - l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) - out_prompt += keep - return out_prompt, l + The prompt string has the zero-width brackets recognized by shells + (\x01 and \x02) removed. The length ignores anything between those + brackets as well as any ANSI escape sequences. + """ + out_prompt = unbracket(prompt, including_content=False) + visible_prompt = unbracket(prompt, including_content=True) + return out_prompt, wlen(visible_prompt) def bow(self, p: int | None = None) -> int: """Return the 0-based index of the word break preceding p most @@ -525,7 +467,7 @@ def max_row(self) -> int: def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, - returning `default' if there is None. Defaults to 1. + returning 'default' if there is None. Defaults to 1. """ if self.arg is None: return default @@ -533,10 +475,10 @@ def get_arg(self, default: int = 1) -> int: def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: """Return what should be in the left-hand margin for line - `lineno'.""" + 'lineno'.""" if self.arg is not None and cursor_on_line: prompt = f"(arg: {self.arg}) " - elif self.paste_mode and not self.in_bracketed_paste: + elif self.paste_mode: prompt = "(paste) " elif "\n" in self.buffer: if lineno == 0: @@ -549,7 +491,8 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: prompt = self.ps1 if self.can_colorize: - prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + t = THEME() + prompt = f"{t.prompt}{prompt}{t.reset}" return prompt def push_input_trans(self, itrans: input.KeymapTranslator) -> None: @@ -564,9 +507,9 @@ def setpos_from_xy(self, x: int, y: int) -> None: pos = 0 i = 0 while i < y: - prompt_len, character_widths = self.screeninfo[i] - offset = len(character_widths) - character_widths.count(0) - in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + prompt_len, char_widths = self.screeninfo[i] + offset = len(char_widths) + in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width if in_wrapped_line: pos += offset - 1 # -1 cause backslash is not in buffer else: @@ -577,6 +520,7 @@ def setpos_from_xy(self, x: int, y: int) -> None: cur_x = self.screeninfo[i][0] while cur_x < x: if self.screeninfo[i][1][j] == 0: + j += 1 # prevent potential future infinite loop continue cur_x += self.screeninfo[i][1][j] j += 1 @@ -586,29 +530,33 @@ def setpos_from_xy(self, x: int, y: int) -> None: def pos2xy(self) -> tuple[int, int]: """Return the x, y coordinates of position 'pos'.""" - # this *is* incomprehensible, yes. - p, y = 0, 0 - l2: list[int] = [] + + prompt_len, y = 0, 0 + char_widths: list[int] = [] pos = self.pos assert 0 <= pos <= len(self.buffer) + + # optimize for the common case: typing at the end of the buffer if pos == len(self.buffer) and len(self.screeninfo) > 0: y = len(self.screeninfo) - 1 - p, l2 = self.screeninfo[y] - return p + sum(l2) + l2.count(0), y + prompt_len, char_widths = self.screeninfo[y] + return prompt_len + sum(char_widths), y + + for prompt_len, char_widths in self.screeninfo: + offset = len(char_widths) + in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width + if in_wrapped_line: + offset -= 1 # need to remove line-wrapping backslash - for p, l2 in self.screeninfo: - l = len(l2) - l2.count(0) - in_wrapped_line = p + sum(l2) >= self.console.width - offset = l - 1 if in_wrapped_line else l # need to remove backslash if offset >= pos: break - if p + sum(l2) >= self.console.width: - pos -= l - 1 # -1 cause backslash is not in buffer - else: - pos -= l + 1 # +1 cause newline is in buffer + if not in_wrapped_line: + offset += 1 # there's a newline in buffer + + pos -= offset y += 1 - return p + sum(l2[:pos]), y + return prompt_len + sum(char_widths[:pos]), y def insert(self, text: str | list[str]) -> None: """Insert 'text' at the insertion point.""" @@ -619,6 +567,7 @@ def insert(self, text: str | list[str]) -> None: def update_cursor(self) -> None: """Move the cursor to reflect changes in self.pos""" self.cxy = self.pos2xy() + trace("update_cursor({pos}) = {cxy}", pos=self.pos, cxy=self.cxy) self.console.move_cursor(*self.cxy) def after_command(self, cmd: Command) -> None: @@ -670,6 +619,16 @@ def suspend(self) -> SimpleContextManager: setattr(self, arg, prev_state[arg]) self.prepare() + @contextmanager + def suspend_colorization(self) -> SimpleContextManager: + try: + old_can_colorize = self.can_colorize + self.can_colorize = False + yield + finally: + self.can_colorize = old_can_colorize + + def finish(self) -> None: """Called when a command signals that we're finished.""" pass @@ -685,9 +644,6 @@ def update_screen(self) -> None: def refresh(self) -> None: """Recalculate and refresh the screen.""" - if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n": - return - # this call sets up self.cxy, so call it first. self.screen = self.calc_screen() self.console.refresh(self.screen, self.cxy) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 888185eb03b..23b8fa6b9c7 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -32,20 +32,22 @@ from dataclasses import dataclass, field import os -from site import gethistoryfile # type: ignore[attr-defined] +from site import gethistoryfile import sys from rlcompleter import Completer as RLCompleter from . import commands, historical_reader from .completing_reader import CompletingReader from .console import Console as ConsoleType +from ._module_completer import ModuleCompleter, make_default_module_completer Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] -try: - from .unix_console import UnixConsole as Console, _error -except ImportError: + +if os.name == "nt": from .windows_console import WindowsConsole as Console, _error +else: + from .unix_console import UnixConsole as Console, _error ENCODING = sys.getdefaultencoding() or "latin1" @@ -89,6 +91,7 @@ # "set_pre_input_hook", "set_startup_hook", "write_history_file", + "append_history_file", # ---- multiline extensions ---- "multiline_input", ] @@ -99,7 +102,7 @@ class ReadlineConfig: readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") - + module_completer: ModuleCompleter = field(default_factory=make_default_module_completer) @dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): @@ -132,6 +135,9 @@ def get_stem(self) -> str: return "".join(b[p + 1 : self.pos]) def get_completions(self, stem: str) -> list[str]: + module_completions = self.get_module_completions() + if module_completions is not None: + return module_completions if len(stem) == 0 and self.more_lines is not None: b = self.buffer p = self.pos @@ -161,6 +167,10 @@ def get_completions(self, stem: str) -> list[str]: result.sort() return result + def get_module_completions(self) -> list[str] | None: + line = self.get_line() + return self.config.module_completer.get_completions(line) + def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: cut = len(self.history) - maxlength @@ -268,10 +278,6 @@ def do(self) -> None: r = self.reader # type: ignore[assignment] r.dirty = True # this is needed to hide the completion menu, if visible - if self.reader.in_bracketed_paste: - r.insert("\n") - return - # if there are already several lines and the cursor # is not on the last one, always insert a new \n. text = r.get_unicode() @@ -446,6 +452,7 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None: del buffer[:] if line: history.append(line) + self.set_history_length(self.get_current_history_length()) def write_history_file(self, filename: str = gethistoryfile()) -> None: maxlength = self.saved_history_length @@ -457,6 +464,19 @@ def write_history_file(self, filename: str = gethistoryfile()) -> None: entry = entry.replace("\n", "\r\n") # multiline history support f.write(entry + "\n") + def append_history_file(self, filename: str = gethistoryfile()) -> None: + reader = self.get_reader() + saved_length = self.get_history_length() + length = self.get_current_history_length() - saved_length + history = reader.get_trimmed_history(length) + f = open(os.path.expanduser(filename), "a", + encoding="utf-8", newline="\n") + with f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + self.set_history_length(saved_length + length) + def clear_history(self) -> None: del self.get_reader().history[:] @@ -526,6 +546,7 @@ def insert_text(self, text: str) -> None: get_current_history_length = _wrapper.get_current_history_length read_history_file = _wrapper.read_history_file write_history_file = _wrapper.write_history_file +append_history_file = _wrapper.append_history_file clear_history = _wrapper.clear_history get_history_item = _wrapper.get_history_item remove_history_item = _wrapper.remove_history_item @@ -587,6 +608,7 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + _wrapper.config.module_completer = ModuleCompleter(namespace) _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 66e66eae7ea..6508f0233b9 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -26,18 +26,14 @@ from __future__ import annotations import _sitebuiltins -import linecache import functools import os import sys import code +import warnings +import errno -from .readline import _get_reader, multiline_input - -TYPE_CHECKING = False - -if TYPE_CHECKING: - from typing import Any +from .readline import _get_reader, multiline_input, append_history_file _error: tuple[type[Exception], ...] | type[Exception] @@ -115,6 +111,10 @@ def run_multiline_interactive_console( more_lines = functools.partial(_more_lines, console) input_n = 0 + _is_x_showrefcount_set = sys._xoptions.get("showrefcount") + _is_pydebug_build = hasattr(sys, "gettotalrefcount") + show_ref_count = _is_x_showrefcount_set and _is_pydebug_build + def maybe_run_command(statement: str) -> bool: statement = statement.strip() if statement in console.locals or statement not in REPL_COMMANDS: @@ -125,12 +125,12 @@ def maybe_run_command(statement: str) -> bool: command = REPL_COMMANDS[statement] if callable(command): # Make sure that history does not change because of commands - with reader.suspend_history(): + with reader.suspend_history(), reader.suspend_colorization(): command() return True return False - while 1: + while True: try: try: sys.stdout.flush() @@ -148,20 +148,34 @@ def maybe_run_command(statement: str) -> bool: continue input_name = f"" - linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] assert not more + try: + append_history_file() + except (FileNotFoundError, PermissionError, OSError) as e: + warnings.warn(f"failed to open the history file for writing: {e}") + input_n += 1 except KeyboardInterrupt: r = _get_reader() + r.cmpltn_reset() if r.input_trans is r.isearch_trans: r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) r.dirty = True r.refresh() - r.in_bracketed_paste = False console.write("\nKeyboardInterrupt\n") console.resetbuffer() except MemoryError: console.write("\nMemoryError\n") console.resetbuffer() + except SystemExit: + raise + except: + console.showtraceback() + console.resetbuffer() + if show_ref_count: + console.write( + f"[{sys.gettotalrefcount()} refs," + f" {sys.getallocatedblocks()} blocks]\n" + ) diff --git a/Lib/_pyrepl/terminfo.py b/Lib/_pyrepl/terminfo.py new file mode 100644 index 00000000000..d02ef69cce0 --- /dev/null +++ b/Lib/_pyrepl/terminfo.py @@ -0,0 +1,488 @@ +"""Pure Python curses-like terminal capability queries.""" + +from dataclasses import dataclass, field +import errno +import os +from pathlib import Path +import re +import struct + + +# Terminfo constants +MAGIC16 = 0o432 # Magic number for 16-bit terminfo format +MAGIC32 = 0o1036 # Magic number for 32-bit terminfo format + +# Special values for absent/cancelled capabilities +ABSENT_BOOLEAN = -1 +ABSENT_NUMERIC = -1 +CANCELLED_NUMERIC = -2 +ABSENT_STRING = None +CANCELLED_STRING = None + + +# Standard string capability names from ncurses Caps file +# This matches the order used by ncurses when compiling terminfo +# fmt: off +_STRING_NAMES: tuple[str, ...] = ( + "cbt", "bel", "cr", "csr", "tbc", "clear", "el", "ed", "hpa", "cmdch", + "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", "ll", + "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold", + "smcup", "smdc", "dim", "smir", "invis", "prot", "rev", "smso", "smul", + "ech", "rmacs", "sgr0", "rmcup", "rmdc", "rmir", "rmso", "rmul", "flash", + "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", "kbs", "ktbc", + "kclr", "kctab", "kdch1", "kdl1", "kcud1", "krmir", "kel", "ked", "kf0", + "kf1", "kf10", "kf2", "kf3", "kf4", "kf5", "kf6", "kf7", "kf8", "kf9", + "khome", "kich1", "kil1", "kcub1", "kll", "knp", "kpp", "kcuf1", "kind", + "kri", "khts", "kcuu1", "rmkx", "smkx", "lf0", "lf1", "lf10", "lf2", "lf3", + "lf4", "lf5", "lf6", "lf7", "lf8", "lf9", "rmm", "smm", "nel", "pad", "dch", + "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey", + "pfloc", "pfx", "mc0", "mc4", "mc5", "rep", "rs1", "rs2", "rs3", "rf", "rc", + "vpa", "sc", "ind", "ri", "sgr", "hts", "wind", "ht", "tsl", "uc", "hu", + "iprog", "ka1", "ka3", "kb2", "kc1", "kc3", "mc5p", "rmp", "acsc", "pln", + "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "enacs", "smln", + "rmln", "kbeg", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "kend", "kent", + "kext", "kfnd", "khlp", "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt", + "kprv", "kprt", "krdo", "kref", "krfr", "krpl", "krst", "kres", "ksav", + "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "kDC", "kDL", + "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "kIC", "kLFT", + "kMSG", "kMOV", "kNXT", "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT", + "kRES", "kSAV", "kSPD", "kUND", "rfi", "kf11", "kf12", "kf13", "kf14", + "kf15", "kf16", "kf17", "kf18", "kf19", "kf20", "kf21", "kf22", "kf23", + "kf24", "kf25", "kf26", "kf27", "kf28", "kf29", "kf30", "kf31", "kf32", + "kf33", "kf34", "kf35", "kf36", "kf37", "kf38", "kf39", "kf40", "kf41", + "kf42", "kf43", "kf44", "kf45", "kf46", "kf47", "kf48", "kf49", "kf50", + "kf51", "kf52", "kf53", "kf54", "kf55", "kf56", "kf57", "kf58", "kf59", + "kf60", "kf61", "kf62", "kf63", "el1", "mgc", "smgl", "smgr", "fln", "sclk", + "dclk", "rmclk", "cwin", "wingo", "hup","dial", "qdial", "tone", "pulse", + "hook", "pause", "wait", "u0", "u1", "u2", "u3", "u4", "u5", "u6", "u7", + "u8", "u9", "op", "oc", "initc", "initp", "scp", "setf", "setb", "cpi", + "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm", + "snlq", "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm", + "rmicm", "rshm", "rsubm", "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1", + "mvpa", "mcuu1", "porder", "mcud", "mcub", "mcuf", "mcuu", "scs", "smgb", + "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", "rbim", "rcsd", + "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp", + "getm", "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds", + "s3ds", "smglr", "smgtb", "birep", "binel", "bicr", "colornm", "defbi", + "endbi", "setcolor", "slines", "dispc", "smpch", "rmpch", "smsc", "rmsc", + "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", "ethlm", + "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbc", "OTko", "OTma", + "OTG2", "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH", + "OTGV", "OTGC","meml", "memu", "box1" +) +# fmt: on + + +def _get_terminfo_dirs() -> list[Path]: + """Get list of directories to search for terminfo files. + + Based on ncurses behavior in: + - ncurses/tinfo/db_iterator.c:_nc_next_db() + - ncurses/tinfo/read_entry.c:_nc_read_entry() + """ + dirs = [] + + terminfo = os.environ.get("TERMINFO") + if terminfo: + dirs.append(terminfo) + + try: + home = Path.home() + dirs.append(str(home / ".terminfo")) + except RuntimeError: + pass + + # Check TERMINFO_DIRS + terminfo_dirs = os.environ.get("TERMINFO_DIRS", "") + if terminfo_dirs: + for d in terminfo_dirs.split(":"): + if d: + dirs.append(d) + + dirs.extend( + [ + "/etc/terminfo", + "/lib/terminfo", + "/usr/lib/terminfo", + "/usr/share/terminfo", + "/usr/share/lib/terminfo", + "/usr/share/misc/terminfo", + "/usr/local/lib/terminfo", + "/usr/local/share/terminfo", + ] + ) + + return [Path(d) for d in dirs if Path(d).is_dir()] + + +def _validate_terminal_name_or_raise(terminal_name: str) -> None: + if not isinstance(terminal_name, str): + raise TypeError("`terminal_name` must be a string") + + if not terminal_name: + raise ValueError("`terminal_name` cannot be empty") + + if "\x00" in terminal_name: + raise ValueError("NUL character found in `terminal_name`") + + t = Path(terminal_name) + if len(t.parts) > 1: + raise ValueError("`terminal_name` cannot contain path separators") + + +def _read_terminfo_file(terminal_name: str) -> bytes: + """Find and read terminfo file for given terminal name. + + Terminfo files are stored in directories using the first character + of the terminal name as a subdirectory. + """ + _validate_terminal_name_or_raise(terminal_name) + first_char = terminal_name[0].lower() + filename = terminal_name + + for directory in _get_terminfo_dirs(): + path = directory / first_char / filename + if path.is_file(): + return path.read_bytes() + + # Try with hex encoding of first char (for special chars) + hex_dir = "%02x" % ord(first_char) + path = directory / hex_dir / filename + if path.is_file(): + return path.read_bytes() + + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) + + +# Hard-coded terminal capabilities for common terminals +# This is a minimal subset needed by PyREPL +_TERMINAL_CAPABILITIES = { + # ANSI/xterm-compatible terminals + "ansi": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column + "cud1": b"\n", # Move cursor down 1 row + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[2J", # Clear screen and home cursor + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l", # Make cursor invisible + "cnorm": b"\x1b[?12l\x1b[?25h", # Make cursor normal (visible) + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Padding (not used in modern terminals) + "pad": b"", + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1bOB", # Down arrow + "kend": b"\x1bOF", # End key + "kent": b"\x1bOM", # Enter key + "khome": b"\x1bOH", # Home key + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1bOD", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1bOC", # Right arrow + "kcuu1": b"\x1bOA", # Up arrow + # Function keys F1-F20 + "kf1": b"\x1bOP", + "kf2": b"\x1bOQ", + "kf3": b"\x1bOR", + "kf4": b"\x1bOS", + "kf5": b"\x1b[15~", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[1;2P", + "kf14": b"\x1b[1;2Q", + "kf15": b"\x1b[1;2R", + "kf16": b"\x1b[1;2S", + "kf17": b"\x1b[15;2~", + "kf18": b"\x1b[17;2~", + "kf19": b"\x1b[18;2~", + "kf20": b"\x1b[19;2~", + }, + # Dumb terminal - minimal capabilities + "dumb": { + "bel": b"\x07", # Bell + "cud1": b"\n", # Move down 1 row (newline) + "ind": b"\n", # Scroll up one line (newline) + }, + # Linux console + "linux": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column (backspace) + "cud1": b"\n", # Move cursor down 1 row (newline) + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[J", # Clear screen and home cursor (different from ansi!) + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"\x1b[@", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l\x1b[?1c", # Make cursor invisible + "cnorm": b"\x1b[?25h\x1b[?0c", # Make cursor normal + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1b[B", # Down arrow + "kend": b"\x1b[4~", # End key (different from ansi!) + "khome": b"\x1b[1~", # Home key (different from ansi!) + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1b[D", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1b[C", # Right arrow + "kcuu1": b"\x1b[A", # Up arrow + # Function keys + "kf1": b"\x1b[[A", + "kf2": b"\x1b[[B", + "kf3": b"\x1b[[C", + "kf4": b"\x1b[[D", + "kf5": b"\x1b[[E", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[25~", + "kf14": b"\x1b[26~", + "kf15": b"\x1b[28~", + "kf16": b"\x1b[29~", + "kf17": b"\x1b[31~", + "kf18": b"\x1b[32~", + "kf19": b"\x1b[33~", + "kf20": b"\x1b[34~", + }, +} + +# Map common TERM values to capability sets +_TERM_ALIASES = { + "xterm": "ansi", + "xterm-color": "ansi", + "xterm-256color": "ansi", + "screen": "ansi", + "screen-256color": "ansi", + "tmux": "ansi", + "tmux-256color": "ansi", + "vt100": "ansi", + "vt220": "ansi", + "rxvt": "ansi", + "rxvt-unicode": "ansi", + "rxvt-unicode-256color": "ansi", + "unknown": "dumb", +} + + +@dataclass +class TermInfo: + terminal_name: str | bytes | None + fallback: bool = True + + _capabilities: dict[str, bytes] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Initialize terminal capabilities for the given terminal type. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_setup.c:setupterm() and _nc_setupterm() + - ncurses/tinfo/lib_setup.c:TINFO_SETUP_TERM() + + This version first attempts to read terminfo database files like ncurses, + then, if `fallback` is True, falls back to hardcoded capabilities for + common terminal types. + """ + # If termstr is None or empty, try to get from environment + if not self.terminal_name: + self.terminal_name = os.environ.get("TERM") or "ANSI" + + if isinstance(self.terminal_name, bytes): + self.terminal_name = self.terminal_name.decode("ascii") + + try: + self._parse_terminfo_file(self.terminal_name) + except (OSError, ValueError): + if not self.fallback: + raise + + term_type = _TERM_ALIASES.get( + self.terminal_name, self.terminal_name + ) + if term_type not in _TERMINAL_CAPABILITIES: + term_type = "dumb" + self._capabilities = _TERMINAL_CAPABILITIES[term_type].copy() + + def _parse_terminfo_file(self, terminal_name: str) -> None: + """Parse a terminfo file. + + Populate the _capabilities dict for easy retrieval + + Based on ncurses implementation in: + - ncurses/tinfo/read_entry.c:_nc_read_termtype() + - ncurses/tinfo/read_entry.c:_nc_read_file_entry() + - ncurses/tinfo/lib_ti.c:tigetstr() + """ + data = _read_terminfo_file(terminal_name) + too_short = f"TermInfo file for {terminal_name!r} too short" + offset = 12 + if len(data) < offset: + raise ValueError(too_short) + + magic, name_size, bool_count, num_count, str_count, str_size = ( + struct.unpack(" len(data): + raise ValueError(too_short) + + # Read string offsets + end_offset = offset + 2 * str_count + if offset > len(data): + raise ValueError(too_short) + string_offset_data = data[offset:end_offset] + string_offsets = [ + off for [off] in struct.iter_unpack(" len(data): + raise ValueError(too_short) + string_table = data[offset : offset + str_size] + + # Extract strings from string table + capabilities = {} + for cap, off in zip(_STRING_NAMES, string_offsets): + if off < 0: + # CANCELLED_STRING; we do not store those + continue + elif off < len(string_table): + # Find null terminator + end = string_table.find(0, off) + if end >= 0: + capabilities[cap] = string_table[off:end] + # in other cases this is ABSENT_STRING; we don't store those. + + # Note: we don't support extended capabilities since PyREPL doesn't + # need them. + + self._capabilities = capabilities + + def get(self, cap: str) -> bytes | None: + """Get terminal capability string by name. + """ + if not isinstance(cap, str): + raise TypeError(f"`cap` must be a string, not {type(cap)}") + + return self._capabilities.get(cap) + + +def tparm(cap_bytes: bytes, *params: int) -> bytes: + """Parameterize a terminal capability string. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_tparm.c:tparm() + - ncurses/tinfo/lib_tparm.c:tparam_internal() + + The ncurses version implements a full stack-based interpreter for + terminfo parameter strings. This pure Python version implements only + the subset of parameter substitution operations needed by PyREPL: + - %i (increment parameters for 1-based indexing) + - %p[1-9]%d (parameter substitution) + - %p[1-9]%{n}%+%d (parameter plus constant) + """ + if not isinstance(cap_bytes, bytes): + raise TypeError(f"`cap` must be bytes, not {type(cap_bytes)}") + + result = cap_bytes + + # %i - increment parameters (1-based instead of 0-based) + increment = b"%i" in result + if increment: + result = result.replace(b"%i", b"") + + # Replace %p1%d, %p2%d, etc. with actual parameter values + for i in range(len(params)): + pattern = b"%%p%d%%d" % (i + 1) + if pattern in result: + value = params[i] + if increment: + value += 1 + result = result.replace(pattern, str(value).encode("ascii")) + + # Handle %p1%{1}%+%d (parameter plus constant) + # Used in some cursor positioning sequences + pattern_re = re.compile(rb"%p(\d)%\{(\d+)\}%\+%d") + matches = list(pattern_re.finditer(result)) + for match in reversed(matches): # reversed to maintain positions + param_idx = int(match.group(1)) + constant = int(match.group(2)) + value = params[param_idx] + constant + result = ( + result[: match.start()] + + str(value).encode("ascii") + + result[match.end() :] + ) + + return result diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index a8eb2433cd3..943ee12f964 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys # types if False: @@ -12,10 +13,22 @@ trace_file = open(trace_filename, "a") -def trace(line: str, *k: object, **kw: object) -> None: - if trace_file is None: - return - if k or kw: - line = line.format(*k, **kw) - trace_file.write(line + "\n") - trace_file.flush() + +if sys.platform == "emscripten": + from posix import _emscripten_log + + def trace(line: str, *k: object, **kw: object) -> None: + if "PYREPL_TRACE" not in os.environ: + return + if k or kw: + line = line.format(*k, **kw) + _emscripten_log(line) + +else: + def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index f9d48b828c7..c5b7ebc1a40 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -1,8 +1,10 @@ from collections.abc import Callable, Iterator -Callback = Callable[[], object] -SimpleContextManager = Iterator[None] -KeySpec = str # like r"\C-c" -CommandName = str # like "interrupt" -EventTuple = tuple[CommandName, str] -Completer = Callable[[str, int], str | None] +type Callback = Callable[[], object] +type SimpleContextManager = Iterator[None] +type KeySpec = str # like r"\C-c" +type CommandName = str # like "interrupt" +type EventTuple = tuple[CommandName, str] +type Completer = Callable[[str, int], str | None] +type CharBuffer = list[str] +type CharWidths = list[int] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index e69c96b1159..937b5df6ff7 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -29,31 +29,41 @@ import struct import termios import time +import types import platform from fcntl import ioctl -from . import curses +from . import terminfo from .console import Console, Event -from .fancy_termios import tcgetattr, tcsetattr +from .fancy_termios import tcgetattr, tcsetattr, TermState from .trace import trace from .unix_eventqueue import EventQueue from .utils import wlen +# declare posix optional to allow None assignment on other platforms +posix: types.ModuleType | None +try: + import posix +except ImportError: + posix = None TYPE_CHECKING = False # types if TYPE_CHECKING: - from typing import IO, Literal, overload + from typing import AbstractSet, IO, Literal, overload, cast else: overload = lambda func: None + cast = lambda typ, val: val class InvalidTerminal(RuntimeError): - pass + def __init__(self, message: str) -> None: + super().__init__(errno.EIO, message) -_error = (termios.error, curses.error, InvalidTerminal) +_error = (termios.error, InvalidTerminal) +_error_codes_to_ignore = frozenset([errno.EIO, errno.ENXIO, errno.EPERM]) SIGWINCH_EVENT = "repaint" @@ -110,7 +120,7 @@ def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: try: poll: type[select.poll] = select.poll except AttributeError: - # this is exactly the minumum necessary to support what we + # this is exactly the minimum necessary to support what we # do with poll objects class MinimalPoll: def __init__(self): @@ -118,12 +128,13 @@ def __init__(self): def register(self, fd, flag): self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* def poll(self, timeout: float | None = None) -> list[int]: if timeout is None: r, w, e = select.select([self.fd], [], []) else: - r, w, e = select.select([self.fd], [], [], timeout/1000) + r, w, e = select.select([self.fd], [], [], timeout / 1000) return r poll = MinimalPoll # type: ignore[assignment] @@ -150,19 +161,28 @@ def __init__( self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) - self.input_buffer = b"" - self.input_buffer_pos = 0 - curses.setupterm(term or None, self.output_fd) + self.terminfo = terminfo.TermInfo(term or None) self.term = term + self.is_apple_terminal = ( + platform.system() == "Darwin" + and os.getenv("TERM_PROGRAM") == "Apple_Terminal" + ) + + try: + self.__input_fd_set(tcgetattr(self.input_fd), ignore=frozenset()) + except _error as e: + raise RuntimeError(f"termios failure ({e.args[1]})") @overload - def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + def _my_getstr( + cap: str, optional: Literal[False] = False + ) -> bytes: ... @overload def _my_getstr(cap: str, optional: bool) -> bytes | None: ... def _my_getstr(cap: str, optional: bool = False) -> bytes | None: - r = curses.tigetstr(cap) + r = self.terminfo.get(cap) if not optional and r is None: raise InvalidTerminal( f"terminal doesn't have the required {cap} capability" @@ -196,26 +216,19 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding) - self.cursor_visible = 1 - - def more_in_buffer(self) -> bool: - return bool( - self.input_buffer - and self.input_buffer_pos < len(self.input_buffer) + self.event_queue = EventQueue( + self.input_fd, self.encoding, self.terminfo ) + self.cursor_visible = 1 - def __read(self, n: int) -> bytes: - if not self.more_in_buffer(): - self.input_buffer = os.read(self.input_fd, 10000) + signal.signal(signal.SIGCONT, self._sigcont_handler) - ret = self.input_buffer[self.input_buffer_pos : self.input_buffer_pos + n] - self.input_buffer_pos += len(ret) - if self.input_buffer_pos >= len(self.input_buffer): - self.input_buffer = b"" - self.input_buffer_pos = 0 - return ret + def _sigcont_handler(self, signum, frame): + self.restore() + self.prepare() + def __read(self, n: int) -> bytes: + return os.read(self.input_fd, n) def change_encoding(self, encoding: str) -> None: """ @@ -238,8 +251,9 @@ def refresh(self, screen, c_xy): if not self.__gone_tall: while len(self.screen) < min(len(screen), self.height): self.__hide_cursor() - self.__move(0, len(self.screen) - 1) - self.__write("\n") + if self.screen: + self.__move(0, len(self.screen) - 1) + self.__write("\n") self.posxy = 0, len(self.screen) self.screen.append("") else: @@ -328,6 +342,8 @@ def prepare(self): """ Prepare the console for input/output operations. """ + self.__buffer = [] + self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) @@ -339,17 +355,15 @@ def prepare(self): raw.lflag |= termios.ISIG raw.cc[termios.VMIN] = 1 raw.cc[termios.VTIME] = 0 - tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + self.__input_fd_set(raw) # In macOS terminal we need to deactivate line wrap via ANSI escape code - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer = [] - self.posxy = 0, 0 self.__gone_tall = 0 self.__move = self.__move_short @@ -371,13 +385,18 @@ def restore(self): self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() - tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + self.__input_fd_set(self.__svtermstate) - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): - signal.signal(signal.SIGWINCH, self.old_sigwinch) + try: + signal.signal(signal.SIGWINCH, self.old_sigwinch) + except ValueError as e: + import threading + if threading.current_thread() is threading.main_thread(): + raise e del self.old_sigwinch def push_char(self, char: int | bytes) -> None: @@ -410,6 +429,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.get() else: continue + elif err.errno == errno.EIO: + raise SystemExit(errno.EIO) else: raise else: @@ -422,7 +443,6 @@ def wait(self, timeout: float | None = None) -> bool: """ return ( not self.event_queue.empty() - or self.more_in_buffer() or bool(self.pollob.poll(timeout)) ) @@ -525,6 +545,7 @@ def getpending(self): e.raw += e.raw amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + trace("getpending({a})", a=amount) raw = self.__read(amount) data = str(raw, self.encoding, "replace") e.data += data @@ -566,11 +587,9 @@ def clear(self): @property def input_hook(self): - try: - import posix - except ImportError: - return None - if posix._is_inputhook_installed(): + # avoid inline imports here so the repl doesn't get flooded + # with import logging from -X importtime=2 + if posix is not None and posix._is_inputhook_installed(): return posix._inputhook def __enable_bracketed_paste(self) -> None: @@ -602,14 +621,14 @@ def __setup_movement(self): if self._dch1: self.dch1 = self._dch1 elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) + self.dch1 = terminfo.tparm(self._dch, 1) else: self.dch1 = None if self._ich1: self.ich1 = self._ich1 elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) + self.ich1 = terminfo.tparm(self._ich, 1) else: self.ich1 = None @@ -634,7 +653,7 @@ def __write_changed_line(self, y, oldline, newline, px_coord): # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape - # sequene + # sequence while ( x_coord < minlen and oldline[x_pos] == newline[x_pos] @@ -706,7 +725,7 @@ def __write(self, text): self.__buffer.append((text, 0)) def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), 1)) def __maybe_write_code(self, fmt, *args): if fmt: @@ -790,9 +809,9 @@ def __tputs(self, fmt, prog=delayprog): will never do anyone any good.""" # using .get() means that things will blow up # only if the bps is actually needed (which I'm - # betting is pretty unlkely) + # betting is pretty unlikely) bps = ratedict.get(self.__svtermstate.ospeed) - while 1: + while True: m = prog.search(fmt) if not m: os.write(self.output_fd, fmt) @@ -808,3 +827,17 @@ def __tputs(self, fmt, prog=delayprog): os.write(self.output_fd, self._pad * nchars) else: time.sleep(float(delay) / 1000.0) + + def __input_fd_set( + self, + state: TermState, + ignore: AbstractSet[int] = _error_codes_to_ignore, + ) -> bool: + try: + tcsetattr(self.input_fd, termios.TCSADRAIN, state) + except termios.error as te: + if te.args[0] not in ignore: + raise + return False + else: + return True diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 70cfade26e2..2a9cca59e74 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -18,12 +18,9 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from collections import deque - -from . import keymap -from .console import Event -from . import curses +from .terminfo import TermInfo from .trace import trace +from .base_eventqueue import BaseEventQueue from termios import tcgetattr, VERASE import os @@ -57,96 +54,24 @@ b'\033Oc': 'ctrl right', } -def get_terminal_keycodes() -> dict[bytes, str]: +def get_terminal_keycodes(ti: TermInfo) -> dict[bytes, str]: """ Generates a dictionary mapping terminal keycodes to human-readable names. """ keycodes = {} for key, terminal_code in TERMINAL_KEYNAMES.items(): - keycode = curses.tigetstr(terminal_code) + keycode = ti.get(terminal_code) trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key keycodes.update(CTRL_ARROW_KEYCODES) return keycodes -class EventQueue: - def __init__(self, fd: int, encoding: str) -> None: - self.keycodes = get_terminal_keycodes() + +class EventQueue(BaseEventQueue): + def __init__(self, fd: int, encoding: str, ti: TermInfo) -> None: + keycodes = get_terminal_keycodes(ti) if os.isatty(fd): backspace = tcgetattr(fd)[6][VERASE] - self.keycodes[backspace] = "backspace" - self.compiled_keymap = keymap.compile_keymap(self.keycodes) - self.keymap = self.compiled_keymap - trace("keymap {k!r}", k=self.keymap) - self.encoding = encoding - self.events: deque[Event] = deque() - self.buf = bytearray() - - def get(self) -> Event | None: - """ - Retrieves the next event from the queue. - """ - if self.events: - return self.events.popleft() - else: - return None - - def empty(self) -> bool: - """ - Checks if the queue is empty. - """ - return not self.events - - def flush_buf(self) -> bytearray: - """ - Flushes the buffer and returns its contents. - """ - old = self.buf - self.buf = bytearray() - return old - - def insert(self, event: Event) -> None: - """ - Inserts an event into the queue. - """ - trace('added event {event}', event=event) - self.events.append(event) - - def push(self, char: int | bytes) -> None: - """ - Processes a character by updating the buffer and handling special key mappings. - """ - ord_char = char if isinstance(char, int) else ord(char) - char = bytes(bytearray((ord_char,))) - self.buf.append(ord_char) - if char in self.keymap: - if self.keymap is self.compiled_keymap: - #sanity check, buffer is empty when a special key comes - assert len(self.buf) == 1 - k = self.keymap[char] - trace('found map {k!r}', k=k) - if isinstance(k, dict): - self.keymap = k - else: - self.insert(Event('key', k, self.flush_buf())) - self.keymap = self.compiled_keymap - - elif self.buf and self.buf[0] == 27: # escape - # escape sequence not recognized by our keymap: propagate it - # outside so that i can be recognized as an M-... key (see also - # the docstring in keymap.py - trace('unrecognized escape sequence, propagating...') - self.keymap = self.compiled_keymap - self.insert(Event('key', '\033', bytearray(b'\033'))) - for _c in self.flush_buf()[1:]: - self.push(_c) - - else: - try: - decoded = bytes(self.buf).decode(self.encoding) - except UnicodeError: - return - else: - self.insert(Event('key', decoded, self.flush_buf())) - self.keymap = self.compiled_keymap + keycodes[backspace] = "backspace" + BaseEventQueue.__init__(self, encoding, keycodes) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 4651717bd7e..06cddef851b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -1,25 +1,391 @@ +from __future__ import annotations +import builtins +import functools +import keyword import re +import token as T +import tokenize import unicodedata -import functools +import _colorize + +from collections import deque +from io import StringIO +from tokenize import TokenInfo as TI +from typing import Iterable, Iterator, Match, NamedTuple, Self + +from .types import CharBuffer, CharWidths +from .trace import trace ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") +ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") +ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) +IDENTIFIERS_AFTER = {"def", "class"} +KEYWORD_CONSTANTS = {"True", "False", "None"} +BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} + + +def THEME(**kwargs): + # Not cached: the user can modify the theme inside the interactive session. + return _colorize.get_theme(**kwargs).syntax + + +class Span(NamedTuple): + """Span indexing that's inclusive on both ends.""" + + start: int + end: int + + @classmethod + def from_re(cls, m: Match[str], group: int | str) -> Self: + re_span = m.span(group) + return cls(re_span[0], re_span[1] - 1) + + @classmethod + def from_token(cls, token: TI, line_len: list[int]) -> Self: + end_offset = -1 + if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE} + and token.string.endswith(("{", "}"))): + # gh-134158: a visible trailing brace comes from a double brace in input + end_offset += 1 + + return cls( + line_len[token.start[0] - 1] + token.start[1], + line_len[token.end[0] - 1] + token.end[1] + end_offset, + ) + + +class ColorSpan(NamedTuple): + span: Span + tag: str @functools.cache def str_width(c: str) -> int: if ord(c) < 128: return 1 + # gh-139246 for zero-width joiner and combining characters + if unicodedata.combining(c): + return 0 + category = unicodedata.category(c) + if category == "Cf" and c != "\u00ad": + return 0 w = unicodedata.east_asian_width(c) - if w in ('N', 'Na', 'H', 'A'): + if w in ("N", "Na", "H", "A"): return 1 return 2 def wlen(s: str) -> int: - if len(s) == 1 and s != '\x1a': + if len(s) == 1 and s != "\x1a": return str_width(s) length = sum(str_width(i) for i in s) # remove lengths of any escape sequences sequence = ANSI_ESCAPE_SEQUENCE.findall(s) - ctrl_z_cnt = s.count('\x1a') + ctrl_z_cnt = s.count("\x1a") return length - sum(len(i) for i in sequence) + ctrl_z_cnt + + +def unbracket(s: str, including_content: bool = False) -> str: + r"""Return `s` with \001 and \002 characters removed. + + If `including_content` is True, content between \001 and \002 is also + stripped. + """ + if including_content: + return ZERO_WIDTH_BRACKET.sub("", s) + return s.translate(ZERO_WIDTH_TRANS) + + +def gen_colors(buffer: str) -> Iterator[ColorSpan]: + """Returns a list of index spans to color using the given color tag. + + The input `buffer` should be a valid start of a Python code block, i.e. + it cannot be a block starting in the middle of a multiline string. + """ + sio = StringIO(buffer) + line_lengths = [0] + [len(line) for line in sio.readlines()] + # make line_lengths cumulative + for i in range(1, len(line_lengths)): + line_lengths[i] += line_lengths[i-1] + + sio.seek(0) + gen = tokenize.generate_tokens(sio.readline) + last_emitted: ColorSpan | None = None + try: + for color in gen_colors_from_token_stream(gen, line_lengths): + yield color + last_emitted = color + except SyntaxError: + return + except tokenize.TokenError as te: + yield from recover_unterminated_string( + te, line_lengths, last_emitted, buffer + ) + + +def recover_unterminated_string( + exc: tokenize.TokenError, + line_lengths: list[int], + last_emitted: ColorSpan | None, + buffer: str, +) -> Iterator[ColorSpan]: + msg, loc = exc.args + if loc is None: + return + + line_no, column = loc + + if msg.startswith( + ( + "unterminated string literal", + "unterminated f-string literal", + "unterminated t-string literal", + "EOF in multi-line string", + "unterminated triple-quoted f-string literal", + "unterminated triple-quoted t-string literal", + ) + ): + start = line_lengths[line_no - 1] + column - 1 + end = line_lengths[-1] - 1 + + # in case FSTRING_START was already emitted + if last_emitted and start <= last_emitted.span.start: + trace("before last emitted = {s}", s=start) + start = last_emitted.span.end + 1 + + span = Span(start, end) + trace("yielding span {a} -> {b}", a=span.start, b=span.end) + yield ColorSpan(span, "string") + else: + trace( + "unhandled token error({buffer}) = {te}", + buffer=repr(buffer), + te=str(exc), + ) + + +def gen_colors_from_token_stream( + token_generator: Iterator[TI], + line_lengths: list[int], +) -> Iterator[ColorSpan]: + token_window = prev_next_window(token_generator) + + is_def_name = False + bracket_level = 0 + for prev_token, token, next_token in token_window: + assert token is not None + if token.start == token.end: + continue + + match token.type: + case ( + T.STRING + | T.FSTRING_START | T.FSTRING_MIDDLE | T.FSTRING_END + | T.TSTRING_START | T.TSTRING_MIDDLE | T.TSTRING_END + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "string") + case T.COMMENT: + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "comment") + case T.NUMBER: + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "number") + case T.OP: + if token.string in "([{": + bracket_level += 1 + elif token.string in ")]}": + bracket_level -= 1 + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "op") + case T.NAME: + if is_def_name: + is_def_name = False + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "definition") + elif keyword.iskeyword(token.string): + span_cls = "keyword" + if token.string in KEYWORD_CONSTANTS: + span_cls = "keyword_constant" + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, span_cls) + if token.string in IDENTIFIERS_AFTER: + is_def_name = True + elif ( + keyword.issoftkeyword(token.string) + and bracket_level == 0 + and is_soft_keyword_used(prev_token, token, next_token) + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "soft_keyword") + elif ( + token.string in BUILTINS + and not (prev_token and prev_token.exact_type == T.DOT) + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "builtin") + + +keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"} +keyword_first_sets_case = {"False", "None", "True"} + + +def is_soft_keyword_used(*tokens: TI | None) -> bool: + """Returns True if the current token is a keyword in this context. + + For the `*tokens` to match anything, they have to be a three-tuple of + (previous, current, next). + """ + trace("is_soft_keyword_used{t}", t=tokens) + match tokens: + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + TI(string="match"), + TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) + | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") + ): + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + TI(string="match"), + TI(T.NAME, string=s) + ): + if keyword.iskeyword(s): + return s in keyword_first_sets_match + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="case"), + TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) + | TI(T.OP, string="(" | "*" | "-" | "[" | "{") + ): + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="case"), + TI(T.NAME, string=s) + ): + if keyword.iskeyword(s): + return s in keyword_first_sets_case + return True + case (TI(string="case"), TI(string="_"), TI(string=":")): + return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="type"), + TI(T.NAME, string=s) + ): + return not keyword.iskeyword(s) + case _: + return False + + +def disp_str( + buffer: str, + colors: list[ColorSpan] | None = None, + start_index: int = 0, + force_color: bool = False, +) -> tuple[CharBuffer, CharWidths]: + r"""Decompose the input buffer into a printable variant with applied colors. + + Returns a tuple of two lists: + - the first list is the input buffer, character by character, with color + escape codes added (while those codes contain multiple ASCII characters, + each code is considered atomic *and is attached for the corresponding + visible character*); + - the second list is the visible width of each character in the input + buffer. + + Note on colors: + - The `colors` list, if provided, is partially consumed within. We're using + a list and not a generator since we need to hold onto the current + unfinished span between calls to disp_str in case of multiline strings. + - The `colors` list is computed from the start of the input block. `buffer` + is only a subset of that input block, a single line within. This is why + we need `start_index` to inform us which position is the start of `buffer` + actually within user input. This allows us to match color spans correctly. + + Examples: + >>> utils.disp_str("a = 9") + (['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1]) + + >>> line = "while 1:" + >>> colors = list(utils.gen_colors(line)) + >>> utils.disp_str(line, colors=colors) + (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1]) + + """ + chars: CharBuffer = [] + char_widths: CharWidths = [] + + if not buffer: + return chars, char_widths + + while colors and colors[0].span.end < start_index: + # move past irrelevant spans + colors.pop(0) + + theme = THEME(force_color=force_color) + pre_color = "" + post_color = "" + if colors and colors[0].span.start < start_index: + # looks like we're continuing a previous color (e.g. a multiline str) + pre_color = theme[colors[0].tag] + + for i, c in enumerate(buffer, start_index): + if colors and colors[0].span.start == i: # new color starts now + pre_color = theme[colors[0].tag] + + if c == "\x1a": # CTRL-Z on Windows + chars.append(c) + char_widths.append(2) + elif ord(c) < 128: + chars.append(c) + char_widths.append(1) + elif unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + chars.append(c) + char_widths.append(len(c)) + else: + chars.append(c) + char_widths.append(str_width(c)) + + if colors and colors[0].span.end == i: # current color ends now + post_color = theme.reset + colors.pop(0) + + chars[-1] = pre_color + chars[-1] + post_color + pre_color = "" + post_color = "" + + if colors and colors[0].span.start < i and colors[0].span.end > i: + # even though the current color should be continued, reset it for now. + # the next call to `disp_str()` will revive it. + chars[-1] += theme.reset + + return chars, char_widths + + +def prev_next_window[T]( + iterable: Iterable[T] +) -> Iterator[tuple[T | None, ...]]: + """Generates three-tuples of (previous, current, next) items. + + On the first iteration previous is None. On the last iteration next + is None. In case of exception next is None and the exception is re-raised + on a subsequent next() call. + + Inspired by `sliding_window` from `itertools` recipes. + """ + + iterator = iter(iterable) + window = deque((None, next(iterator)), maxlen=3) + try: + for x in iterator: + window.append(x) + yield tuple(window) + except Exception: + raise + finally: + window.append(None) + yield tuple(window) diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index fffadd5e2ec..46c6030748b 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -22,11 +22,9 @@ import io import os import sys -import time -import msvcrt -from collections import deque import ctypes +import types from ctypes.wintypes import ( _COORD, WORD, @@ -42,9 +40,10 @@ from .console import Event, Console from .trace import trace from .utils import wlen +from .windows_eventqueue import EventQueue try: - from ctypes import GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] + from ctypes import get_last_error, GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] except: # Keep MyPy happy off Windows from ctypes import CDLL as WinDLL, cdll as windll @@ -52,11 +51,20 @@ def GetLastError() -> int: return 42 + def get_last_error() -> int: + return 42 + class WinError(OSError): # type: ignore[no-redef] def __init__(self, err: int | None, descr: str | None = None) -> None: self.err = err self.descr = descr +# declare nt optional to allow None assignment on other platforms +nt: types.ModuleType | None +try: + import nt +except ImportError: + nt = None TYPE_CHECKING = False @@ -94,7 +102,9 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: 0x83: "f20", # VK_F20 } -# Console escape codes: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences +# Virtual terminal output sequences +# Reference: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#output-sequences +# Check `windows_eventqueue.py` for input sequences ERASE_IN_LINE = "\x1b[K" MOVE_LEFT = "\x1b[{}D" MOVE_RIGHT = "\x1b[{}C" @@ -102,10 +112,25 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: MOVE_DOWN = "\x1b[{}B" CLEAR = "\x1b[H\x1b[J" +# State of control keys: https://learn.microsoft.com/en-us/windows/console/key-event-record-str +ALT_ACTIVE = 0x01 | 0x02 +CTRL_ACTIVE = 0x04 | 0x08 + +WAIT_TIMEOUT = 0x102 +WAIT_FAILED = 0xFFFFFFFF + +# from winbase.h +INFINITE = 0xFFFFFFFF + class _error(Exception): pass +def _supports_vt(): + try: + return nt._supports_virtual_terminal() + except AttributeError: + return False class WindowsConsole(Console): def __init__( @@ -117,17 +142,29 @@ def __init__( ): super().__init__(f_in, f_out, term, encoding) + self.__vt_support = _supports_vt() + + if self.__vt_support: + trace('console supports virtual terminal') + + # Save original console modes so we can recover on cleanup. + original_input_mode = DWORD() + GetConsoleMode(InHandle, original_input_mode) + trace(f'saved original input mode 0x{original_input_mode.value:x}') + self.__original_input_mode = original_input_mode.value + SetConsoleMode( OutHandle, ENABLE_WRAP_AT_EOL_OUTPUT | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING, ) + self.screen: list[str] = [] self.width = 80 self.height = 25 self.__offset = 0 - self.event_queue: deque[Event] = deque() + self.event_queue = EventQueue(encoding) try: self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined] except ValueError: @@ -146,8 +183,9 @@ def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: while len(self.screen) < min(len(screen), self.height): self._hide_cursor() - self._move_relative(0, len(self.screen) - 1) - self.__write("\n") + if self.screen: + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") self.posxy = 0, len(self.screen) self.screen.append("") @@ -204,11 +242,9 @@ def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: @property def input_hook(self): - try: - import nt - except ImportError: - return None - if nt._is_inputhook_installed(): + # avoid inline imports here so the repl doesn't get flooded + # with import logging from -X importtime=2 + if nt is not None and nt._is_inputhook_installed(): return nt._inputhook def __write_changed_line( @@ -232,7 +268,7 @@ def __write_changed_line( # reuse the oldline as much as possible, but stop as soon as we # encounter an ESCAPE, because it might be the start of an escape - # sequene + # sequence while ( x_coord < minlen and oldline[x_pos] == newline[x_pos] @@ -247,18 +283,13 @@ def __write_changed_line( self._erase_to_end() self.__write(newline[x_pos:]) - if wlen(newline) == self.width: - # If we wrapped we want to start at the next line - self._move_relative(0, y + 1) - self.posxy = 0, y + 1 - else: - self.posxy = wlen(newline), y + self.posxy = min(wlen(newline), self.width - 1), y - if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) def _scroll( self, top: int, bottom: int, left: int | None = None, right: int | None = None @@ -291,6 +322,12 @@ def _enable_blinking(self): def _disable_blinking(self): self.__write("\x1b[?12l") + def _enable_bracketed_paste(self) -> None: + self.__write("\x1b[?2004h") + + def _disable_bracketed_paste(self) -> None: + self.__write("\x1b[?2004l") + def __write(self, text: str) -> None: if "\x1a" in text: text = ''.join(["^Z" if x == '\x1a' else x for x in text]) @@ -320,8 +357,15 @@ def prepare(self) -> None: self.__gone_tall = 0 self.__offset = 0 + if self.__vt_support: + SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT) + self._enable_bracketed_paste() + def restore(self) -> None: - pass + if self.__vt_support: + # Recover to original mode before running REPL + self._disable_bracketed_paste() + SetConsoleMode(InHandle, self.__original_input_mode) def _move_relative(self, x: int, y: int) -> None: """Moves relative to the current posxy""" @@ -342,7 +386,7 @@ def move_cursor(self, x: int, y: int) -> None: raise ValueError(f"Bad cursor position {x}, {y}") if y < self.__offset or y >= self.__offset + self.height: - self.event_queue.insert(0, Event("scroll", "")) + self.event_queue.insert(Event("scroll", "")) else: self._move_relative(x, y) self.posxy = x, y @@ -371,14 +415,7 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self, block: bool = True) -> INPUT_RECORD | None: - if not block: - events = DWORD() - if not GetNumberOfConsoleInputEvents(InHandle, events): - raise WinError(GetLastError()) - if not events.value: - return None - + def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): @@ -386,15 +423,26 @@ def _read_input(self, block: bool = True) -> INPUT_RECORD | None: return rec + def _read_input_bulk( + self, n: int + ) -> tuple[ctypes.Array[INPUT_RECORD], int]: + rec = (n * INPUT_RECORD)() + read = DWORD() + if not ReadConsoleInput(InHandle, rec, n, read): + raise WinError(GetLastError()) + + return rec, read.value + def get_event(self, block: bool = True) -> Event | None: """Return an Event instance. Returns None if |block| is false and there is no event pending, otherwise waits for the completion of an event.""" - if self.event_queue: - return self.event_queue.pop() - while True: - rec = self._read_input(block) + if not block and not self.wait(timeout=0): + return None + + while self.event_queue.empty(): + rec = self._read_input() if rec is None: return None @@ -407,31 +455,47 @@ def get_event(self, block: bool = True) -> Event | None: continue return None - key = rec.Event.KeyEvent.uChar.UnicodeChar + key_event = rec.Event.KeyEvent + raw_key = key = key_event.uChar.UnicodeChar - if rec.Event.KeyEvent.uChar.UnicodeChar == "\r": - # Make enter make unix-like - return Event(evt="key", data="\n", raw=b"\n") - elif rec.Event.KeyEvent.wVirtualKeyCode == 8: + if key == "\r": + # Make enter unix-like + return Event(evt="key", data="\n") + elif key_event.wVirtualKeyCode == 8: # Turn backspace directly into the command - return Event( - evt="key", - data="backspace", - raw=rec.Event.KeyEvent.uChar.UnicodeChar, - ) - elif rec.Event.KeyEvent.uChar.UnicodeChar == "\x00": + key = "backspace" + elif key == "\x00": # Handle special keys like arrow keys and translate them into the appropriate command - code = VK_MAP.get(rec.Event.KeyEvent.wVirtualKeyCode) - if code: - return Event( - evt="key", data=code, raw=rec.Event.KeyEvent.uChar.UnicodeChar - ) + key = VK_MAP.get(key_event.wVirtualKeyCode) + if key: + if key_event.dwControlKeyState & CTRL_ACTIVE: + key = f"ctrl {key}" + elif key_event.dwControlKeyState & ALT_ACTIVE: + # queue the key, return the meta command + self.event_queue.insert(Event(evt="key", data=key)) + return Event(evt="key", data="\033") # keymap.py uses this for meta + return Event(evt="key", data=key) if block: continue return None - - return Event(evt="key", data=key, raw=rec.Event.KeyEvent.uChar.UnicodeChar) + elif self.__vt_support: + # If virtual terminal is enabled, scanning VT sequences + for char in raw_key.encode(self.event_queue.encoding, "replace"): + self.event_queue.push(char) + continue + + if key_event.dwControlKeyState & ALT_ACTIVE: + # Do not swallow characters that have been entered via AltGr: + # Windows internally converts AltGr to CTRL+ALT, see + # https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-vkkeyscanw + if not key_event.dwControlKeyState & CTRL_ACTIVE: + # queue the key, return the meta command + self.event_queue.insert(Event(evt="key", data=key)) + return Event(evt="key", data="\033") # keymap.py uses this for meta + + return Event(evt="key", data=key) + return self.event_queue.get() def push_char(self, char: int | bytes) -> None: """ @@ -446,7 +510,7 @@ def clear(self) -> None: """Wipe the screen""" self.__write(CLEAR) self.posxy = 0, 0 - self.screen = [""] + self.screen = [] def finish(self) -> None: """Move the cursor to the end of the display and otherwise get @@ -472,18 +536,53 @@ def forgetinput(self) -> None: def getpending(self) -> Event: """Return the characters that have been typed but not yet processed.""" - return Event("key", "", b"") + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + if e2: + e.data += e2.data + + recs, rec_count = self._read_input_bulk(1024) + for i in range(rec_count): + rec = recs[i] + # In case of a legacy console, we do not only receive a keydown + # event, but also a keyup event - and for uppercase letters + # an additional SHIFT_PRESSED event. + if rec and rec.EventType == KEY_EVENT: + key_event = rec.Event.KeyEvent + if not key_event.bKeyDown: + continue + ch = key_event.uChar.UnicodeChar + if ch == "\x00": + # ignore SHIFT_PRESSED and special keys + continue + if ch == "\r": + ch += "\n" + e.data += ch + return e - def wait(self, timeout: float | None) -> bool: + def wait_for_event(self, timeout: float | None) -> bool: """Wait for an event.""" - # Poor man's Windows select loop - start_time = time.time() - while True: - if msvcrt.kbhit(): # type: ignore[attr-defined] - return True - if timeout and time.time() - start_time > timeout / 1000: - return False - time.sleep(0.01) + if timeout is None: + timeout = INFINITE + else: + timeout = int(timeout) + ret = WaitForSingleObject(InHandle, timeout) + if ret == WAIT_FAILED: + raise WinError(get_last_error()) + elif ret == WAIT_TIMEOUT: + return False + return True + + def wait(self, timeout: float | None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.wait_for_event(timeout) + ) def repaint(self) -> None: raise NotImplementedError("No repaint support") @@ -553,6 +652,13 @@ class INPUT_RECORD(Structure): MOUSE_EVENT = 0x02 WINDOW_BUFFER_SIZE_EVENT = 0x04 +ENABLE_PROCESSED_INPUT = 0x0001 +ENABLE_LINE_INPUT = 0x0002 +ENABLE_ECHO_INPUT = 0x0004 +ENABLE_MOUSE_INPUT = 0x0010 +ENABLE_INSERT_MODE = 0x0020 +ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200 + ENABLE_PROCESSED_OUTPUT = 0x01 ENABLE_WRAP_AT_EOL_OUTPUT = 0x02 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04 @@ -584,6 +690,10 @@ class INPUT_RECORD(Structure): ] ScrollConsoleScreenBuffer.restype = BOOL + GetConsoleMode = _KERNEL32.GetConsoleMode + GetConsoleMode.argtypes = [HANDLE, POINTER(DWORD)] + GetConsoleMode.restype = BOOL + SetConsoleMode = _KERNEL32.SetConsoleMode SetConsoleMode.argtypes = [HANDLE, DWORD] SetConsoleMode.restype = BOOL @@ -592,14 +702,15 @@ class INPUT_RECORD(Structure): ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)] ReadConsoleInput.restype = BOOL - GetNumberOfConsoleInputEvents = _KERNEL32.GetNumberOfConsoleInputEvents - GetNumberOfConsoleInputEvents.argtypes = [HANDLE, POINTER(DWORD)] - GetNumberOfConsoleInputEvents.restype = BOOL FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer FlushConsoleInputBuffer.argtypes = [HANDLE] FlushConsoleInputBuffer.restype = BOOL + WaitForSingleObject = _KERNEL32.WaitForSingleObject + WaitForSingleObject.argtypes = [HANDLE, DWORD] + WaitForSingleObject.restype = DWORD + OutHandle = GetStdHandle(STD_OUTPUT_HANDLE) InHandle = GetStdHandle(STD_INPUT_HANDLE) else: @@ -610,9 +721,10 @@ def _win_only(*args, **kwargs): GetStdHandle = _win_only GetConsoleScreenBufferInfo = _win_only ScrollConsoleScreenBuffer = _win_only + GetConsoleMode = _win_only SetConsoleMode = _win_only ReadConsoleInput = _win_only - GetNumberOfConsoleInputEvents = _win_only FlushConsoleInputBuffer = _win_only + WaitForSingleObject = _win_only OutHandle = 0 InHandle = 0 diff --git a/Lib/_pyrepl/windows_eventqueue.py b/Lib/_pyrepl/windows_eventqueue.py new file mode 100644 index 00000000000..d99722f9a16 --- /dev/null +++ b/Lib/_pyrepl/windows_eventqueue.py @@ -0,0 +1,42 @@ +""" +Windows event and VT sequence scanner +""" + +from .base_eventqueue import BaseEventQueue + + +# Reference: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#input-sequences +VT_MAP: dict[bytes, str] = { + b'\x1b[A': 'up', + b'\x1b[B': 'down', + b'\x1b[C': 'right', + b'\x1b[D': 'left', + b'\x1b[1;5D': 'ctrl left', + b'\x1b[1;5C': 'ctrl right', + + b'\x1b[H': 'home', + b'\x1b[F': 'end', + + b'\x7f': 'backspace', + b'\x1b[2~': 'insert', + b'\x1b[3~': 'delete', + b'\x1b[5~': 'page up', + b'\x1b[6~': 'page down', + + b'\x1bOP': 'f1', + b'\x1bOQ': 'f2', + b'\x1bOR': 'f3', + b'\x1bOS': 'f4', + b'\x1b[15~': 'f5', + b'\x1b[17~': 'f6', + b'\x1b[18~': 'f7', + b'\x1b[19~': 'f8', + b'\x1b[20~': 'f9', + b'\x1b[21~': 'f10', + b'\x1b[23~': 'f11', + b'\x1b[24~': 'f12', +} + +class EventQueue(BaseEventQueue): + def __init__(self, encoding: str) -> None: + BaseEventQueue.__init__(self, encoding, VT_MAP) From ec491873605aff47d95185f17e500095ec4025af Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 10 May 2026 13:26:16 +0300 Subject: [PATCH 2/7] Add `test_pyrepl` from 3.14.4 --- Lib/test/test_pyrepl/__init__.py | 11 + Lib/test/test_pyrepl/__main__.py | 4 + Lib/test/test_pyrepl/support.py | 171 ++ Lib/test/test_pyrepl/test_eventqueue.py | 198 ++ Lib/test/test_pyrepl/test_input.py | 102 + Lib/test/test_pyrepl/test_interact.py | 301 +++ Lib/test/test_pyrepl/test_keymap.py | 120 + Lib/test/test_pyrepl/test_pyrepl.py | 2148 ++++++++++++++++++ Lib/test/test_pyrepl/test_reader.py | 560 +++++ Lib/test/test_pyrepl/test_terminfo.py | 651 ++++++ Lib/test/test_pyrepl/test_unix_console.py | 369 +++ Lib/test/test_pyrepl/test_utils.py | 105 + Lib/test/test_pyrepl/test_windows_console.py | 621 +++++ 13 files changed, 5361 insertions(+) create mode 100644 Lib/test/test_pyrepl/__init__.py create mode 100644 Lib/test/test_pyrepl/__main__.py create mode 100644 Lib/test/test_pyrepl/support.py create mode 100644 Lib/test/test_pyrepl/test_eventqueue.py create mode 100644 Lib/test/test_pyrepl/test_input.py create mode 100644 Lib/test/test_pyrepl/test_interact.py create mode 100644 Lib/test/test_pyrepl/test_keymap.py create mode 100644 Lib/test/test_pyrepl/test_pyrepl.py create mode 100644 Lib/test/test_pyrepl/test_reader.py create mode 100644 Lib/test/test_pyrepl/test_terminfo.py create mode 100644 Lib/test/test_pyrepl/test_unix_console.py create mode 100644 Lib/test/test_pyrepl/test_utils.py create mode 100644 Lib/test/test_pyrepl/test_windows_console.py diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py new file mode 100644 index 00000000000..2f37bff6df8 --- /dev/null +++ b/Lib/test/test_pyrepl/__init__.py @@ -0,0 +1,11 @@ +import os +import sys +from test.support import import_helper, load_package_tests + + +if sys.platform != "win32": + import_helper.import_module("termios") + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_pyrepl/__main__.py b/Lib/test/test_pyrepl/__main__.py new file mode 100644 index 00000000000..cbe9e01d0df --- /dev/null +++ b/Lib/test/test_pyrepl/__main__.py @@ -0,0 +1,4 @@ +import unittest +from test.test_pyrepl import load_tests + +unittest.main() diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py new file mode 100644 index 00000000000..4f7f9d77933 --- /dev/null +++ b/Lib/test/test_pyrepl/support.py @@ -0,0 +1,171 @@ +from code import InteractiveConsole +from functools import partial +from typing import Iterable +from unittest.mock import MagicMock + +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent +from _pyrepl.utils import unbracket, ANSI_ESCAPE_SEQUENCE + + +class ScreenEqualMixin: + def assert_screen_equal( + self, reader: ReadlineAlikeReader, expected: str, clean: bool = False + ): + actual = clean_screen(reader) if clean else reader.screen + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + +def multiline_input(reader: ReadlineAlikeReader, namespace: dict | None = None): + saved = reader.more_lines + try: + reader.more_lines = partial(more_lines, namespace=namespace) + reader.ps1 = reader.ps2 = ">>> " + reader.ps3 = reader.ps4 = "... " + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + +def more_lines(text: str, namespace: dict | None = None): + if namespace is None: + namespace = {} + src = _strip_final_indent(text) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def code_to_events(code: str): + for c in code: + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) + + +def clean_screen(reader: ReadlineAlikeReader) -> list[str]: + """Cleans color and console characters out of a screen output. + + This is useful for screen testing, it increases the test readability since + it strips out all the unreadable side of the screen. + """ + output = [] + for line in reader.screen: + line = unbracket(line, including_content=True) + line = ANSI_ESCAPE_SEQUENCE.sub("", line) + for prefix in (reader.ps1, reader.ps2, reader.ps3, reader.ps4): + if line.startswith(prefix): + line = line[len(prefix):] + break + output.append(line) + return output + + +def prepare_reader(console: Console, **kwargs): + config = ReadlineConfig(readline_completer=kwargs.pop("readline_completer", None)) + reader = ReadlineAlikeReader(console=console, config=config) + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def prepare_console(events: Iterable[Event], **kwargs) -> MagicMock | Console: + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def handle_all_events( + events, prepare_console=prepare_console, prepare_reader=prepare_reader +): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + except KeyboardInterrupt: + pass + return reader, console + + +handle_events_narrow_console = partial( + handle_all_events, + prepare_console=partial(prepare_console, width=10), +) + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8") -> None: + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block: bool = True) -> Event | None: + return next(self.events) + + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: int | bytes) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self, timeout: float | None = None) -> bool: + return True + + def repaint(self) -> None: + pass diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py new file mode 100644 index 00000000000..69d9612b70d --- /dev/null +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -0,0 +1,198 @@ +import tempfile +import unittest +from unittest.mock import patch +from test import support + +from _pyrepl import terminfo + +try: + from _pyrepl.console import Event + from _pyrepl import base_eventqueue +except ImportError: + pass + +try: + from _pyrepl import unix_eventqueue +except ImportError: + pass + +try: + from _pyrepl import windows_eventqueue +except ImportError: + pass + +class EventQueueTestBase: + """OS-independent mixin""" + def make_eventqueue(self) -> base_eventqueue.BaseEventQueue: + raise NotImplementedError() + + def test_get(self): + eq = self.make_eventqueue() + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = self.make_eventqueue() + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = self.make_eventqueue() + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = self.make_eventqueue() + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.base_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = self.make_eventqueue() + eq.keymap = {b"a": "b"} + eq.push(b"a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.base_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = self.make_eventqueue() + eq.keymap = {b"c": "d"} + eq.push(b"a") + mock_keymap.compile_keymap.assert_called() + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.base_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = self.make_eventqueue() + eq.keymap = {b"a": {b"b": "c"}} + eq.push(b"a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.push(b"b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push(b"d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.base_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = self.make_eventqueue() + eq.keymap = {b"a": {b"b": "c"}} + eq.push(b"a") + mock_keymap.compile_keymap.assert_called() + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push(b"\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push(b"b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = self.make_eventqueue() + eq.keymap = {} + eq.push(b"\x1b") + eq.push(b"[") + eq.push(b"A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = self.make_eventqueue() + eq.keymap = {} + eq.push(b"\x1b") + eq.push(b"[") + eq.push(b"Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") + + def test_push_unicode_character_as_str(self): + eq = self.make_eventqueue() + eq.keymap = {} + with self.assertRaises(AssertionError): + eq.push("ч") + with self.assertRaises(AssertionError): + eq.push("ñ") + + def test_push_unicode_character_two_bytes(self): + eq = self.make_eventqueue() + eq.keymap = {} + + encoded = "ч".encode(eq.encoding, "replace") + self.assertEqual(len(encoded), 2) + + eq.push(encoded[0]) + e = eq.get() + self.assertIsNone(e) + + eq.push(encoded[1]) + e = eq.get() + self.assertEqual(e.evt, "key") + self.assertEqual(e.data, "ч") + + def test_push_single_chars_and_unicode_character_as_str(self): + eq = self.make_eventqueue() + eq.keymap = {} + + def _event(evt, data, raw=None): + r = raw if raw is not None else data.encode(eq.encoding) + e = Event(evt, data, r) + return e + + def _push(keys): + for k in keys: + eq.push(k) + + self.assertIsInstance("ñ", str) + + # If an exception happens during push, the existing events must be + # preserved and we can continue to push. + _push(b"b") + with self.assertRaises(AssertionError): + _push("ñ") + _push(b"a") + + self.assertEqual(eq.get(), _event("key", "b")) + self.assertEqual(eq.get(), _event("key", "a")) + + +class EmptyTermInfo(terminfo.TermInfo): + def get(self, cap: str) -> bytes: + return b"" + + +@unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") +class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): + def setUp(self): + self.file = tempfile.TemporaryFile() + + def tearDown(self) -> None: + self.file.close() + + def make_eventqueue(self) -> base_eventqueue.BaseEventQueue: + ti = EmptyTermInfo("ansi") + return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8", ti) + + +@unittest.skipUnless(support.MS_WINDOWS, "No Windows event queue on Unix") +class TestWindowsEventQueue(EventQueueTestBase, unittest.TestCase): + def make_eventqueue(self) -> base_eventqueue.BaseEventQueue: + return windows_eventqueue.EventQueue("utf-8") diff --git a/Lib/test/test_pyrepl/test_input.py b/Lib/test/test_pyrepl/test_input.py new file mode 100644 index 00000000000..c78c876c2c4 --- /dev/null +++ b/Lib/test/test_pyrepl/test_input.py @@ -0,0 +1,102 @@ +import unittest + +from _pyrepl.console import Event +from _pyrepl.input import KeymapTranslator + + +class KeymapTranslatorTests(unittest.TestCase): + def test_push_single_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_multiple_keys(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_invalid_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "b") + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["b"])) + + def test_push_invalid_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_character_key(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "a") + translator.push(evt) + result = translator.get() + self.assertEqual(result, ("command_a", ["a"])) + + def test_push_character_key_with_stack(self): + keymap = [("ab", "command_ab")] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + evt3 = Event("key", "c") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + result = translator.get() + self.assertEqual(result, ("command_ab", ["a", "b"])) + + def test_push_transition_key_interrupted(self): + keymap = [("a", {"b": "command_ab"})] + translator = KeymapTranslator(keymap) + evt1 = Event("key", "a") + evt2 = Event("key", "c") + evt3 = Event("key", "b") + translator.push(evt1) + translator.push(evt2) + translator.push(evt3) + result = translator.get() + self.assertEqual(result, (None, ["a", "c"])) + + def test_push_invalid_key_with_unicode_category(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + evt = Event("key", "\u0003") # Control character + translator.push(evt) + result = translator.get() + self.assertEqual(result, (None, ["\u0003"])) + + def test_empty(self): + keymap = [("a", "command_a")] + translator = KeymapTranslator(keymap) + self.assertTrue(translator.empty()) + evt = Event("key", "a") + translator.push(evt) + self.assertFalse(translator.empty()) + translator.get() + self.assertTrue(translator.empty()) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py new file mode 100644 index 00000000000..f0837ee94e9 --- /dev/null +++ b/Lib/test/test_pyrepl/test_interact.py @@ -0,0 +1,301 @@ +import contextlib +import io +import warnings +import unittest +from unittest.mock import patch +from textwrap import dedent + +from test.support import force_not_colorized + +from _pyrepl.console import InteractiveColoredConsole +from _pyrepl.simple_interact import _more_lines + +class TestSimpleInteract(unittest.TestCase): + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + class A: + def foo(self): + + + pass + + class B: + def bar(self): + pass + + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with ( + patch.object(InteractiveColoredConsole, "showsyntaxerror") as showsyntaxerror, + patch.object(InteractiveColoredConsole, "runsource", wraps=console.runsource) as runsource, + contextlib.redirect_stdout(f), + ): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + showsyntaxerror.assert_not_called() + + + def test_multiple_statements_output(self): + namespace = {} + code = dedent("""\ + b = 1 + b + a = 1 + a + """) + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "1\n") + + @force_not_colorized + def test_multiple_statements_fail_early(self): + console = InteractiveColoredConsole() + code = dedent("""\ + raise Exception('foobar') + print('spam', 'eggs', sep='&') + """) + f = io.StringIO() + with contextlib.redirect_stderr(f): + console.runsource(code) + self.assertIn('Exception: foobar', f.getvalue()) + self.assertNotIn('spam&eggs', f.getvalue()) + + def test_empty(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() + with contextlib.redirect_stdout(f): + more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] + self.assertFalse(more) + self.assertEqual(f.getvalue(), "") + + def test_runsource_compiles_and_runs_code(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + with patch.object(console, "runcode") as mock_runcode: + console.runsource(source) + mock_runcode.assert_called_once() + + def test_runsource_returns_false_for_successful_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!')" + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + + @force_not_colorized + def test_runsource_returns_false_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn('SyntaxError', f.getvalue()) + + @force_not_colorized + def test_runsource_show_syntax_error_location(self): + console = InteractiveColoredConsole() + source = "def f(x, x): ..." + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + r = """ + def f(x, x): ... + ^ +SyntaxError: duplicate argument 'x' in function definition""" + self.assertIn(r, f.getvalue()) + + def test_runsource_shows_syntax_error_for_failed_compilation(self): + console = InteractiveColoredConsole() + source = "print('Hello, world!'" + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + source = dedent("""\ + match 1: + case {0: _, 0j: _}: + pass + """) + with patch.object(console, "showsyntaxerror") as mock_showsyntaxerror: + console.runsource(source) + mock_showsyntaxerror.assert_called_once() + + def test_runsource_survives_null_bytes(self): + console = InteractiveColoredConsole() + source = "\x00\n" + f = io.StringIO() + with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertIn("source code string cannot contain null bytes", f.getvalue()) + + def test_no_active_future(self): + console = InteractiveColoredConsole() + source = dedent("""\ + x: int = 1 + print(__annotate__(1)) + """) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': }\n") + + def test_future_annotations(self): + console = InteractiveColoredConsole() + source = dedent("""\ + from __future__ import annotations + def g(x: int): ... + print(g.__annotations__) + """) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertEqual(f.getvalue(), "{'x': 'int'}\n") + + def test_future_barry_as_flufl(self): + console = InteractiveColoredConsole() + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource("from __future__ import barry_as_FLUFL\n") + result = console.runsource("""print("black" <> 'blue')\n""") + self.assertFalse(result) + self.assertEqual(f.getvalue(), "True\n") + + +class TestMoreLines(unittest.TestCase): + def test_invalid_syntax_single_line(self): + namespace = {} + code = "if foo" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_empty_line(self): + namespace = {} + code = "" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_valid_single_statement(self): + namespace = {} + code = "foo = 1" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_assignment(self): + namespace = {} + code = dedent("""\ + foo = [ + 1, + 2, + 3, + ]""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiline_single_block(self): + namespace = {} + code = dedent("""\ + def foo(): + '''docs''' + + return 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_statements_single_line(self): + namespace = {} + code = "foo = 1;bar = 2" + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_statements(self): + namespace = {} + code = dedent("""\ + import time + + foo = 1""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float""") + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + def test_multiple_blocks_empty_newline(self): + namespace = {} + code = dedent("""\ + from dataclasses import dataclass + + @dataclass + class Point: + x: float + y: float + """) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_multiple_blocks_indented_newline(self): + namespace = {} + code = ( + "from dataclasses import dataclass\n" + "\n" + "@dataclass\n" + "class Point:\n" + " x: float\n" + " y: float\n" + " " + ) + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + + def test_incomplete_statement(self): + namespace = {} + code = "if foo:" + console = InteractiveColoredConsole(namespace, filename="") + self.assertTrue(_more_lines(console, code)) + + +class TestWarnings(unittest.TestCase): + def test_pep_765_warning(self): + """ + Test that a SyntaxWarning emitted from the + AST optimizer is only shown once in the REPL. + """ + # gh-131927 + console = InteractiveColoredConsole() + code = dedent("""\ + def f(): + try: + return 1 + finally: + return 2 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + console.runsource(code) + + count = sum("'return' in a 'finally' block" in str(w.message) + for w in caught) + self.assertEqual(count, 1) diff --git a/Lib/test/test_pyrepl/test_keymap.py b/Lib/test/test_pyrepl/test_keymap.py new file mode 100644 index 00000000000..2c97066b2c7 --- /dev/null +++ b/Lib/test/test_pyrepl/test_keymap.py @@ -0,0 +1,120 @@ +import string +import unittest + +from _pyrepl.keymap import _keynames, _escapes, parse_keys, compile_keymap, KeySpecError + + +class TestParseKeys(unittest.TestCase): + def test_single_character(self): + """Ensure that single ascii characters or single digits are parsed as single characters.""" + test_cases = [(key, [key]) for key in string.ascii_letters + string.digits] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_keynames(self): + """Ensure that keynames are parsed to their corresponding mapping. + + A keyname is expected to be of the following form: \\ such as \\ + which would get parsed as "left". + """ + test_cases = [(f"\\<{keyname}>", [parsed_keyname]) for keyname, parsed_keyname in _keynames.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_escape_sequences(self): + """Ensure that escaping sequences are parsed to their corresponding mapping.""" + test_cases = [(f"\\{escape}", [parsed_escape]) for escape, parsed_escape in _escapes.items()] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_control_sequences(self): + """Ensure that supported control sequences are parsed successfully.""" + keys = ["@", "[", "]", "\\", "^", "_", "\\", "\\"] + keys.extend(string.ascii_letters) + test_cases = [(f"\\C-{key}", chr(ord(key) & 0x1F)) for key in []] + for test_key, expected_keys in test_cases: + with self.subTest(f"{test_key} should be parsed as {expected_keys}"): + self.assertEqual(parse_keys(test_key), expected_keys) + + def test_meta_sequences(self): + self.assertEqual(parse_keys("\\M-a"), ["\033", "a"]) + self.assertEqual(parse_keys("\\M-b"), ["\033", "b"]) + self.assertEqual(parse_keys("\\M-c"), ["\033", "c"]) + + def test_combinations(self): + self.assertEqual(parse_keys("\\C-a\\n\\"), ["\x01", "\n", "up"]) + self.assertEqual(parse_keys("\\M-a\\t\\"), ["\033", "a", "\t", "down"]) + + def test_keyspec_errors(self): + cases = [ + ("\\Ca", "\\C must be followed by `-'"), + ("\\ca", "\\C must be followed by `-'"), + ("\\C-\\C-", "doubled \\C-"), + ("\\Ma", "\\M must be followed by `-'"), + ("\\ma", "\\M must be followed by `-'"), + ("\\M-\\M-", "doubled \\M-"), + ("\\", "unrecognised keyname"), + ("\\大", "unknown backslash escape"), + ("\\C-\\", "\\C- followed by invalid key") + ] + for test_keys, expected_err in cases: + with self.subTest(f"{test_keys} should give error {expected_err}"): + with self.assertRaises(KeySpecError) as e: + parse_keys(test_keys) + self.assertIn(expected_err, str(e.exception)) + + def test_index_errors(self): + test_cases = ["\\", "\\C", "\\C-\\C"] + for test_keys in test_cases: + with self.assertRaises(IndexError): + parse_keys(test_keys) + + +class TestCompileKeymap(unittest.TestCase): + def test_empty_keymap(self): + keymap = {} + result = compile_keymap(keymap) + self.assertEqual(result, {}) + + def test_single_keymap(self): + keymap = {b"a": "action"} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": "action"}) + + def test_nested_keymap(self): + keymap = {b"a": {b"b": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action"}}) + + def test_empty_value(self): + keymap = {b"a": {b"": "action"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action"}}) + + def test_multiple_empty_values(self): + keymap = {b"a": {b"": "action1", b"b": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"": "action1", b"b": "action2"}}) + + def test_multiple_keymaps(self): + keymap = {b"a": {b"b": "action1", b"c": "action2"}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": "action1", b"c": "action2"}}) + + def test_nested_multiple_keymaps(self): + keymap = {b"a": {b"b": {b"c": "action"}}} + result = compile_keymap(keymap) + self.assertEqual(result, {b"a": {b"b": {b"c": "action"}}}) + + def test_clashing_definitions(self): + km = {b'a': 'c', b'a' + b'b': 'd'} + with self.assertRaises(KeySpecError): + compile_keymap(km) + + def test_non_bytes_key(self): + with self.assertRaises(TypeError): + compile_keymap({123: 'a'}) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py new file mode 100644 index 00000000000..60561e5663f --- /dev/null +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -0,0 +1,2148 @@ +import importlib +import io +import itertools +import os +import pathlib +import pkgutil +import re +import rlcompleter +import select +import subprocess +import sys +import tempfile +from pkgutil import ModuleInfo +from unittest import TestCase, skipUnless, skipIf, SkipTest +from unittest.mock import Mock, patch +from test.support import force_not_colorized, make_clean_env, Py_DEBUG +from test.support import has_subprocess_support, SHORT_TIMEOUT, STDLIB_DIR +from test.support.import_helper import import_module +from test.support.os_helper import EnvironmentVarGuard, unlink + +from .support import ( + FakeConsole, + ScreenEqualMixin, + handle_all_events, + handle_events_narrow_console, + more_lines, + multiline_input, + code_to_events, +) +from _pyrepl.console import Event +from _pyrepl._module_completer import ( + ImportParser, + ModuleCompleter, + HARDCODED_SUBMODULES, +) +from _pyrepl.readline import ( + ReadlineAlikeReader, + ReadlineConfig, + _ReadlineWrapper, +) +from _pyrepl.readline import multiline_input as readline_multiline_input + +try: + import pty +except ImportError: + pty = None +try: + import readline as readline_module +except ImportError: + readline_module = None + + +class ReplTestCase(TestCase): + def setUp(self): + if not has_subprocess_support: + raise SkipTest("test module requires subprocess") + + def run_repl( + self, + repl_input: str | list[str], + env: dict | None = None, + *, + cmdline_args: list[str] | None = None, + cwd: str | None = None, + skip: bool = False, + timeout: float = SHORT_TIMEOUT, + exit_on_output: str | None = None, + ) -> tuple[str, int]: + temp_dir = None + if cwd is None: + temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + cwd = temp_dir.name + try: + return self._run_repl( + repl_input, + env=env, + cmdline_args=cmdline_args, + cwd=cwd, + skip=skip, + timeout=timeout, + exit_on_output=exit_on_output, + ) + finally: + if temp_dir is not None: + temp_dir.cleanup() + + def _run_repl( + self, + repl_input: str | list[str], + *, + env: dict | None, + cmdline_args: list[str] | None, + cwd: str, + skip: bool, + timeout: float, + exit_on_output: str | None, + ) -> tuple[str, int]: + assert pty + master_fd, slave_fd = pty.openpty() + cmd = [sys.executable, "-i", "-u"] + if env is None: + cmd.append("-I") + elif "PYTHON_HISTORY" not in env: + env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") + if cmdline_args is not None: + cmd.extend(cmdline_args) + + try: + import termios + except ModuleNotFoundError: + pass + else: + term_attr = termios.tcgetattr(slave_fd) + term_attr[6][termios.VREPRINT] = 0 # pass through CTRL-R + term_attr[6][termios.VINTR] = 0 # pass through CTRL-C + termios.tcsetattr(slave_fd, termios.TCSANOW, term_attr) + + process = subprocess.Popen( + cmd, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + cwd=cwd, + text=True, + close_fds=True, + env=env if env else os.environ, + ) + os.close(slave_fd) + if isinstance(repl_input, list): + repl_input = "\n".join(repl_input) + "\n" + os.write(master_fd, repl_input.encode("utf-8")) + + output = [] + while select.select([master_fd], [], [], timeout)[0]: + try: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + if exit_on_output is not None: + output = ["".join(output)] + if exit_on_output in output[0]: + process.kill() + break + else: + os.close(master_fd) + process.kill() + process.wait(timeout=timeout) + self.fail(f"Timeout while waiting for output, got: {''.join(output)}") + + os.close(master_fd) + try: + exit_code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + exit_code = process.wait() + output = "".join(output) + if skip and "can't use pyrepl" in output: + self.skipTest("pyrepl not available") + return output, exit_code + + +class TestCursorPosition(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_up_arrow_simple(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) + + def test_down_arrow_end_of_input(self): + # fmt: off + code = ( + "def f():\n" + " ...\n" + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) + + def test_left_arrow_simple(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) + + def test_right_arrow_end_of_line(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 1 for simple character + self.assertEqual(reader.cxy, (1, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 0) + self.assertEqual(reader.cxy, (0, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor here (showing 2nd line only): + # < ' 可口可乐; 可口可樂'> + # ^ + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.cxy, (10, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain( + code_to_events("' 可口可乐; 可口可樂'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.cxy, (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + first_line = "for _ in _:" + second_line = " hello" + + # fmt: off + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline + self.assertEqual(reader.cxy, (len(second_line), 1)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + + # fmt: off + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.cxy, (len(last_line), 3)) + + def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " x = '可口可乐; 可口可樂'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = '可口可乐; 可口可樂' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) + + +class TestPyReplAutoindent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_auto_indent_default(self): + # fmt: off + input_code = ( + "def f():\n" + "pass\n\n" + ) + + output_code = ( + "def f():\n" + " pass\n" + " " + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_continuation(self): + # auto indenting according to previous user indentation + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_prev_block(self): + # auto indenting according to indentation in different block + # fmt: off + events = itertools.chain( + code_to_events("def f():\n"), + # add backspace to delete default auto-indent + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + code_to_events( + " pass\n" + "pass\n\n" + ), + code_to_events( + "def g():\n" + "pass\n\n" + ), + ) + + output_code = ( + "def g():\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output1 = multiline_input(reader) + output2 = multiline_input(reader) + self.assertEqual(output2, output_code) + + def test_auto_indent_multiline(self): + # fmt: off + events = itertools.chain( + code_to_events( + "def f():\n" + "pass" + ), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new line should be autoindented + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + code_to_events( + "pass" + ), + [ + # go to end of last line + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # double newline to terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + output_code = ( + "def f():\n" + " pass\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_comment(self): + # fmt: off + events = code_to_events( + "def f(): # foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): # foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_with_multicomment(self): + # fmt: off + events = code_to_events( + "def f(): ## foo\n" + "pass\n\n" + ) + + output_code = ( + "def f(): ## foo\n" + " pass\n" + " " + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_auto_indent_ignore_comments(self): + # fmt: off + events = code_to_events( + "pass #:\n" + ) + + output_code = ( + "pass #:" + ) + # fmt: on + + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + +class TestPyReplOutput(ScreenEqualMixin, TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + reader.can_colorize = False + return reader + + def test_stdin_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdin.fileno()): + return + except OSError as ose: + self.skipTest(f"stdin tty check failed: {ose}") + else: + self.skipTest("stdin is not a tty") + + def test_stdout_is_tty(self): + # Used during test log analysis to figure out if a TTY was available. + try: + if os.isatty(sys.stdout.fileno()): + return + except OSError as ose: + self.skipTest(f"stdout tty check failed: {ose}") + else: + self.skipTest("stdout is not a tty") + + def test_basic(self): + reader = self.prepare_reader(code_to_events("1+1\n")) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assert_screen_equal(reader, "1+1", clean=True) + + def test_get_line_buffer_returns_str(self): + reader = self.prepare_reader(code_to_events("\n")) + wrapper = _ReadlineWrapper(f_in=None, f_out=None, reader=reader) + self.assertIs(type(wrapper.get_line_buffer()), str) + + def test_multiline_edit(self): + events = itertools.chain( + code_to_events("def f():\n...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="delete", raw=bytearray(b"\x7F")), + Event(evt="key", data="right", raw=bytearray(b"g")), + Event(evt="key", data="backspace", raw=bytearray(b"\x08")), + Event(evt="key", data="p", raw=bytearray(b"p")), + Event(evt="key", data="a", raw=bytearray(b"a")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="s", raw=bytearray(b"s")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + reader = self.prepare_reader(events) + + output = multiline_input(reader) + expected = "def f():\n ...\n " + self.assertEqual(output, expected) + self.assert_screen_equal(reader, expected, clean=True) + output = multiline_input(reader) + expected = "def g():\n pass\n " + self.assertEqual(output, expected) + self.assert_screen_equal(reader, expected, clean=True) + + def test_history_navigation_with_up_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assert_screen_equal(reader, "1+1", clean=True) + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assert_screen_equal(reader, "2+2", clean=True) + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assert_screen_equal(reader, "2+2", clean=True) + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assert_screen_equal(reader, "1+1", clean=True) + + def test_history_with_multiline_entries(self): + code = "def foo():\nx = 1\ny = 2\nz = 3\n\ndef bar():\nreturn 42\n\n" + events = list(itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ] + )) + + reader = self.prepare_reader(events) + output = multiline_input(reader) + output = multiline_input(reader) + output = multiline_input(reader) + expected = "def foo():\n x = 1\n y = 2\n z = 3\n " + self.assert_screen_equal(reader, expected, clean=True) + self.assertEqual(output, expected) + + + def test_history_navigation_with_down_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assert_screen_equal(reader, "1+1", clean=True) + + def test_history_search(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assert_screen_equal(reader, "1+1", clean=True) + output = multiline_input(reader) + self.assertEqual(output, "2+2") + self.assert_screen_equal(reader, "2+2", clean=True) + output = multiline_input(reader) + self.assertEqual(output, "3+3") + self.assert_screen_equal(reader, "3+3", clean=True) + output = multiline_input(reader) + self.assertEqual(output, "1+1") + self.assert_screen_equal(reader, "1+1", clean=True) + + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + self.assert_screen_equal(reader, "c\x1d", clean=True) + + def test_history_search_backward(self): + # Test history search backward with "imp" input + events = itertools.chain( + code_to_events("import os\n"), + code_to_events("imp"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search for "imp" in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assert_screen_equal(reader, "import os", clean=True) + + def test_history_search_backward_empty(self): + # Test history search backward with an empty input + events = itertools.chain( + code_to_events("import os\n"), + [ + Event(evt='key', data='page up', raw=bytearray(b'\x1b[5~')), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + # fill the history + reader = self.prepare_reader(events) + multiline_input(reader) + + # search backward in history + output = multiline_input(reader) + self.assertEqual(output, "import os") + self.assert_screen_equal(reader, "import os", clean=True) + + +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + @patch("rlcompleter._readline_available", False) + def test_simple_completion(self): + events = code_to_events("os.getpid\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getpid()") + + def test_completion_with_many_options(self): + # Test with something that initially displays many options + # and then complete from one of them. The first time tab is + # pressed, the options are displayed (which corresponds to + # when the repl shows [ not unique ]) and the second completes + # from one of them. + events = code_to_events("os.\t\tO_AP\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_APPEND") + + def test_empty_namespace_completion(self): + events = code_to_events("os.geten\t\n") + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = code_to_events("py\t\n") + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + def test_up_down_arrow_with_completion_menu(self): + """Up arrow in the middle of unfinished tab completion when the menu is displayed + should work and trigger going back in history. Down arrow should subsequently + get us back to the incomplete command.""" + code = "import os\nos.\t\t" + namespace = {"os": os} + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events, namespace=namespace) + output = multiline_input(reader, namespace) + # This is the first line, nothing to see here + self.assertEqual(output, "import os") + # This is the second line. We pressed up and down arrows + # so we should end up where we were when we initiated tab completion. + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.") + + @patch("_pyrepl.readline._ReadlineWrapper.get_reader") + @patch("sys.stderr", new_callable=io.StringIO) + def test_completion_with_warnings(self, mock_stderr, mock_get_reader): + class Dummy: + @property + def test_func(self): + import warnings + + warnings.warn("warnings\n") + return None + + dummy = Dummy() + events = code_to_events("dummy.test_func.\t\n\n") + namespace = {"dummy": dummy} + reader = self.prepare_reader(events, namespace) + mock_get_reader.return_value = reader + output = readline_multiline_input(more_lines, ">>>", "...") + self.assertEqual(output, "dummy.test_func.__") + self.assertEqual(mock_stderr.getvalue(), "") + + +class TestPyReplModuleCompleter(TestCase): + def setUp(self): + # Make iter_modules() search only the standard library. + # This makes the test more reliable in case there are + # other user packages/scripts on PYTHONPATH which can + # interfere with the completions. + lib_path = os.path.dirname(importlib.__path__[0]) + self._saved_sys_path = sys.path + sys.path = [lib_path] + + def tearDown(self): + sys.path = self._saved_sys_path + + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.module_completer = ModuleCompleter(namespace) + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_import_completions(self): + cases = ( + ("import path\t\n", "import pathlib"), + ("import importlib.\t\tres\t\n", "import importlib.resources"), + ("import importlib.resources.\t\ta\t\n", "import importlib.resources.abc"), + ("import foo, impo\t\n", "import foo, importlib"), + ("import foo as bar, impo\t\n", "import foo as bar, importlib"), + ("from impo\t\n", "from importlib"), + ("from importlib.res\t\n", "from importlib.resources"), + ("from importlib.\t\tres\t\n", "from importlib.resources"), + ("from importlib.resources.ab\t\n", "from importlib.resources.abc"), + ("from importlib import mac\t\n", "from importlib import machinery"), + ("from importlib import res\t\n", "from importlib import resources"), + ("from importlib.res\t import a\t\n", "from importlib.resources import abc"), + ("from __phello__ import s\t\n", "from __phello__ import spam"), # frozen module + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True)]) + @patch("sys.builtin_module_names", ()) + def test_private_completions(self): + cases = ( + # Return public methods by default + ("import \t\n", "import public"), + ("from \t\n", "from public"), + # Return private methods if explicitly specified + ("import _\t\n", "import _private"), + ("from _\t\n", "from _private"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + @patch( + "_pyrepl._module_completer.ModuleCompleter.iter_submodules", + lambda *_: [ + ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True), + ], + ) + def test_sub_module_private_completions(self): + cases = ( + # Return public methods by default + ("from foo import \t\n", "from foo import public"), + # Return private methods if explicitly specified + ("from foo import _\t\n", "from foo import _private"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_builtin_completion_top_level(self): + cases = ( + ("import bui\t\n", "import builtins"), + ("from bui\t\n", "from builtins"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_relative_import_completions(self): + cases = ( + (None, "from .readl\t\n", "from .readl"), + (None, "from . import readl\t\n", "from . import readl"), + ("_pyrepl", "from .readl\t\n", "from .readline"), + ("_pyrepl", "from . import readl\t\n", "from . import readline"), + ("_pyrepl", "from .. import toodeep\t\n", "from .. import toodeep"), + ("concurrent", "from .futures.i\t\n", "from .futures.interpreter"), + ) + for package, code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={"__package__": package}) + output = reader.readline() + self.assertEqual(output, expected) + + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "valid_name", True), + ModuleInfo(None, "invalid-name", True)]) + def test_invalid_identifiers(self): + # Make sure modules which are not valid identifiers + # are not suggested as those cannot be imported via 'import'. + cases = ( + ("import valid\t\n", "import valid_name"), + # 'invalid-name' contains a dash and should not be completed + ("import invalid\t\n", "import invalid"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_no_fallback_on_regular_completion(self): + cases = ( + ("import pri\t\n", "import pri"), + ("from pri\t\n", "from pri"), + ("from typing import Na\t\n", "from typing import Na"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_global_cache(self): + with (tempfile.TemporaryDirectory() as _dir1, + patch.object(sys, "path", [_dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + (dir1 / "mod_aa.py").mkdir() + (dir1 / "mod_bb.py").mkdir() + events = code_to_events("import mod_a\t\nimport mod_b\t\n") + reader = self.prepare_reader(events, namespace={}) + output_1, output_2 = reader.readline(), reader.readline() + self.assertEqual(output_1, "import mod_aa") + self.assertEqual(output_2, "import mod_bb") + + def test_hardcoded_stdlib_submodules(self): + cases = ( + ("import collections.\t\n", "import collections.abc"), + ("from os import \t\n", "from os import path"), + ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"), + ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self): + with (tempfile.TemporaryDirectory() as _dir, + patch.object(sys, "modules", {})): # hide imported module + dir = pathlib.Path(_dir) + (dir / "collections").mkdir() + (dir / "collections" / "__init__.py").touch() + (dir / "collections" / "foo.py").touch() + with patch.object(sys, "path", [_dir, *sys.path]): + events = code_to_events("import collections.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import collections.foo") + + def test_already_imported_stdlib_module_no_other_suggestions(self): + with (tempfile.TemporaryDirectory() as _dir, + patch.object(sys, "path", [_dir, *sys.path])): + dir = pathlib.Path(_dir) + (dir / "collections").mkdir() + (dir / "collections" / "__init__.py").touch() + (dir / "collections" / "foo.py").touch() + + # collections found in dir, but was already imported + # from stdlib at startup -> suggest stdlib submodules only + events = code_to_events("import collections.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import collections.abc") + + def test_already_imported_custom_module_no_suggestions(self): + with (tempfile.TemporaryDirectory() as _dir1, + tempfile.TemporaryDirectory() as _dir2, + patch.object(sys, "path", [_dir2, _dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + (dir1 / "mymodule").mkdir() + (dir1 / "mymodule" / "__init__.py").touch() + (dir1 / "mymodule" / "foo.py").touch() + importlib.import_module("mymodule") + + dir2 = pathlib.Path(_dir2) + (dir2 / "mymodule").mkdir() + (dir2 / "mymodule" / "__init__.py").touch() + (dir2 / "mymodule" / "bar.py").touch() + # Purge FileFinder cache after adding files + pkgutil.get_importer(_dir2).invalidate_caches() + # mymodule found in dir2 before dir1, but it was already imported + # from dir1 -> do not suggest dir2 submodules + events = code_to_events("import mymodule.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import mymodule.") + + del sys.modules["mymodule"] + # mymodule not imported anymore -> suggest dir2 submodules + events = code_to_events("import mymodule.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import mymodule.bar") + + def test_already_imported_custom_file_no_suggestions(self): + # Same as before, but mymodule from dir1 has no submodules + # -> propose nothing + with (tempfile.TemporaryDirectory() as _dir1, + tempfile.TemporaryDirectory() as _dir2, + patch.object(sys, "path", [_dir2, _dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + (dir1 / "mymodule").mkdir() + (dir1 / "mymodule.py").touch() + importlib.import_module("mymodule") + + dir2 = pathlib.Path(_dir2) + (dir2 / "mymodule").mkdir() + (dir2 / "mymodule" / "__init__.py").touch() + (dir2 / "mymodule" / "bar.py").touch() + events = code_to_events("import mymodule.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import mymodule.") + del sys.modules["mymodule"] + + def test_already_imported_module_without_origin_or_spec(self): + with (tempfile.TemporaryDirectory() as _dir1, + patch.object(sys, "path", [_dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + for mod in ("no_origin", "not_has_location", "no_spec"): + (dir1 / mod).mkdir() + (dir1 / mod / "__init__.py").touch() + (dir1 / mod / "foo.py").touch() + pkgutil.get_importer(_dir1).invalidate_caches() + module = importlib.import_module(mod) + assert module.__spec__ + if mod == "no_origin": + module.__spec__.origin = None + elif mod == "not_has_location": + module.__spec__.has_location = False + else: + module.__spec__ = None + events = code_to_events(f"import {mod}.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, f"import {mod}.") + del sys.modules[mod] + + def test_get_path_and_prefix(self): + cases = ( + ('', ('', '')), + ('.', ('.', '')), + ('..', ('..', '')), + ('.foo', ('.', 'foo')), + ('..foo', ('..', 'foo')), + ('..foo.', ('..foo', '')), + ('..foo.bar', ('..foo', 'bar')), + ('.foo.bar.', ('.foo.bar', '')), + ('..foo.bar.', ('..foo.bar', '')), + ('foo', ('', 'foo')), + ('foo.', ('foo', '')), + ('foo.bar', ('foo', 'bar')), + ('foo.bar.', ('foo.bar', '')), + ('foo.bar.baz', ('foo.bar', 'baz')), + ) + completer = ModuleCompleter() + for name, expected in cases: + with self.subTest(name=name): + self.assertEqual(completer.get_path_and_prefix(name), expected) + + def test_parse(self): + cases = ( + ('import ', (None, '')), + ('import foo', (None, 'foo')), + ('import foo,', (None, '')), + ('import foo, ', (None, '')), + ('import foo, bar', (None, 'bar')), + ('import foo, bar, baz', (None, 'baz')), + ('import foo as bar,', (None, '')), + ('import foo as bar, ', (None, '')), + ('import foo as bar, baz', (None, 'baz')), + ('import a.', (None, 'a.')), + ('import a.b', (None, 'a.b')), + ('import a.b.', (None, 'a.b.')), + ('import a.b.c', (None, 'a.b.c')), + ('import a.b.c, foo', (None, 'foo')), + ('import a.b.c, foo.bar', (None, 'foo.bar')), + ('import a.b.c, foo.bar,', (None, '')), + ('import a.b.c, foo.bar, ', (None, '')), + ('from foo', ('foo', None)), + ('from a.', ('a.', None)), + ('from a.b', ('a.b', None)), + ('from a.b.', ('a.b.', None)), + ('from a.b.c', ('a.b.c', None)), + ('from foo import ', ('foo', '')), + ('from foo import a', ('foo', 'a')), + ('from ', ('', None)), + ('from . import a', ('.', 'a')), + ('from .foo import a', ('.foo', 'a')), + ('from ..foo import a', ('..foo', 'a')), + ('from foo import (', ('foo', '')), + ('from foo import ( ', ('foo', '')), + ('from foo import (a', ('foo', 'a')), + ('from foo import (a,', ('foo', '')), + ('from foo import (a, ', ('foo', '')), + ('from foo import (a, c', ('foo', 'c')), + ('from foo import (a as b, c', ('foo', 'c')), + ) + for code, parsed in cases: + parser = ImportParser(code) + actual = parser.parse() + with self.subTest(code=code): + self.assertEqual(actual, parsed) + # The parser should not get tripped up by any + # other preceding statements + _code = f'import xyz\n{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): + self.assertEqual(actual, parsed) + _code = f'import xyz;{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): + self.assertEqual(actual, parsed) + + def test_parse_error(self): + cases = ( + '', + 'import foo ', + 'from foo ', + 'import foo. ', + 'import foo.bar ', + 'from foo ', + 'from foo. ', + 'from foo.bar ', + 'from foo import bar ', + 'from foo import (bar ', + 'from foo import bar, baz ', + 'import foo as', + 'import a. as', + 'import a.b as', + 'import a.b. as', + 'import a.b.c as', + 'import (foo', + 'import (', + 'import .foo', + 'import ..foo', + 'import .foo.bar', + 'import foo; x = 1', + 'import foo; 1,', + 'import a.; x = 1', + 'import a.b; x = 1', + 'import a.b.; x = 1', + 'import a.b.c; x = 1', + 'from foo import a as', + 'from foo import a. as', + 'from foo import a.b as', + 'from foo import a.b. as', + 'from foo import a.b.c as', + 'from foo impo', + 'import import', + 'import from', + 'import as', + 'from import', + 'from from', + 'from as', + 'from foo import import', + 'from foo import from', + 'from foo import as', + 'from \\x', # _tokenize SyntaxError -> tokenize TokenError + 'if 1:\n pass\n\tpass', # _tokenize TabError -> tokenize TabError + ) + for code in cases: + parser = ImportParser(code) + actual = parser.parse() + with self.subTest(code=code): + self.assertEqual(actual, None) + + +class TestHardcodedSubmodules(TestCase): + def test_hardcoded_stdlib_submodules_are_importable(self): + for parent_path, submodules in HARDCODED_SUBMODULES.items(): + for module_name in submodules: + path = f"{parent_path}.{module_name}" + with self.subTest(path=path): + # We can't use importlib.util.find_spec here, + # since some hardcoded submodules parents are + # not proper packages + importlib.import_module(path) + + +class TestPasteEvent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_paste(self): + # fmt: off + code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:\n" + " pass\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + # fmt: off + code = ( + "def f():\n" + " x = y\n" + " \n" + " y = z\n" + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off + code = ( + "def f():\n" + "x = y\n" + "\n" + "y = z\n\n" + ) + + expected = ( + "def f():\n" + " x = y\n" + " " + ) + # fmt: on + + events = code_to_events(code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, expected) + + def test_paste_not_in_paste_mode(self): + # fmt: off + input_code = ( + "def a():\n" + "for x in range(10):\n" + "if x%2:\n" + "print(x)\n" + "else:\n" + "pass\n\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + " if x%2:\n" + " print(x)\n" + " else:" + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste(self): + """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" + # fmt: off + input_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + + output_code = ( + "def a():\n" + " for x in range(10):\n" + "\n" + " if x%2:\n" + " print(x)\n" + "\n" + " else:\n" + " pass\n" + ) + # fmt: on + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + def test_bracketed_paste_single_line(self): + input_code = "oneline" + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, input_code) + + +@skipUnless(pty, "requires pty") +class TestDumbTerminal(ReplTestCase): + def test_dumb_terminal_exits_cleanly(self): + env = os.environ.copy() + env.pop('PYTHON_BASIC_REPL', None) + # Ignore PYTHONSTARTUP to not pollute the output + # with an unrelated traceback. See GH-137568. + env.pop('PYTHONSTARTUP', None) + env.update({"TERM": "dumb"}) + output, exit_code = self.run_repl("exit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("warning: can't use pyrepl", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + +@skipUnless(pty, "requires pty") +@skipIf((os.environ.get("TERM") or "dumb") == "dumb", "can't use pyrepl in dumb terminal") +class TestMain(ReplTestCase): + def setUp(self): + # Cleanup from PYTHON* variables to isolate from local + # user settings, see #121359. Such variables should be + # added later in test methods to patched os.environ. + super().setUp() + patcher = patch('os.environ', new=make_clean_env()) + self.addCleanup(patcher.stop) + patcher.start() + + @force_not_colorized + def test_exposed_globals_in_repl(self): + pre = "['__builtins__'" + post = "'__loader__', '__name__', '__package__', '__spec__']" + output, exit_code = self.run_repl(["sorted(dir())", "exit()"], skip=True) + self.assertEqual(exit_code, 0) + + # if `__main__` is not a file (impossible with pyrepl) + case1 = f"{pre}, '__doc__', {post}" in output + + # if `__main__` is an uncached .py file (no .pyc) + case2 = f"{pre}, '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file and the .py source exists + case3 = f"{pre}, '__cached__', '__doc__', '__file__', {post}" in output + + # if `__main__` is a cached .pyc file but there's no .py source file + case4 = f"{pre}, '__cached__', '__doc__', {post}" in output + + self.assertTrue(case1 or case2 or case3 or case4, output) + + def _assertMatchOK( + self, var: str, expected: str | re.Pattern, actual: str + ) -> None: + if isinstance(expected, re.Pattern): + self.assertTrue( + expected.match(actual), + f"{var}={actual} does not match {expected.pattern}", + ) + else: + self.assertEqual( + actual, + expected, + f"expected {var}={expected}, got {var}={actual}", + ) + + @force_not_colorized + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False, pythonstartup=False): + clean_env = make_clean_env() + clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses + + with tempfile.TemporaryDirectory() as td: + blue = pathlib.Path(td) / "blue" + blue.mkdir() + mod = blue / "calx.py" + mod.write_text("FOO = 42", encoding="utf-8") + startup = blue / "startup.py" + startup.write_text("BAR = 64", encoding="utf-8") + commands = [ + "print(f'^{" + var + "=}')" for var in expectations + ] + ["exit()"] + if pythonstartup: + clean_env["PYTHONSTARTUP"] = str(startup) + if as_file and as_module: + self.fail("as_file and as_module are mutually exclusive") + elif as_file: + output, exit_code = self.run_repl( + commands, + cmdline_args=[str(mod)], + env=clean_env, + skip=True, + ) + elif as_module: + output, exit_code = self.run_repl( + commands, + cmdline_args=["-m", "blue.calx"], + env=clean_env, + cwd=td, + skip=True, + ) + else: + output, exit_code = self.run_repl( + commands, + cmdline_args=[], + env=clean_env, + cwd=td, + skip=True, + ) + + self.assertEqual(exit_code, 0) + for var, expected in expectations.items(): + with self.subTest(var=var, expected=expected): + if m := re.search(rf"\^{var}=(.+?)[\r\n]", output): + self._assertMatchOK(var, expected, actual=m.group(1)) + else: + self.fail(f"{var}= not found in output: {output!r}\n\n{output}") + + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + def test_globals_initialized_as_default(self): + expectations = { + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations) + + def test_globals_initialized_from_pythonstartup(self): + expectations = { + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, pythonstartup=True) + + def test_inspect_keeps_globals_from_inspected_file(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True) + + def test_inspect_keeps_globals_from_inspected_file_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True, pythonstartup=True) + + def test_inspect_keeps_globals_from_inspected_module(self): + expectations = { + "FOO": "42", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True) + + def test_inspect_keeps_globals_from_inspected_module_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True, pythonstartup=True) + + @force_not_colorized + def test_python_basic_repl(self): + env = os.environ.copy() + pyrepl_commands = "clear\nexit()\n" + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(pyrepl_commands, env=env, skip=True) + self.assertEqual(exit_code, 0) + self.assertNotIn("Exception", output) + self.assertNotIn("NameError", output) + self.assertNotIn("Traceback", output) + + basic_commands = "help\nexit()\n" + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(basic_commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("Type help() for interactive help", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + # The site module must not load _pyrepl if PYTHON_BASIC_REPL is set + commands = ("import sys\n" + "print('_pyrepl' in sys.modules)\n" + "exit()\n") + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("False", output) + self.assertNotIn("True", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + @force_not_colorized + def test_no_pyrepl_source_in_exc(self): + # Avoid using _pyrepl/__main__.py in traceback reports + # See https://github.com/python/cpython/issues/129098. + pyrepl_main_file = os.path.join(STDLIB_DIR, "_pyrepl", "__main__.py") + self.assertTrue(os.path.exists(pyrepl_main_file), pyrepl_main_file) + with open(pyrepl_main_file) as fp: + excluded_lines = fp.readlines() + excluded_lines = list(filter(None, map(str.strip, excluded_lines))) + + for filename in ['?', 'unknown-filename', '', '<...>']: + self._test_no_pyrepl_source_in_exc(filename, excluded_lines) + + def _test_no_pyrepl_source_in_exc(self, filename, excluded_lines): + with EnvironmentVarGuard() as env, self.subTest(filename=filename): + env.unset("PYTHON_BASIC_REPL") + commands = (f"eval(compile('spam', {filename!r}, 'eval'))\n" + f"exit()\n") + output, _ = self.run_repl(commands, env=env) + self.assertIn("Traceback (most recent call last)", output) + self.assertIn("NameError: name 'spam' is not defined", output) + for line in excluded_lines: + with self.subTest(line=line): + self.assertNotIn(line, output) + + @force_not_colorized + def test_bad_sys_excepthook_doesnt_crash_pyrepl(self): + env = os.environ.copy() + commands = ("import sys\n" + "sys.excepthook = 1\n" + "1/0\n" + "exit()\n") + + def check(output, exitcode): + self.assertIn("Error in sys.excepthook:", output) + self.assertEqual(output.count("'int' object is not callable"), 1) + self.assertIn("Original exception was:", output) + self.assertIn("division by zero", output) + self.assertEqual(exitcode, 0) + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env, skip=True) + check(output, exit_code) + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + check(output, exit_code) + + def test_not_wiping_history_file(self): + # skip, if readline module is not available + import_module('readline') + + hfile = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(unlink, hfile.name) + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + commands = "123\nspam\nexit()\n" + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + hfile.file.truncate() + hfile.close() + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl(commands, env=env) + self.assertEqual(exit_code, 0) + self.assertIn("123", output) + self.assertIn("spam", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + + @force_not_colorized + def test_correct_filename_in_syntaxerrors(self): + env = os.environ.copy() + commands = "a b c\nexit()\n" + output, exit_code = self.run_repl(commands, env=env, skip=True) + self.assertIn("SyntaxError: invalid syntax", output) + self.assertIn("", output) + commands = " b\nexit()\n" + output, exit_code = self.run_repl(commands, env=env) + self.assertIn("IndentationError: unexpected indent", output) + self.assertIn("", output) + + @force_not_colorized + def test_proper_tracebacklimit(self): + env = os.environ.copy() + for set_tracebacklimit in [True, False]: + commands = ("import sys\n" + + ("sys.tracebacklimit = 1\n" if set_tracebacklimit else "") + + "def x1(): 1/0\n\n" + "def x2(): x1()\n\n" + "def x3(): x2()\n\n" + "x3()\n" + "exit()\n") + + for basic_repl in [True, False]: + if basic_repl: + env["PYTHON_BASIC_REPL"] = "1" + else: + env.pop("PYTHON_BASIC_REPL", None) + with self.subTest(set_tracebacklimit=set_tracebacklimit, + basic_repl=basic_repl): + output, exit_code = self.run_repl(commands, env=env, skip=True) + self.assertIn("in x1", output) + if set_tracebacklimit: + self.assertNotIn("in x2", output) + self.assertNotIn("in x3", output) + self.assertNotIn("in ", output) + else: + self.assertIn("in x2", output) + self.assertIn("in x3", output) + self.assertIn("in ", output) + + def test_null_byte(self): + output, exit_code = self.run_repl("\x00\nexit()\n") + self.assertEqual(exit_code, 0) + self.assertNotIn("TypeError", output) + + @force_not_colorized + def test_non_string_suggestion_candidates(self): + commands = ("import runpy\n" + "runpy._run_module_code('blech', {0: '', 'bluch': ''}, '')\n" + "exit()\n") + + output, exit_code = self.run_repl(commands) + self.assertEqual(exit_code, 0) + self.assertNotIn("all elements in 'candidates' must be strings", output) + self.assertIn("bluch", output) + + def test_readline_history_file(self): + # skip, if readline module is not available + readline = import_module('readline') + if readline.backend != "editline": + self.skipTest("GNU readline is not affected by this issue") + + with tempfile.NamedTemporaryFile() as hfile: + env = os.environ.copy() + env["PYTHON_HISTORY"] = hfile.name + + env["PYTHON_BASIC_REPL"] = "1" + output, exit_code = self.run_repl("spam \nexit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("spam ", output) + self.assertNotEqual(pathlib.Path(hfile.name).stat().st_size, 0) + self.assertIn("spam\\040", pathlib.Path(hfile.name).read_text()) + + env.pop("PYTHON_BASIC_REPL", None) + output, exit_code = self.run_repl("exit\n", env=env) + self.assertEqual(exit_code, 0) + self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text()) + + def test_history_survive_crash(self): + env = os.environ.copy() + + with tempfile.NamedTemporaryFile() as hfile: + env["PYTHON_HISTORY"] = hfile.name + + commands = "1\n2\n3\nexit()\n" + output, exit_code = self.run_repl(commands, env=env, skip=True) + self.assertEqual(exit_code, 0) + + # Run until "0xcafe" is printed (as "51966") and then kill the + # process to simulate a crash. Note that the output also includes + # the echoed input commands. + commands = "spam\nimport time\n0xcafe\ntime.sleep(1000)\nquit\n" + output, exit_code = self.run_repl(commands, env=env, + exit_on_output="51966") + self.assertNotEqual(exit_code, 0) + + history = pathlib.Path(hfile.name).read_text() + self.assertIn("2", history) + self.assertIn("exit()", history) + self.assertIn("spam", history) + self.assertIn("import time", history) + # History is written after each command's output is printed to the + # console, so depending on how quickly the process is killed, + # the last command may or may not be written to the history file. + self.assertNotIn("sleep", history) + self.assertNotIn("quit", history) + + def test_keyboard_interrupt_after_isearch(self): + output, exit_code = self.run_repl("\x12\x03exit\n") + self.assertEqual(exit_code, 0) + + def test_prompt_after_help(self): + output, exit_code = self.run_repl(["help", "q", "exit"]) + + # Regex pattern to remove ANSI escape sequences + ansi_escape = re.compile(r"(\x1B(=|>|(\[)[0-?]*[ -\/]*[@-~]))") + cleaned_output = ansi_escape.sub("", output) + self.assertEqual(exit_code, 0) + + # Ensure that we don't see multiple prompts after exiting `help` + # Extra stuff (newline and `exit` rewrites) are necessary + # because of how run_repl works. + self.assertNotIn(">>> \n>>> >>>", cleaned_output) + + @skipUnless(Py_DEBUG, '-X showrefcount requires a Python debug build') + def test_showrefcount(self): + env = os.environ.copy() + env.pop("PYTHON_BASIC_REPL", "") + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) + + env["PYTHON_BASIC_REPL"] = "1" + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) + + + @force_not_colorized + def test_no_newline(self): + env = os.environ.copy() + env.pop("PYTHON_BASIC_REPL", "") + env["PYTHON_BASIC_REPL"] = "1" + + commands = "print('Something pretty long', end='')\nexit()\n" + expected_output_sequence = "Something pretty long>>> exit()" + + # gh-143394: The basic REPL needs the readline module to turn off + # ECHO terminal attribute. + if readline_module is not None: + basic_output, basic_exit_code = self.run_repl(commands, env=env) + self.assertEqual(basic_exit_code, 0) + self.assertIn(expected_output_sequence, basic_output) + + output, exit_code = self.run_repl(commands) + self.assertEqual(exit_code, 0) + + # Build patterns for escape sequences that don't affect cursor position + # or visual output. Use terminfo to get platform-specific sequences, + # falling back to hard-coded patterns for capabilities not in terminfo. + from _pyrepl.terminfo import TermInfo + ti = TermInfo(os.environ.get("TERM", "")) + + safe_patterns = [] + + # smkx/rmkx - application cursor keys and keypad mode + smkx = ti.get("smkx") + rmkx = ti.get("rmkx") + if smkx: + safe_patterns.append(re.escape(smkx.decode("ascii"))) + if rmkx: + safe_patterns.append(re.escape(rmkx.decode("ascii"))) + if not smkx and not rmkx: + safe_patterns.append(r'\x1b\[\?1[hl]') # application cursor keys + safe_patterns.append(r'\x1b[=>]') # application keypad mode + + # ich1 - insert character (only safe form that inserts exactly 1 char) + ich1 = ti.get("ich1") + if ich1: + safe_patterns.append(re.escape(ich1.decode("ascii")) + r'(?=[ -~])') + else: + safe_patterns.append(r'\x1b\[(?:1)?@(?=[ -~])') + + # civis/cnorm - cursor visibility (may include cursor blinking control) + civis = ti.get("civis") + cnorm = ti.get("cnorm") + if civis: + safe_patterns.append(re.escape(civis.decode("ascii"))) + if cnorm: + safe_patterns.append(re.escape(cnorm.decode("ascii"))) + if not civis and not cnorm: + safe_patterns.append(r'\x1b\[\?25[hl]') # cursor visibility + safe_patterns.append(r'\x1b\[\?12[hl]') # cursor blinking + + # rmam / smam - automatic margins + rmam = ti.get("rmam") + smam = ti.get("smam") + if rmam: + safe_patterns.append(re.escape(rmam.decode("ascii"))) + if smam: + safe_patterns.append(re.escape(smam.decode("ascii"))) + if not rmam and not smam: + safe_patterns.append(r'\x1b\[\?7l') # turn off automatic margins + safe_patterns.append(r'\x1b\[\?7h') # turn on automatic margins + + # Modern extensions not in standard terminfo - always use patterns + safe_patterns.append(r'\x1b\[\?2004[hl]') # bracketed paste mode + safe_patterns.append(r'\x1b\[\?12[hl]') # cursor blinking (may be separate) + safe_patterns.append(r'\x1b\[\?[01]c') # device attributes + + safe_escapes = re.compile('|'.join(safe_patterns)) + cleaned_output = safe_escapes.sub('', output) + self.assertIn(expected_output_sequence, cleaned_output) + + +@skipUnless(sys.platform == "darwin", "macOS only") +class TestMainAppleTerminal(TestMain): + """Test the REPL with Apple Terminal's TERM_PROGRAM set.""" + + def run_repl(self, repl_input, env=None, **kwargs): + if env is None: + env = os.environ.copy() + env["TERM_PROGRAM"] = "Apple_Terminal" + return super().run_repl(repl_input, env=env, **kwargs) + + +class TestPyReplCtrlD(TestCase): + """Test Ctrl+D behavior in _pyrepl to match old pre-3.13 REPL behavior. + + Ctrl+D should: + - Exit on empty buffer (raises EOFError) + - Delete character when cursor is in middle of line + - Perform no operation when cursor is at end of line without newline + - Exit multiline mode when cursor is at end with trailing newline + - Run code up to that point when pressed on blank line with preceding lines + """ + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_ctrl_d_empty_line(self): + """Test that pressing Ctrl+D on empty line exits the program""" + events = [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ] + reader = self.prepare_reader(events) + with self.assertRaises(EOFError): + multiline_input(reader) + + def test_ctrl_d_multiline_with_new_line(self): + """Test that pressing Ctrl+D in multiline mode with trailing newline exits multiline mode""" + events = itertools.chain( + code_to_events("def f():\n pass\n"), # Enter multiline mode with trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ], + ) + reader, _ = handle_all_events(events) + self.assertTrue(reader.finished) + self.assertEqual("def f():\n pass\n", "".join(reader.buffer)) + + def test_ctrl_d_multiline_middle_of_line(self): + """Test that pressing Ctrl+D in multiline mode with cursor in middle deletes character""" + events = itertools.chain( + code_to_events("def f():\n hello world"), # Enter multiline mode + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")) + ] * 5, # move cursor to 'w' in "world" + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should delete 'w' + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello orld", "".join(reader.buffer)) + + def test_ctrl_d_multiline_end_of_line_no_newline(self): + """Test that pressing Ctrl+D at end of line without newline performs no operation""" + events = itertools.chain( + code_to_events("def f():\n hello"), # Enter multiline mode, no trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should be no-op + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello", "".join(reader.buffer)) + + def test_ctrl_d_single_line_middle_of_line(self): + """Test that pressing Ctrl+D in single line mode deletes current character""" + events = itertools.chain( + code_to_events("hello"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], # move left + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hell", "".join(reader.buffer)) + + def test_ctrl_d_single_line_end_no_newline(self): + """Test that pressing Ctrl+D at end of single line without newline does nothing""" + events = itertools.chain( + code_to_events("hello"), # cursor at end of line + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hello", "".join(reader.buffer)) + + +@skipUnless(sys.platform == "win32", "windows console only") +class TestWindowsConsoleEolWrap(TestCase): + def _make_mock_console(self, width=80): + from _pyrepl import windows_console as wc + + console = object.__new__(wc.WindowsConsole) + + console.width = width + console.posxy = (0, 0) + console.screen = [""] + + console._hide_cursor = Mock() + console._show_cursor = Mock() + console._erase_to_end = Mock() + console._move_relative = Mock() + console.move_cursor = Mock() + console._WindowsConsole__write = Mock() + + return console, wc + + def test_short_line_sets_posxy_normally(self): + width = 10 + y = 3 + console, wc = self._make_mock_console(width=width) + old_line = "" + new_line = "a" * 3 + wc.WindowsConsole._WindowsConsole__write_changed_line( + console, y, old_line, new_line, 0 + ) + self.assertEqual(console.posxy, (3, y)) + + def test_exact_width_line_does_not_wrap(self): + width = 10 + y = 3 + console, wc = self._make_mock_console(width=width) + old_line = "" + new_line = "a" * width + + wc.WindowsConsole._WindowsConsole__write_changed_line( + console, y, old_line, new_line, 0 + ) + self.assertEqual(console.posxy, (width - 1, y)) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py new file mode 100644 index 00000000000..b1b6ae16a1e --- /dev/null +++ b/Lib/test/test_pyrepl/test_reader.py @@ -0,0 +1,560 @@ +import itertools +import functools +import rlcompleter +from textwrap import dedent +from unittest import TestCase +from unittest.mock import MagicMock +from test.support import force_colorized_test_class, force_not_colorized_test_class + +from .support import handle_all_events, handle_events_narrow_console +from .support import ScreenEqualMixin, code_to_events +from .support import prepare_reader, prepare_console +from _pyrepl.console import Event +from _pyrepl.reader import Reader +from _colorize import default_theme + + +overrides = {"reset": "z", "soft_keyword": "K"} +colors = {overrides.get(k, k[0].lower()): v for k, v in default_theme.syntax.items()} + + +@force_not_colorized_test_class +class TestReader(ScreenEqualMixin, TestCase): + def test_calc_screen_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equal(reader, f"{9*"a"}\\\na") + + def test_calc_screen_wrap_wide_characters(self): + events = code_to_events(8 * "a" + "樂") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equal(reader, f"{8*"a"}\\\n樂") + + def test_calc_screen_wrap_three_lines(self): + events = code_to_events(20 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equal(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_calc_screen_prompt_handling(self): + def prepare_reader_keep_prompts(*args, **kwargs): + reader = prepare_reader(*args, **kwargs) + del reader.get_prompt + reader.ps1 = ">>> " + reader.ps2 = ">>> " + reader.ps3 = "... " + reader.ps4 = "" + reader.can_colorize = False + reader.paste_mode = False + return reader + + events = code_to_events("if some_condition:\nsome_function()") + reader, _ = handle_events_narrow_console( + events, + prepare_reader=prepare_reader_keep_prompts, + ) + # fmt: off + self.assert_screen_equal( + reader, + ( + ">>> if so\\\n" + "me_condit\\\n" + "ion:\n" + "... s\\\n" + "ome_funct\\\n" + "ion()" + ) + ) + # fmt: on + + def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + + # fmt: off + self.assert_screen_equal( + reader, + ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + ), + clean=True, + ) + # fmt: on + + def test_calc_screen_backspace(self): + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, "aa") + + def test_calc_screen_wrap_removes_after_backspace(self): + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equal(reader, 9 * "a") + + def test_calc_screen_backspace_in_second_line_after_wrap(self): + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equal(reader, f"{9*"a"}\\\na") + + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_setpos_from_xy_multiple_lines(self): + # fmt: off + code = ( + "def foo():\n" + " return 1" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + + def test_up_arrow_after_ctrl_r(self): + events = iter( + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ] + ) + + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, "") + + def test_newline_within_block_trailing_whitespace(self): + # fmt: off + code = ( + "def foo():\n" + "a = 1\n" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + # go to the end of the first line + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # new lines in-block shouldn't terminate the block + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + # end of line 2 + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\x05", raw=bytearray(b"\x1bO5")), + # a double new line in-block should terminate the block + # even if its followed by whitespace + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + no_paste_reader = functools.partial(prepare_reader, paste_mode=False) + reader, _ = handle_all_events(events, prepare_reader=no_paste_reader) + + expected = ( + "def foo():\n" + " \n" + " \n" + " a = 1\n" + " \n" + " " # HistoricalReader will trim trailing whitespace + ) + self.assert_screen_equal(reader, expected, clean=True) + self.assertTrue(reader.finished) + + def test_input_hook_is_called_if_set(self): + input_hook = MagicMock() + + def _prepare_console(events): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + console.input_hook = input_hook + return console + + events = code_to_events("a") + reader, _ = handle_all_events(events, prepare_console=_prepare_console) + + self.assertEqual(len(input_hook.mock_calls), 4) + + def test_keyboard_interrupt_clears_screen(self): + namespace = {"itertools": itertools} + code = "import itertools\nitertools." + events = itertools.chain( + code_to_events(code), + [ + # Two tabs for completion + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\x03", raw=bytearray(b"\x03")), # Ctrl-C + ], + ) + console = prepare_console(events) + reader = prepare_reader( + console, + readline_completer=rlcompleter.Completer(namespace).complete, + ) + try: + # we're not using handle_all_events() here to be able to + # follow the KeyboardInterrupt sequence of events. Normally this + # happens in simple_interact.run_multiline_interactive_console. + while True: + reader.handle1() + except KeyboardInterrupt: + # at this point the completions are still visible + self.assertTrue(len(reader.screen) > 2) + reader.refresh() + # after the refresh, they are gone + self.assertEqual(len(reader.screen), 2) + self.assert_screen_equal(reader, code, clean=True) + else: + self.fail("KeyboardInterrupt not raised.") + + def test_prompt_length(self): + # Handles simple ASCII prompt + ps1 = ">>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 4) + + # Handles ANSI escape sequences + ps1 = "\033[0;32m>>> \033[0m" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles ANSI escape sequences bracketed in \001 .. \002 + ps1 = "\001\033[0;32m\002>>> \001\033[0m\002" + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m>>> \033[0m") + self.assertEqual(l, 4) + + # Handles wide characters in prompt + ps1 = "樂>> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, ps1) + self.assertEqual(l, 5) + + # Handles wide characters AND ANSI sequences together + ps1 = "\001\033[0;32m\002樂>\001\033[0m\002> " + prompt, l = Reader.process_prompt(ps1) + self.assertEqual(prompt, "\033[0;32m樂>\033[0m> ") + self.assertEqual(l, 5) + + def test_completions_updated_on_key_press(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain( + code_to_events(code), + [ + # Two tabs for completion + Event(evt="key", data="\t", raw=bytearray(b"\t")), + Event(evt="key", data="\t", raw=bytearray(b"\t")), + ], + code_to_events("a"), + ) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete, + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + actual = reader.screen + self.assertEqual(len(actual), 2) + self.assertEqual(actual[0], f"{code}a") + self.assertEqual(actual[1].rstrip(), "itertools.accumulate(") + + def test_key_press_on_tab_press_once(self): + namespace = {"itertools": itertools} + code = "itertools." + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="\t", raw=bytearray(b"\t")), + ], + code_to_events("a"), + ) + + completing_reader = functools.partial( + prepare_reader, + readline_completer=rlcompleter.Completer(namespace).complete, + ) + reader, _ = handle_all_events(events, prepare_reader=completing_reader) + + self.assert_screen_equal(reader, f"{code}a") + + def test_pos2xy_with_no_columns(self): + console = prepare_console([]) + reader = prepare_reader(console) + # Simulate a resize to 0 columns + reader.screeninfo = [] + self.assertEqual(reader.pos2xy(), (0, 0)) + + def test_setpos_from_xy_for_non_printing_char(self): + code = "# non \u200c printing character" + events = code_to_events(code) + + reader, _ = handle_all_events(events) + reader.setpos_from_xy(8, 0) + self.assertEqual(reader.pos, 7) + +@force_colorized_test_class +class TestReaderInColor(ScreenEqualMixin, TestCase): + def test_syntax_highlighting_basic(self): + code = dedent( + """\ + import re, sys + def funct(case: str = sys.platform) -> None: + match = re.search( + "(me)", + ''' + Come on + Come on now + You know that it's time to emerge + ''', + ) + match case: + case "emscripten": print("on the web") + case "ios" | "android": + print("on the phone") + case _: print('arms around', match.group(1)) + type type = type[type] + """ + ) + expected = dedent( + """\ + {k}import{z} re{o},{z} sys + {a}{k}def{z} {d}funct{z}{o}({z}case{o}:{z} {b}str{z} {o}={z} sys{o}.{z}platform{o}){z} {o}->{z} {k}None{z}{o}:{z} + match {o}={z} re{o}.{z}search{o}({z} + {s}"(me)"{z}{o},{z} + {s}'''{z} + {s} Come on{z} + {s} Come on now{z} + {s} You know that it's time to emerge{z} + {s} '''{z}{o},{z} + {o}){z} + {K}match{z} case{o}:{z} + {K}case{z} {s}"emscripten"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the web"{z}{o}){z} + {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} + {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} + {K}case{z} {K}_{z}{o}:{z} {b}print{z}{o}({z}{s}'arms around'{z}{o},{z} match{o}.{z}group{o}({z}{n}1{z}{o}){z}{o}){z} + {K}type{z} {b}type{z} {o}={z} {b}type{z}{o}[{z}{b}type{z}{o}]{z} + """ + ) + expected_sync = expected.format(a="", **colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected_sync) + self.assertEqual(reader.pos, 419) + self.assertEqual(reader.cxy, (0, 16)) + + async_msg = "{k}async{z} ".format(**colors) + expected_async = expected.format(a=async_msg, **colors) + more_events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 15, + code_to_events("async "), + ) + reader, _ = handle_all_events(more_events) + self.assert_screen_equal(reader, expected_async) + self.assertEqual(reader.pos, 21) + self.assertEqual(reader.cxy, (6, 1)) + + def test_syntax_highlighting_incomplete_string_first_line(self): + code = dedent( + """\ + def unfinished_function(arg: str = "still typing + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z}arg{o}:{z} {b}str{z} {o}={z} {s}"still typing{z} + """ + ).format(**colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + + def test_syntax_highlighting_incomplete_string_another_line(self): + code = dedent( + """\ + def unfinished_function( + arg: str = "still typing + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z} + arg{o}:{z} {b}str{z} {o}={z} {s}"still typing{z} + """ + ).format(**colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + + def test_syntax_highlighting_incomplete_multiline_string(self): + code = dedent( + """\ + def unfinished_function(): + '''Still writing + the docstring + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z} + {s}'''Still writing{z} + {s} the docstring{z} + """ + ).format(**colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + + def test_syntax_highlighting_incomplete_fstring(self): + code = dedent( + """\ + def unfinished_function(): + var = f"Single-quote but { + 1 + + + 1 + } multi-line! + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z} + var {o}={z} {s}f"{z}{s}Single-quote but {z}{o}{OB}{z} + {n}1{z} + {o}+{z} + {n}1{z} + {o}{CB}{z}{s} multi-line!{z} + """ + ).format(OB="{", CB="}", **colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + + def test_syntax_highlighting_indentation_error(self): + code = dedent( + """\ + def unfinished_function(): + var = 1 + oops + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z} + var {o}={z} {n}1{z} + oops + """ + ).format(**colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + + def test_syntax_highlighting_literal_brace_in_fstring_or_tstring(self): + code = dedent( + """\ + f"{{" + f"}}" + f"a{{b" + f"a}}b" + f"a{{b}}c" + t"a{{b}}c" + f"{{{0}}}" + f"{ {0} }" + """ + ) + expected = dedent( + """\ + {s}f"{z}{s}<<{z}{s}"{z} + {s}f"{z}{s}>>{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a>>{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}t"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}f"{z}{s}<<{z}{o}<{z}{n}0{z}{o}>{z}{s}>>{z}{s}"{z} + {s}f"{z}{o}<{z} {o}<{z}{n}0{z}{o}>{z} {o}>{z}{s}"{z} + """ + ).format(**colors).replace("<", "{").replace(">", "}") + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.maxDiff=None + self.assert_screen_equal(reader, expected) + + def test_control_characters(self): + code = 'flag = "🏳️‍🌈"' + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, 'flag = "🏳️\\u200d🌈"', clean=True) + self.assert_screen_equal(reader, 'flag {o}={z} {s}"🏳️\\u200d🌈"{z}'.format(**colors)) diff --git a/Lib/test/test_pyrepl/test_terminfo.py b/Lib/test/test_pyrepl/test_terminfo.py new file mode 100644 index 00000000000..562cf5c905b --- /dev/null +++ b/Lib/test/test_pyrepl/test_terminfo.py @@ -0,0 +1,651 @@ +"""Tests comparing PyREPL's pure Python curses implementation with the standard curses module.""" + +import json +import os +import subprocess +import sys +import unittest +from test.support import requires, has_subprocess_support +from textwrap import dedent + +# Only run these tests if curses is available +requires("curses") + +try: + import _curses +except ImportError: + try: + import curses as _curses + except ImportError: + _curses = None + +from _pyrepl import terminfo + + +ABSENT_STRING = terminfo.ABSENT_STRING +CANCELLED_STRING = terminfo.CANCELLED_STRING + + +class TestCursesCompatibility(unittest.TestCase): + """Test that PyREPL's curses implementation matches the standard curses behavior. + + Python's `curses` doesn't allow calling `setupterm()` again with a different + $TERM in the same process, so we subprocess all `curses` tests to get correctly + set up terminfo.""" + + @classmethod + def setUpClass(cls): + if _curses is None: + raise unittest.SkipTest( + "`curses` capability provided to regrtest but `_curses` not importable" + ) + + if not has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + # we need to ensure there's a terminfo database on the system and that + # `infocmp` works + cls.infocmp("dumb") + + def setUp(self): + self.original_term = os.environ.get("TERM", None) + + def tearDown(self): + if self.original_term is not None: + os.environ["TERM"] = self.original_term + elif "TERM" in os.environ: + del os.environ["TERM"] + + @classmethod + def infocmp(cls, term) -> list[str]: + all_caps = [] + try: + result = subprocess.run( + ["infocmp", "-l1", term], + capture_output=True, + text=True, + check=True, + ) + except Exception: + raise unittest.SkipTest("calling `infocmp` failed on the system") + + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("#"): + if "terminfo" not in line and "termcap" in line: + # PyREPL terminfo doesn't parse termcap databases + raise unittest.SkipTest( + "curses using termcap.db: no terminfo database on" + " the system" + ) + elif "=" in line: + cap_name = line.split("=")[0] + all_caps.append(cap_name) + + return all_caps + + def test_setupterm_basic(self): + """Test basic setupterm functionality.""" + # Test with explicit terminal type + test_terms = ["xterm", "xterm-256color", "vt100", "ansi"] + + for term in test_terms: + with self.subTest(term=term): + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(term)}, 1) + print(json.dumps({{"success": True}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + # Set up with PyREPL curses + try: + terminfo.TermInfo(term, fallback=False) + pyrepl_success = True + except Exception as e: + pyrepl_success = False + pyrepl_error = e + + # Both should succeed or both should fail + if std_success: + self.assertTrue( + pyrepl_success, + f"Standard curses succeeded but PyREPL failed for {term}", + ) + else: + # If standard curses failed, PyREPL might still succeed with fallback + # This is acceptable as PyREPL has hardcoded fallbacks + pass + + def test_setupterm_none(self): + """Test setupterm with None (uses TERM from environment).""" + # Test with current TERM + ncurses_code = dedent( + """ + import _curses + import json + try: + _curses.setupterm(None, 1) + print(json.dumps({"success": True})) + except Exception as e: + print(json.dumps({"success": False, "error": str(e)})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + try: + terminfo.TermInfo(None, fallback=False) + pyrepl_success = True + except Exception: + pyrepl_success = False + + # Both should have same result + if std_success: + self.assertTrue( + pyrepl_success, + "Standard curses succeeded but PyREPL failed for None", + ) + + def test_tigetstr_common_capabilities(self): + """Test tigetstr for common terminal capabilities.""" + # Test with a known terminal type + term = "xterm" + + # Get ALL capabilities from infocmp + all_caps = self.infocmp(term) + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + + ncurses_data = json.loads(result.stdout) + + ti = terminfo.TermInfo(term, fallback=False) + + # Test every single capability + for cap in all_caps: + if cap not in ncurses_data or ncurses_data[cap] == "error": + continue + + with self.subTest(capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap}: ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_tigetstr_input_types(self): + """Test tigetstr with different input types.""" + term = "xterm" + cap = "cup" + + # Test standard curses behavior with string in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with string input + try: + std_str_result = _curses.tigetstr({repr(cap)}) + std_accepts_str = True + if std_str_result is None: + std_str_val = None + elif std_str_result == -1: + std_str_val = -1 + else: + std_str_val = list(std_str_result) + except TypeError: + std_accepts_str = False + std_str_val = None + + print(json.dumps({{ + "accepts_str": std_accepts_str, + "str_result": std_str_val + }})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # PyREPL behavior with string + try: + pyrepl_str_result = ti.get(cap) + pyrepl_accepts_str = True + except TypeError: + pyrepl_accepts_str = False + + # PyREPL should also only accept strings for compatibility + with self.assertRaises(TypeError): + ti.get(cap.encode("ascii")) + + # Both should accept string input + self.assertEqual( + pyrepl_accepts_str, + ncurses_data["accepts_str"], + "PyREPL and standard curses should have same string handling", + ) + self.assertTrue( + pyrepl_accepts_str, "PyREPL should accept string input" + ) + + def test_tparm_basic(self): + """Test basic tparm functionality.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test cursor positioning (cup) + cup = ti.get("cup") + if cup and cup not in {ABSENT_STRING, CANCELLED_STRING}: + # Test various parameter combinations + test_cases = [ + (0, 0), # Top-left + (5, 10), # Arbitrary position + (23, 79), # Bottom-right of standard terminal + (999, 999), # Large values + ] + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Get cup capability + cup = _curses.tigetstr('cup') + results = {{}} + + for row, col in {repr(test_cases)}: + try: + result = _curses.tparm(cup, row, col) + results[f"{{row}},{{col}}"] = list(result) + except Exception as e: + results[f"{{row}},{{col}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for row, col in test_cases: + with self.subTest(row=row, col=col): + # Standard curses tparm from subprocess + key = f"{row},{col}" + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + # PyREPL curses tparm + pyrepl_result = terminfo.tparm(cup, row, col) + + # Results should be identical + self.assertEqual( + pyrepl_result, + std_result, + f"tparm(cup, {row}, {col}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + else: + raise unittest.SkipTest( + "test_tparm_basic() requires the `cup` capability" + ) + + def test_tparm_multiple_params(self): + """Test tparm with capabilities using multiple parameters.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test capabilities that take parameters + param_caps = { + "cub": 1, # cursor_left with count + "cuf": 1, # cursor_right with count + "cuu": 1, # cursor_up with count + "cud": 1, # cursor_down with count + "dch": 1, # delete_character with count + "ich": 1, # insert_character with count + } + + # Get all capabilities from PyREPL first + pyrepl_caps = {} + for cap in param_caps: + cap_value = ti.get(cap) + if cap_value and cap_value not in { + ABSENT_STRING, + CANCELLED_STRING, + }: + pyrepl_caps[cap] = cap_value + + if not pyrepl_caps: + self.skipTest("No parametrized capabilities found") + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + param_caps = {repr(param_caps)} + test_values = [1, 5, 10, 99] + results = {{}} + + for cap in param_caps: + cap_value = _curses.tigetstr(cap) + if cap_value and cap_value != -1: + for value in test_values: + try: + result = _curses.tparm(cap_value, value) + results[f"{{cap}},{{value}}"] = list(result) + except Exception as e: + results[f"{{cap}},{{value}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for cap, cap_value in pyrepl_caps.items(): + with self.subTest(capability=cap): + # Test with different parameter values + for value in [1, 5, 10, 99]: + key = f"{cap},{value}" + if key in ncurses_data: + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + pyrepl_result = terminfo.tparm(cap_value, value) + self.assertEqual( + pyrepl_result, + std_result, + f"tparm({cap}, {value}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + + def test_tparm_null_handling(self): + """Test tparm with None/null input.""" + term = "xterm" + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with None + try: + _curses.tparm(None) + raises_typeerror = False + except TypeError: + raises_typeerror = True + except Exception as e: + raises_typeerror = False + error_type = type(e).__name__ + + print(json.dumps({{"raises_typeerror": raises_typeerror}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # Test with None - both should raise TypeError + if ncurses_data["raises_typeerror"]: + with self.assertRaises(TypeError): + terminfo.tparm(None) + else: + # If ncurses doesn't raise TypeError, PyREPL shouldn't either + try: + terminfo.tparm(None) + except TypeError: + self.fail("PyREPL raised TypeError but ncurses did not") + + def test_special_terminals(self): + """Test with special terminal types.""" + special_terms = [ + "dumb", # Minimal terminal + "unknown", # Should fall back to defaults + "linux", # Linux console + "screen", # GNU Screen + "tmux", # tmux + ] + + # Get all string capabilities from ncurses + for term in special_terms: + with self.subTest(term=term): + all_caps = self.infocmp(term) + ncurses_code = dedent( + f""" + import _curses + import json + import sys + + try: + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + # Convert bytes to list of ints for JSON + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + except Exception as e: + print(json.dumps({{"error": str(e)}})) + """ + ) + + # Get ncurses results + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"Failed to get ncurses data for {term}: {result.stderr}" + ) + + try: + ncurses_data = json.loads(result.stdout) + except json.JSONDecodeError: + self.fail( + f"Failed to parse ncurses output for {term}: {result.stdout}" + ) + + if "error" in ncurses_data and len(ncurses_data) == 1: + # ncurses failed to setup this terminal + # PyREPL should still work with fallback + ti = terminfo.TermInfo(term, fallback=True) + continue + + ti = terminfo.TermInfo(term, fallback=False) + + # Compare all capabilities + for cap in all_caps: + if cap not in ncurses_data: + continue + + with self.subTest(term=term, capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + # Convert back to bytes + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + # Both should return the same value + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap} for {term}: " + f"ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_terminfo_fallback(self): + """Test that PyREPL falls back gracefully when terminfo is not found.""" + # Use a non-existent terminal type + fake_term = "nonexistent-terminal-type-12345" + + # Check if standard curses can setup this terminal in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(fake_term)}, 1) + print(json.dumps({{"success": True}})) + except _curses.error: + print(json.dumps({{"success": False, "error": "curses.error"}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + if ncurses_data["success"]: + # If it succeeded, skip this test as we can't test fallback + self.skipTest( + f"System unexpectedly has terminfo for '{fake_term}'" + ) + + # PyREPL should succeed with fallback + try: + ti = terminfo.TermInfo(fake_term, fallback=True) + pyrepl_ok = True + except Exception: + pyrepl_ok = False + + self.assertTrue( + pyrepl_ok, "PyREPL should fall back for unknown terminals" + ) + + # Should still be able to get basic capabilities + bel = ti.get("bel") + self.assertIsNotNone( + bel, "PyREPL should provide basic capabilities after fallback" + ) + + def test_invalid_terminal_names(self): + cases = [ + (42, TypeError), + ("", ValueError), + ("w\x00t", ValueError), + (f"..{os.sep}name", ValueError), + ] + + for term, exc in cases: + with self.subTest(term=term): + with self.assertRaises(exc): + terminfo._validate_terminal_name_or_raise(term) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py new file mode 100644 index 00000000000..a1ee6d4878f --- /dev/null +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -0,0 +1,369 @@ +import errno +import itertools +import os +import signal +import sys +import threading +import unittest +from functools import partial +from test.support import os_helper, force_not_colorized_test_class +from test.support import threading_helper + +from unittest import TestCase +from unittest.mock import MagicMock, call, patch, ANY, Mock + +from .support import handle_all_events, code_to_events + +try: + from _pyrepl.console import Event + from _pyrepl.unix_console import UnixConsole +except ImportError: + pass + +from _pyrepl.terminfo import _TERMINAL_CAPABILITIES + +TERM_CAPABILITIES = _TERMINAL_CAPABILITIES["ansi"] + + +def unix_console(events, **kwargs): + console = UnixConsole(term="xterm") + console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + console.wait = MagicMock() + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +handle_events_unix_console = partial( + handle_all_events, + prepare_console=unix_console, +) +handle_events_narrow_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, width=5), +) +handle_events_short_unix_console = partial( + handle_all_events, + prepare_console=partial(unix_console, height=1), +) +handle_events_unix_console_height_3 = partial( + handle_all_events, prepare_console=partial(unix_console, height=3) +) + + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch( + "_pyrepl.terminfo.tparm", + lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), +) +@patch( + "termios.tcgetattr", + lambda _: [ + 27394, + 3, + 19200, + 536872399, + 38400, + 38400, + [ + b"\x04", + b"\xff", + b"\xff", + b"\x7f", + b"\x17", + b"\x15", + b"\x12", + b"\x00", + b"\x03", + b"\x1c", + b"\x1a", + b"\x19", + b"\x11", + b"\x13", + b"\x16", + b"\x0f", + b"\x01", + b"\x00", + b"\x14", + b"\x00", + ], + ], +) +@patch("termios.tcsetattr", lambda a, b, c: None) +@patch("os.write") +@force_not_colorized_test_class +class TestConsole(TestCase): + def test_no_newline(self, _os_write): + code = "1" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + self.assertNotIn(call(ANY, b'\n'), _os_write.mock_calls) + con.restore() + + def test_newline(self, _os_write): + code = "\n" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"\n") + con.restore() + + def test_simple_addition(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_wrap(self, _os_write): + code = "12+34" + events = code_to_events(code) + _, con = handle_events_narrow_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, b"2") + _os_write.assert_any_call(ANY, b"+") + _os_write.assert_any_call(ANY, b"3") + _os_write.assert_any_call(ANY, b"\\") + _os_write.assert_any_call(ANY, b"\n") + _os_write.assert_any_call(ANY, b"4") + con.restore() + + def test_cursor_left(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + con.restore() + + def test_cursor_left_right(self, _os_write): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + con.restore() + + def test_cursor_up(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + con.restore() + + def test_cursor_up_down(self, _os_write): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + con.restore() + + def test_cursor_back_write(self, _os_write): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"1") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + _os_write.assert_any_call(ANY, b"2") + con.restore() + + def test_multiline_function_move_up_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = handle_events_short_unix_console(events) + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + con.restore() + + def test_resize_bigger_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_short_unix_console(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ri"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b"def f():"), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self, _os_write): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = handle_events_unix_console_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + _os_write.assert_has_calls( + [ + call(ANY, TERM_CAPABILITIES["ind"] + b":"), + call(ANY, TERM_CAPABILITIES["cup"] + b":0,0"), + call(ANY, b" foo"), + ] + ) + console.restore() + con.restore() + + def test_getheightwidth_with_invalid_environ(self, _os_write): + # gh-128636 + console = UnixConsole(term="xterm") + with os_helper.EnvironmentVarGuard() as env: + env["LINES"] = "" + self.assertIsInstance(console.getheightwidth(), tuple) + env["COLUMNS"] = "" + self.assertIsInstance(console.getheightwidth(), tuple) + os.environ = [] + self.assertIsInstance(console.getheightwidth(), tuple) + + @unittest.skipUnless(sys.platform == "darwin", "requires macOS") + def test_restore_with_invalid_environ_on_macos(self, _os_write): + # gh-128636 for macOS + console = UnixConsole(term="xterm") + with os_helper.EnvironmentVarGuard(): + os.environ = [] + console.prepare() # needed to call restore() + console.restore() # this should succeed + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_restore_in_thread(self, _os_write): + # gh-139391: ensure that console.restore() silently suppresses + # exceptions when calling signal.signal() from a non-main thread. + console = unix_console([]) + console.old_sigwinch = signal.SIG_DFL + thread = threading.Thread(target=console.restore) + thread.start() + thread.join() # this should not raise + + +@unittest.skipIf(sys.platform == "win32", "No Unix console on Windows") +class TestUnixConsoleEIOHandling(TestCase): + + @patch('_pyrepl.unix_console.tcsetattr') + @patch('_pyrepl.unix_console.tcgetattr') + def test_eio_error_handling_in_restore(self, mock_tcgetattr, mock_tcsetattr): + + import termios + mock_termios = Mock() + mock_termios.iflag = 0 + mock_termios.oflag = 0 + mock_termios.cflag = 0 + mock_termios.lflag = 0 + mock_termios.cc = [0] * 32 + mock_termios.copy.return_value = mock_termios + mock_tcgetattr.return_value = mock_termios + + console = UnixConsole(term="xterm") + console.prepare() + + mock_tcsetattr.side_effect = termios.error(errno.EIO, "Input/output error") + + # EIO error should be handled gracefully in restore() + console.restore() diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py new file mode 100644 index 00000000000..656a1e441e0 --- /dev/null +++ b/Lib/test/test_pyrepl/test_utils.py @@ -0,0 +1,105 @@ +from unittest import TestCase + +from _pyrepl.utils import str_width, wlen, prev_next_window, gen_colors + + +class TestUtils(TestCase): + def test_str_width(self): + characters = [ + 'a', + '1', + '_', + '!', + '\x1a', + '\u263A', + '\uffb9', + '\N{LATIN SMALL LETTER E WITH ACUTE}', # é + '\N{LATIN SMALL LETTER E WITH CEDILLA}', # ȩ + '\u00ad', + ] + for c in characters: + self.assertEqual(str_width(c), 1) + + zero_width_characters = [ + '\N{COMBINING ACUTE ACCENT}', + '\N{ZERO WIDTH JOINER}', + ] + for c in zero_width_characters: + with self.subTest(character=c): + self.assertEqual(str_width(c), 0) + + characters = [chr(99989), chr(99999)] + for c in characters: + self.assertEqual(str_width(c), 2) + + def test_wlen(self): + for c in ['a', 'b', '1', '!', '_']: + self.assertEqual(wlen(c), 1) + self.assertEqual(wlen('\x1a'), 2) + + char_east_asian_width_N = chr(3800) + self.assertEqual(wlen(char_east_asian_width_N), 1) + char_east_asian_width_W = chr(4352) + self.assertEqual(wlen(char_east_asian_width_W), 2) + + self.assertEqual(wlen('hello'), 5) + self.assertEqual(wlen('hello' + '\x1a'), 7) + self.assertEqual(wlen('e\N{COMBINING ACUTE ACCENT}'), 1) + self.assertEqual(wlen('a\N{ZERO WIDTH JOINER}b'), 2) + + def test_prev_next_window(self): + def gen_normal(): + yield 1 + yield 2 + yield 3 + yield 4 + + pnw = prev_next_window(gen_normal()) + self.assertEqual(next(pnw), (None, 1, 2)) + self.assertEqual(next(pnw), (1, 2, 3)) + self.assertEqual(next(pnw), (2, 3, 4)) + self.assertEqual(next(pnw), (3, 4, None)) + with self.assertRaises(StopIteration): + next(pnw) + + def gen_short(): + yield 1 + + pnw = prev_next_window(gen_short()) + self.assertEqual(next(pnw), (None, 1, None)) + with self.assertRaises(StopIteration): + next(pnw) + + def gen_raise(): + yield from gen_normal() + 1/0 + + pnw = prev_next_window(gen_raise()) + self.assertEqual(next(pnw), (None, 1, 2)) + self.assertEqual(next(pnw), (1, 2, 3)) + self.assertEqual(next(pnw), (2, 3, 4)) + self.assertEqual(next(pnw), (3, 4, None)) + with self.assertRaises(ZeroDivisionError): + next(pnw) + + def test_gen_colors_keyword_highlighting(self): + cases = [ + # no highlights + ("a.set", [(".", "op")]), + ("obj.list", [(".", "op")]), + ("obj.match", [(".", "op")]), + ("b. \\\n format", [(".", "op")]), + # highlights + ("set", [("set", "builtin")]), + ("list", [("list", "builtin")]), + (" \n dict", [("dict", "builtin")]), + ] + for code, expected_highlights in cases: + with self.subTest(code=code): + colors = list(gen_colors(code)) + # Extract (text, tag) pairs for comparison + actual_highlights = [] + for color in colors: + span_text = code[color.span.start:color.span.end + 1] + actual_highlights.append((span_text, color.tag)) + self.assertEqual(actual_highlights, expected_highlights) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py new file mode 100644 index 00000000000..3587b834f3c --- /dev/null +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -0,0 +1,621 @@ +import sys +import unittest + +if sys.platform != "win32": + raise unittest.SkipTest("test only relevant on win32") + + +import itertools +from functools import partial +from test.support import force_not_colorized_test_class +from typing import Iterable +from unittest import TestCase +from unittest.mock import MagicMock, call + +from .support import handle_all_events, code_to_events +from .support import prepare_reader as default_prepare_reader + +try: + from _pyrepl.console import Event, Console + from _pyrepl.windows_console import ( + WindowsConsole, + MOVE_LEFT, + MOVE_RIGHT, + MOVE_UP, + MOVE_DOWN, + ERASE_IN_LINE, + ) + import _pyrepl.windows_console as wc +except ImportError: + pass + + +@force_not_colorized_test_class +class WindowsConsoleTests(TestCase): + def console(self, events, **kwargs) -> Console: + console = WindowsConsole() + console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) + console.wait = MagicMock() + console._scroll = MagicMock() + console._hide_cursor = MagicMock() + console._show_cursor = MagicMock() + console._getscrollbacksize = MagicMock(42) + console.out = MagicMock() + + height = kwargs.get("height", 25) + width = kwargs.get("width", 80) + console.getheightwidth = MagicMock(side_effect=lambda: (height, width)) + + console.prepare() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + def handle_events( + self, + events: Iterable[Event], + prepare_console=None, + prepare_reader=None, + **kwargs, + ): + prepare_console = prepare_console or partial(self.console, **kwargs) + prepare_reader = prepare_reader or default_prepare_reader + return handle_all_events(events, prepare_console, prepare_reader) + + def handle_events_narrow(self, events): + return self.handle_events(events, width=5) + + def handle_events_short(self, events, **kwargs): + return self.handle_events(events, height=1, **kwargs) + + def handle_events_height_3(self, events): + return self.handle_events(events, height=3) + + def test_no_newline(self): + code = "1" + events = code_to_events(code) + _, con = self.handle_events(events) + self.assertNotIn(call(b'\n'), con.out.write.mock_calls) + con.restore() + + def test_newline(self): + code = "\n" + events = code_to_events(code) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"\n") + con.restore() + + def test_simple_addition(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_wrap(self): + code = "12+34" + events = code_to_events(code) + _, con = self.handle_events_narrow(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(b"2") + con.out.write.assert_any_call(b"+") + con.out.write.assert_any_call(b"3") + con.out.write.assert_any_call(b"\\") + con.out.write.assert_any_call(b"\n") + con.out.write.assert_any_call(b"4") + con.restore() + + def test_resize_wider(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events_narrow(events) + + console.height = 20 + console.width = 80 + console.getheightwidth = MagicMock(lambda _: (20, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(self.move_right(2)) + con.out.write.assert_any_call(self.move_up(2)) + con.out.write.assert_any_call(b"567890") + + con.restore() + + def test_resize_narrower(self): + code = "1234567890" + events = code_to_events(code) + reader, console = self.handle_events(events) + + console.height = 20 + console.width = 4 + console.getheightwidth = MagicMock(lambda _: (20, 4)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + + con.out.write.assert_any_call(b"456\\") + con.out.write.assert_any_call(b"789\\") + + con.restore() + + def test_cursor_left(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.restore() + + def test_cursor_left_right(self): + code = "1" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(self.move_right()) + con.restore() + + def test_cursor_up(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_cursor_up_down(self): + code = "1\n2+3" + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(self.move_up()) + con.out.write.assert_any_call(self.move_down()) + con.restore() + + def test_cursor_back_write(self): + events = itertools.chain( + code_to_events("1"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + code_to_events("2"), + ) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"1") + con.out.write.assert_any_call(self.move_left()) + con.out.write.assert_any_call(b"21") + con.restore() + + def test_multiline_function_move_up_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(5)) + con.out.write.assert_any_call(self.move_up()) + con.restore() + + def test_multiline_function_move_up_down_short_terminal(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="scroll", data=None), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="scroll", data=None), + ], + ) + _, con = self.handle_events_short(events) + con.out.write.assert_any_call(self.move_left(8)) + con.out.write.assert_any_call(self.erase_in_line()) + con.restore() + + def test_resize_bigger_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_short(events) + + console.height = 2 + console.getheightwidth = MagicMock(lambda _: (2, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(b"def f():"), + call(self.move_left(3)), + call(self.move_down()), + ] + ) + console.restore() + con.restore() + + def test_resize_smaller_on_multiline_function(self): + # fmt: off + code = ( + "def f():\n" + " foo" + ) + # fmt: on + + events = itertools.chain(code_to_events(code)) + reader, console = self.handle_events_height_3(events) + + console.height = 1 + console.getheightwidth = MagicMock(lambda _: (1, 80)) + + def same_reader(_): + return reader + + def same_console(events): + console.get_event = MagicMock(side_effect=events) + return console + + _, con = handle_all_events( + [Event(evt="resize", data=None)], + prepare_reader=same_reader, + prepare_console=same_console, + ) + con.out.write.assert_has_calls( + [ + call(self.move_left(5)), + call(self.move_up()), + call(self.erase_in_line()), + call(b" foo"), + ] + ) + console.restore() + con.restore() + + def move_up(self, lines=1): + return MOVE_UP.format(lines).encode("utf8") + + def move_down(self, lines=1): + return MOVE_DOWN.format(lines).encode("utf8") + + def move_left(self, cols=1): + return MOVE_LEFT.format(cols).encode("utf8") + + def move_right(self, cols=1): + return MOVE_RIGHT.format(cols).encode("utf8") + + def erase_in_line(self): + return ERASE_IN_LINE.encode("utf8") + + def test_multiline_ctrl_z(self): + # see gh-126332 + code = "abcdefghi" + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data='\x1a', raw=bytearray(b'\x1a')), + Event(evt="key", data='\x1a', raw=bytearray(b'\x1a')), + ], + ) + reader, con = self.handle_events_narrow(events) + self.assertEqual(reader.cxy, (2, 3)) + con.restore() + + +class WindowsConsoleGetEventTests(TestCase): + # Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes + VK_BACK = 0x08 + VK_RETURN = 0x0D + VK_LEFT = 0x25 + VK_7 = 0x37 + VK_M = 0x4D + # Used for miscellaneous characters; it can vary by keyboard. + # For the US standard keyboard, the '" key. + # For the German keyboard, the Ä key. + VK_OEM_7 = 0xDE + + # State of control keys: https://learn.microsoft.com/en-us/windows/console/key-event-record-str + RIGHT_ALT_PRESSED = 0x0001 + RIGHT_CTRL_PRESSED = 0x0004 + LEFT_ALT_PRESSED = 0x0002 + LEFT_CTRL_PRESSED = 0x0008 + ENHANCED_KEY = 0x0100 + SHIFT_PRESSED = 0x0010 + + + def get_event(self, input_records, **kwargs) -> Console: + self.console = WindowsConsole(encoding='utf-8') + self.mock = MagicMock(side_effect=input_records) + self.console._read_input = self.mock + self.console._WindowsConsole__vt_support = kwargs.get("vt_support", + False) + self.console.wait = MagicMock(return_value=True) + event = self.console.get_event(block=False) + return event + + def get_input_record(self, unicode_char, vcode=0, control=0): + return wc.INPUT_RECORD( + wc.KEY_EVENT, + wc.ConsoleEvent(KeyEvent= + wc.KeyEvent( + bKeyDown=True, + wRepeatCount=1, + wVirtualKeyCode=vcode, + wVirtualScanCode=0, # not used + uChar=wc.Char(unicode_char), + dwControlKeyState=control + ))) + + def test_EmptyBuffer(self): + self.assertEqual(self.get_event([None]), None) + self.assertEqual(self.mock.call_count, 1) + + def test_WINDOW_BUFFER_SIZE_EVENT(self): + ir = wc.INPUT_RECORD( + wc.WINDOW_BUFFER_SIZE_EVENT, + wc.ConsoleEvent(WindowsBufferSizeEvent= + wc.WindowsBufferSizeEvent( + wc._COORD(0, 0)))) + self.assertEqual(self.get_event([ir]), Event("resize", "")) + self.assertEqual(self.mock.call_count, 1) + + def test_KEY_EVENT_up_ignored(self): + ir = wc.INPUT_RECORD( + wc.KEY_EVENT, + wc.ConsoleEvent(KeyEvent= + wc.KeyEvent(bKeyDown=False))) + self.assertEqual(self.get_event([ir]), None) + self.assertEqual(self.mock.call_count, 1) + + def test_unhandled_events(self): + for event in (wc.FOCUS_EVENT, wc.MENU_EVENT, wc.MOUSE_EVENT): + ir = wc.INPUT_RECORD( + event, + # fake data, nothing is read except bKeyDown + wc.ConsoleEvent(KeyEvent= + wc.KeyEvent(bKeyDown=False))) + self.assertEqual(self.get_event([ir]), None) + self.assertEqual(self.mock.call_count, 1) + + def test_enter(self): + ir = self.get_input_record("\r", self.VK_RETURN) + self.assertEqual(self.get_event([ir]), Event("key", "\n")) + self.assertEqual(self.mock.call_count, 1) + + def test_backspace(self): + ir = self.get_input_record("\x08", self.VK_BACK) + self.assertEqual( + self.get_event([ir]), Event("key", "backspace")) + self.assertEqual(self.mock.call_count, 1) + + def test_m(self): + ir = self.get_input_record("m", self.VK_M) + self.assertEqual(self.get_event([ir]), Event("key", "m")) + self.assertEqual(self.mock.call_count, 1) + + def test_M(self): + ir = self.get_input_record("M", self.VK_M, self.SHIFT_PRESSED) + self.assertEqual(self.get_event([ir]), Event("key", "M")) + self.assertEqual(self.mock.call_count, 1) + + def test_left(self): + # VK_LEFT is sent as ENHANCED_KEY + ir = self.get_input_record("\x00", self.VK_LEFT, self.ENHANCED_KEY) + self.assertEqual(self.get_event([ir]), Event("key", "left")) + self.assertEqual(self.mock.call_count, 1) + + def test_left_RIGHT_CTRL_PRESSED(self): + ir = self.get_input_record( + "\x00", self.VK_LEFT, self.RIGHT_CTRL_PRESSED | self.ENHANCED_KEY) + self.assertEqual( + self.get_event([ir]), Event("key", "ctrl left")) + self.assertEqual(self.mock.call_count, 1) + + def test_left_LEFT_CTRL_PRESSED(self): + ir = self.get_input_record( + "\x00", self.VK_LEFT, self.LEFT_CTRL_PRESSED | self.ENHANCED_KEY) + self.assertEqual( + self.get_event([ir]), Event("key", "ctrl left")) + self.assertEqual(self.mock.call_count, 1) + + def test_left_RIGHT_ALT_PRESSED(self): + ir = self.get_input_record( + "\x00", self.VK_LEFT, self.RIGHT_ALT_PRESSED | self.ENHANCED_KEY) + self.assertEqual(self.get_event([ir]), Event(evt="key", data="\033")) + self.assertEqual( + self.console.get_event(), Event("key", "left")) + # self.mock is not called again, since the second time we read from the + # command queue + self.assertEqual(self.mock.call_count, 1) + + def test_left_LEFT_ALT_PRESSED(self): + ir = self.get_input_record( + "\x00", self.VK_LEFT, self.LEFT_ALT_PRESSED | self.ENHANCED_KEY) + self.assertEqual(self.get_event([ir]), Event(evt="key", data="\033")) + self.assertEqual( + self.console.get_event(), Event("key", "left")) + self.assertEqual(self.mock.call_count, 1) + + def test_m_LEFT_ALT_PRESSED_and_LEFT_CTRL_PRESSED(self): + # For the shift keys, Windows does not send anything when + # ALT and CTRL are both pressed, so let's test with VK_M. + # get_event() receives this input, but does not + # generate an event. + # This is for e.g. an English keyboard layout, for a + # German layout this returns `µ`, see test_AltGr_m. + ir = self.get_input_record( + "\x00", self.VK_M, self.LEFT_ALT_PRESSED | self.LEFT_CTRL_PRESSED) + self.assertEqual(self.get_event([ir]), None) + self.assertEqual(self.mock.call_count, 1) + + def test_m_LEFT_ALT_PRESSED(self): + ir = self.get_input_record( + "m", vcode=self.VK_M, control=self.LEFT_ALT_PRESSED) + self.assertEqual(self.get_event([ir]), Event(evt="key", data="\033")) + self.assertEqual(self.console.get_event(), Event("key", "m")) + self.assertEqual(self.mock.call_count, 1) + + def test_m_RIGHT_ALT_PRESSED(self): + ir = self.get_input_record( + "m", vcode=self.VK_M, control=self.RIGHT_ALT_PRESSED) + self.assertEqual(self.get_event([ir]), Event(evt="key", data="\033")) + self.assertEqual(self.console.get_event(), Event("key", "m")) + self.assertEqual(self.mock.call_count, 1) + + def test_AltGr_7(self): + # E.g. on a German keyboard layout, '{' is entered via + # AltGr + 7, where AltGr is the right Alt key on the keyboard. + # In this case, Windows automatically sets + # RIGHT_ALT_PRESSED = 0x0001 + LEFT_CTRL_PRESSED = 0x0008 + # This can also be entered like + # LeftAlt + LeftCtrl + 7 or + # LeftAlt + RightCtrl + 7 + # See https://learn.microsoft.com/en-us/windows/console/key-event-record-str + # https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-vkkeyscanw + ir = self.get_input_record( + "{", vcode=self.VK_7, + control=self.RIGHT_ALT_PRESSED | self.LEFT_CTRL_PRESSED) + self.assertEqual(self.get_event([ir]), Event("key", "{")) + self.assertEqual(self.mock.call_count, 1) + + def test_AltGr_m(self): + # E.g. on a German keyboard layout, this yields 'µ' + # Let's use LEFT_ALT_PRESSED and RIGHT_CTRL_PRESSED this + # time, to cover that, too. See above in test_AltGr_7. + ir = self.get_input_record( + "µ", vcode=self.VK_M, control=self.LEFT_ALT_PRESSED | self.RIGHT_CTRL_PRESSED) + self.assertEqual(self.get_event([ir]), Event("key", "µ")) + self.assertEqual(self.mock.call_count, 1) + + def test_umlaut_a_german(self): + ir = self.get_input_record("ä", self.VK_OEM_7) + self.assertEqual(self.get_event([ir]), Event("key", "ä")) + self.assertEqual(self.mock.call_count, 1) + + # virtual terminal tests + # Note: wVirtualKeyCode, wVirtualScanCode and dwControlKeyState + # are always zero in this case. + # "\r" and backspace are handled specially, everything else + # is handled in "elif self.__vt_support:" in WindowsConsole.get_event(). + # Hence, only one regular key ("m") and a terminal sequence + # are sufficient to test here, the real tests happen in test_eventqueue + # and test_keymap. + + def test_enter_vt(self): + ir = self.get_input_record("\r") + self.assertEqual(self.get_event([ir], vt_support=True), + Event("key", "\n")) + self.assertEqual(self.mock.call_count, 1) + + def test_backspace_vt(self): + ir = self.get_input_record("\x7f") + self.assertEqual(self.get_event([ir], vt_support=True), + Event("key", "backspace", b"\x7f")) + self.assertEqual(self.mock.call_count, 1) + + def test_up_vt(self): + irs = [self.get_input_record(x) for x in "\x1b[A"] + self.assertEqual(self.get_event(irs, vt_support=True), + Event(evt='key', data='up', raw=bytearray(b'\x1b[A'))) + self.assertEqual(self.mock.call_count, 3) + + # All tests above assume that there is always keyboard data to read, + # because for simplicity we just use + # self.console.wait = MagicMock(return_value=True) + def test_wait_empty(self): + console = WindowsConsole(encoding='utf-8') + console.wait_for_event = MagicMock(return_value=True) + self.assertTrue(console.event_queue.empty()) + timeout = 2.0 + self.assertTrue(console.wait(timeout)) + self.assertEqual(console.wait_for_event.call_count, 1) + self.assertEqual(console.wait_for_event.mock_calls[0], call(timeout)) + + timeout = 1.1 + console.wait_for_event = MagicMock(return_value=False) + self.assertFalse(console.wait(timeout)) + self.assertEqual(console.wait_for_event.call_count, 1) + self.assertEqual(console.wait_for_event.mock_calls[0], call(timeout)) + + def test_wait_not_empty(self): + console = WindowsConsole(encoding='utf-8') + console.wait_for_event = MagicMock(return_value=True) + console.event_queue.push(b"a") + self.assertFalse(console.event_queue.empty()) + self.assertTrue(console.wait(0.0)) + self.assertEqual(console.wait_for_event.call_count, 0) + + +if __name__ == "__main__": + unittest.main() From e4921934e9329835c9f2e3253a3f97fe5cf6543b Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 10 May 2026 13:44:09 +0300 Subject: [PATCH 3/7] Patches for test suite to run --- Lib/test/test_pyrepl/test_unix_console.py | 68 +++++++++++++++++++---- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index a1ee6d4878f..8f84d183c23 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -57,15 +57,12 @@ def unix_console(events, **kwargs): handle_all_events, prepare_console=partial(unix_console, height=3) ) +def __rustpython_patch_terminfo_tparm(s, *args): + return s + b":" + b",".join(str(i).encode() for i in args) -@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") -@patch( - "_pyrepl.terminfo.tparm", - lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), -) -@patch( - "termios.tcgetattr", - lambda _: [ + +def __rustpython_patch_termios_tcgetattr(_): + return [ 27394, 3, 19200, @@ -94,9 +91,60 @@ def unix_console(events, **kwargs): b"\x14", b"\x00", ], - ], + ] + + +def __rustpython_patch_termios_tcsetattr(a, b, c): + return None + +@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") +@patch( + "_pyrepl.terminfo.tparm", + # TODO: RUSTPYTHON; SyntaxError: the symbol 's' must be present in the symbol table + # lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), + __rustpython_patch_terminfo_tparm, +) +# TODO: RUSTPYTHON; SyntaxError: no symbol table available in lambda (type: Lambda) +# @patch( +# "termios.tcgetattr", +# lambda _: [ +# 27394, +# 3, +# 19200, +# 536872399, +# 38400, +# 38400, +# [ +# b"\x04", +# b"\xff", +# b"\xff", +# b"\x7f", +# b"\x17", +# b"\x15", +# b"\x12", +# b"\x00", +# b"\x03", +# b"\x1c", +# b"\x1a", +# b"\x19", +# b"\x11", +# b"\x13", +# b"\x16", +# b"\x0f", +# b"\x01", +# b"\x00", +# b"\x14", +# b"\x00", +# ], +# ], +# ) +@patch("termios.tcgetattr", __rustpython_patch_termios_tcgetattr) +@patch( + "termios.tcsetattr", + # TODO: RUSTPYTHON; SyntaxError: no symbol table available in lambda (type: Lambda) + # lambda a, b, c: None + __rustpython_patch_termios_tcsetattr ) -@patch("termios.tcsetattr", lambda a, b, c: None) @patch("os.write") @force_not_colorized_test_class class TestConsole(TestCase): From 3fd6004a27b4cf187eaff30d3ae8f08906e7bbec Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 10 May 2026 13:55:30 +0300 Subject: [PATCH 4/7] Add test markers --- Lib/test/test_pyrepl/test_interact.py | 9 +++++++++ Lib/test/test_pyrepl/test_pyrepl.py | 16 ++++++++++++++++ Lib/test/test_pyrepl/test_reader.py | 3 +++ 3 files changed, 28 insertions(+) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index f0837ee94e9..e4f90db3304 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -102,6 +102,7 @@ def test_runsource_returns_false_for_failed_compilation(self): self.assertFalse(result) self.assertIn('SyntaxError', f.getvalue()) + @unittest.expectedFailure # TODO: RUSTPYTHON @force_not_colorized def test_runsource_show_syntax_error_location(self): console = InteractiveColoredConsole() @@ -116,6 +117,7 @@ def f(x, x): ... SyntaxError: duplicate argument 'x' in function definition""" self.assertIn(r, f.getvalue()) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_runsource_shows_syntax_error_for_failed_compilation(self): console = InteractiveColoredConsole() source = "print('Hello, world!'" @@ -131,6 +133,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): console.runsource(source) mock_showsyntaxerror.assert_called_once() + @unittest.expectedFailure # TODO: RUSTPYTHON def test_runsource_survives_null_bytes(self): console = InteractiveColoredConsole() source = "\x00\n" @@ -152,6 +155,7 @@ def test_no_active_future(self): self.assertFalse(result) self.assertEqual(f.getvalue(), "{'x': }\n") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_future_annotations(self): console = InteractiveColoredConsole() source = dedent("""\ @@ -165,6 +169,7 @@ def g(x: int): ... self.assertFalse(result) self.assertEqual(f.getvalue(), "{'x': 'int'}\n") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_future_barry_as_flufl(self): console = InteractiveColoredConsole() f = io.StringIO() @@ -205,6 +210,7 @@ def test_multiline_single_assignment(self): console = InteractiveColoredConsole(namespace, filename="") self.assertFalse(_more_lines(console, code)) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiline_single_block(self): namespace = {} code = dedent("""\ @@ -221,6 +227,7 @@ def test_multiple_statements_single_line(self): console = InteractiveColoredConsole(namespace, filename="") self.assertFalse(_more_lines(console, code)) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_statements(self): namespace = {} code = dedent("""\ @@ -230,6 +237,7 @@ def test_multiple_statements(self): console = InteractiveColoredConsole(namespace, filename="") self.assertTrue(_more_lines(console, code)) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_blocks(self): namespace = {} code = dedent("""\ @@ -277,6 +285,7 @@ def test_incomplete_statement(self): class TestWarnings(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pep_765_warning(self): """ Test that a SyntaxWarning emitted from the diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 60561e5663f..c85f12db687 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -18,6 +18,8 @@ from test.support.import_helper import import_module from test.support.os_helper import EnvironmentVarGuard, unlink +import unittest # XXX RUSTPYTHON + from .support import ( FakeConsole, ScreenEqualMixin, @@ -464,6 +466,7 @@ def prepare_reader(self, events): reader = ReadlineAlikeReader(console=console, config=config) return reader + @unittest.expectedFailure # TODO: RUSTPYTHON def test_auto_indent_default(self): # fmt: off input_code = ( @@ -483,6 +486,7 @@ def test_auto_indent_default(self): output = multiline_input(reader) self.assertEqual(output, output_code) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_auto_indent_continuation(self): # auto indenting according to previous user indentation # fmt: off @@ -510,6 +514,7 @@ def test_auto_indent_continuation(self): output = multiline_input(reader) self.assertEqual(output, output_code) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_auto_indent_prev_block(self): # auto indenting according to indentation in different block # fmt: off @@ -541,6 +546,7 @@ def test_auto_indent_prev_block(self): output2 = multiline_input(reader) self.assertEqual(output2, output_code) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_auto_indent_multiline(self): # fmt: off events = itertools.chain( @@ -580,6 +586,7 @@ def test_auto_indent_multiline(self): output = multiline_input(reader) self.assertEqual(output, output_code) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_auto_indent_with_comment(self): # fmt: off events = code_to_events( @@ -598,6 +605,7 @@ def test_auto_indent_with_comment(self): output = multiline_input(reader) self.assertEqual(output, output_code) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_auto_indent_with_multicomment(self): # fmt: off events = code_to_events( @@ -672,6 +680,7 @@ def test_get_line_buffer_returns_str(self): wrapper = _ReadlineWrapper(f_in=None, f_out=None, reader=reader) self.assertIs(type(wrapper.get_line_buffer()), str) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiline_edit(self): events = itertools.chain( code_to_events("def f():\n...\n\n"), @@ -735,6 +744,7 @@ def test_history_navigation_with_up_arrow(self): self.assertEqual(output, "1+1") self.assert_screen_equal(reader, "1+1", clean=True) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_history_with_multiline_entries(self): code = "def foo():\nx = 1\ny = 2\nz = 3\n\ndef bar():\nreturn 42\n\n" events = list(itertools.chain( @@ -856,6 +866,7 @@ def prepare_reader(self, events, namespace): reader = ReadlineAlikeReader(console=console, config=config) return reader + @unittest.expectedFailure # TODO: RUSTPYTHON @patch("rlcompleter._readline_available", False) def test_simple_completion(self): events = code_to_events("os.getpid\t\n") @@ -1037,6 +1048,7 @@ def test_builtin_completion_top_level(self): output = reader.readline() self.assertEqual(output, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_relative_import_completions(self): cases = ( (None, "from .readl\t\n", "from .readl"), @@ -1109,6 +1121,7 @@ def test_hardcoded_stdlib_submodules(self): output = reader.readline() self.assertEqual(output, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self): with (tempfile.TemporaryDirectory() as _dir, patch.object(sys, "modules", {})): # hide imported module @@ -1233,6 +1246,7 @@ def test_get_path_and_prefix(self): with self.subTest(name=name): self.assertEqual(completer.get_path_and_prefix(name), expected) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse(self): cases = ( ('import ', (None, '')), @@ -1412,6 +1426,7 @@ def test_paste_mid_newlines(self): output = multiline_input(reader) self.assertEqual(output, code) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_paste_mid_newlines_not_in_paste_mode(self): # fmt: off code = ( @@ -1433,6 +1448,7 @@ def test_paste_mid_newlines_not_in_paste_mode(self): output = multiline_input(reader) self.assertEqual(output, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_paste_not_in_paste_mode(self): # fmt: off input_code = ( diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index b1b6ae16a1e..12af361e0d9 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -13,6 +13,7 @@ from _pyrepl.reader import Reader from _colorize import default_theme +import unittest # # XXX: RUSTPYTHON overrides = {"reset": "z", "soft_keyword": "K"} colors = {overrides.get(k, k[0].lower()): v for k, v in default_theme.syntax.items()} @@ -178,6 +179,7 @@ def test_up_arrow_after_ctrl_r(self): reader, _ = handle_all_events(events) self.assert_screen_equal(reader, "") + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Lists differ def test_newline_within_block_trailing_whitespace(self): # fmt: off code = ( @@ -521,6 +523,7 @@ def unfinished_function(): self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Lists differ def test_syntax_highlighting_literal_brace_in_fstring_or_tstring(self): code = dedent( """\ From 65cca86d79f60141b612eabb7f323e9ee8fbaedb Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 10 May 2026 14:05:51 +0300 Subject: [PATCH 5/7] Add a dummy `posix._is_inputhook_installed` function --- crates/vm/src/stdlib/posix.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 95abf264fbf..b633495574a 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -2820,4 +2820,10 @@ mod posix_sched { Ok(ret) } } + + #[pyfunction] + fn _is_inputhook_installed() -> bool { + // TODO: Implement the actual logic here + false + } } From a4bb6571d265ab28e3b52ff8f0c32764c6a500b1 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 10 May 2026 14:30:58 +0300 Subject: [PATCH 6/7] Add a dummy `nt._is_inputhook_installed` function --- crates/vm/src/stdlib/nt.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs index 554012987aa..07482e0d5d2 100644 --- a/crates/vm/src/stdlib/nt.rs +++ b/crates/vm/src/stdlib/nt.rs @@ -2239,4 +2239,10 @@ pub(crate) mod module { super::super::os::module_exec(vm, module)?; Ok(()) } + + #[pyfunction] + fn _is_inputhook_installed() -> bool { + // TODO: Implement the actual logic here + false + } } From 88515c61ad4b63f9a68c5169ec14d0bed5278e82 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sun, 10 May 2026 14:34:35 +0300 Subject: [PATCH 7/7] Mark some failing tests --- Lib/test/test_pyrepl/test_windows_console.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index 3587b834f3c..0af3b894453 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -448,6 +448,7 @@ def test_unhandled_events(self): self.assertEqual(self.get_event([ir]), None) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_enter(self): ir = self.get_input_record("\r", self.VK_RETURN) self.assertEqual(self.get_event([ir]), Event("key", "\n")) @@ -459,11 +460,13 @@ def test_backspace(self): self.get_event([ir]), Event("key", "backspace")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_m(self): ir = self.get_input_record("m", self.VK_M) self.assertEqual(self.get_event([ir]), Event("key", "m")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_M(self): ir = self.get_input_record("M", self.VK_M, self.SHIFT_PRESSED) self.assertEqual(self.get_event([ir]), Event("key", "M")) @@ -519,6 +522,7 @@ def test_m_LEFT_ALT_PRESSED_and_LEFT_CTRL_PRESSED(self): self.assertEqual(self.get_event([ir]), None) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_m_LEFT_ALT_PRESSED(self): ir = self.get_input_record( "m", vcode=self.VK_M, control=self.LEFT_ALT_PRESSED) @@ -526,6 +530,7 @@ def test_m_LEFT_ALT_PRESSED(self): self.assertEqual(self.console.get_event(), Event("key", "m")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_m_RIGHT_ALT_PRESSED(self): ir = self.get_input_record( "m", vcode=self.VK_M, control=self.RIGHT_ALT_PRESSED) @@ -533,6 +538,7 @@ def test_m_RIGHT_ALT_PRESSED(self): self.assertEqual(self.console.get_event(), Event("key", "m")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_AltGr_7(self): # E.g. on a German keyboard layout, '{' is entered via # AltGr + 7, where AltGr is the right Alt key on the keyboard. @@ -549,6 +555,7 @@ def test_AltGr_7(self): self.assertEqual(self.get_event([ir]), Event("key", "{")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_AltGr_m(self): # E.g. on a German keyboard layout, this yields 'µ' # Let's use LEFT_ALT_PRESSED and RIGHT_CTRL_PRESSED this @@ -558,6 +565,7 @@ def test_AltGr_m(self): self.assertEqual(self.get_event([ir]), Event("key", "µ")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_umlaut_a_german(self): ir = self.get_input_record("ä", self.VK_OEM_7) self.assertEqual(self.get_event([ir]), Event("key", "ä")) @@ -572,18 +580,21 @@ def test_umlaut_a_german(self): # are sufficient to test here, the real tests happen in test_eventqueue # and test_keymap. + @unittest.expectedFailure # TODO: RUSTPYTHON def test_enter_vt(self): ir = self.get_input_record("\r") self.assertEqual(self.get_event([ir], vt_support=True), Event("key", "\n")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_backspace_vt(self): ir = self.get_input_record("\x7f") self.assertEqual(self.get_event([ir], vt_support=True), Event("key", "backspace", b"\x7f")) self.assertEqual(self.mock.call_count, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_up_vt(self): irs = [self.get_input_record(x) for x in "\x1b[A"] self.assertEqual(self.get_event(irs, vt_support=True),