diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 1c8741b5a55..916bb25d74d 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -239,11 +239,11 @@ def visit_NamedExpr(self, node): self.traverse(node.value) def visit_Import(self, node): - self.fill("import ") + self.fill("lazy import " if node.is_lazy else "import ") self.interleave(lambda: self.write(", "), self.traverse, node.names) def visit_ImportFrom(self, node): - self.fill("from ") + self.fill("lazy from " if node.is_lazy else "from ") self.write("." * (node.level or 0)) if node.module: self.write(node.module) @@ -738,9 +738,13 @@ def visit_SetComp(self, node): def visit_DictComp(self, node): with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) + if node.value: + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + else: + self.write("**") + self.traverse(node.key) for gen in node.generators: self.traverse(gen) diff --git a/Lib/_colorize.py b/Lib/_colorize.py index d6673f6692f..27eb7f13bac 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,21 +1,19 @@ +import builtins import os import sys from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field, Field +lazy from typing import IO, Literal, Self, ClassVar COLORIZE = True - -# types -if False: - from typing import IO, Self, ClassVar - _theme: Theme +_theme: Theme +type BackgroundStyle = Literal["dark", "light"] class ANSIColors: RESET = "\x1b[0m" - BLACK = "\x1b[30m" BLUE = "\x1b[34m" CYAN = "\x1b[36m" @@ -74,6 +72,19 @@ class ANSIColors: setattr(NoColors, attr, "") +class CursesColors: + """Curses color constants for terminal UI theming.""" + BLACK = 0 + RED = 1 + GREEN = 2 + YELLOW = 3 + BLUE = 4 + MAGENTA = 5 + CYAN = 6 + WHITE = 7 + DEFAULT = -1 + + # # Experimental theming support (see gh-133346) # @@ -168,10 +179,227 @@ class Argparse(ThemeSection): short_option: str = ANSIColors.BOLD_GREEN label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN + default: str = ANSIColors.GREY + interpolated_value: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + error: str = ANSIColors.BOLD_MAGENTA + warning: str = ANSIColors.BOLD_YELLOW + message: str = ANSIColors.MAGENTA + + +@dataclass(frozen=True, kw_only=True) +class Ast(ThemeSection): + node: str = ANSIColors.CYAN + field: str = ANSIColors.BLUE + attribute: str = ANSIColors.GREY + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Calendar(ThemeSection): + header: str = ANSIColors.BOLD + highlight: str = ANSIColors.BLACK + ANSIColors.BACKGROUND_YELLOW + weekday: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Difflib(ThemeSection): + """A 'git diff'-like theme for `difflib.unified_diff`.""" + added: str = ANSIColors.GREEN + context: str = ANSIColors.RESET # context lines + header: str = ANSIColors.BOLD # eg "---" and "+++" lines + hunk: str = ANSIColors.CYAN # the "@@" lines + removed: str = ANSIColors.RED + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class FancyCompleter(ThemeSection): + # functions and methods + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN + + # numbers + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW + + # others + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN + + +@dataclass(frozen=True, kw_only=True) +class HttpServer(ThemeSection): + error: str = ANSIColors.YELLOW + path: str = ANSIColors.CYAN + serving: str = ANSIColors.GREEN + size: str = ANSIColors.GREY + status_informational: str = ANSIColors.RESET + status_ok: str = ANSIColors.GREEN + status_redirect: str = ANSIColors.INTENSE_CYAN + status_client_error: str = ANSIColors.YELLOW + status_server_error: str = ANSIColors.RED + timestamp: str = ANSIColors.GREY + url: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class LiveProfiler(ThemeSection): + """Theme section for the live profiling TUI (Tachyon profiler). + + Colors use CursesColors constants (BLACK, RED, GREEN, YELLOW, + BLUE, MAGENTA, CYAN, WHITE, DEFAULT). + """ + # Header colors + title_fg: int = CursesColors.CYAN + title_bg: int = CursesColors.DEFAULT + + # Status display colors + pid_fg: int = CursesColors.CYAN + uptime_fg: int = CursesColors.GREEN + time_fg: int = CursesColors.YELLOW + interval_fg: int = CursesColors.MAGENTA + + # Thread view colors + thread_all_fg: int = CursesColors.GREEN + thread_single_fg: int = CursesColors.MAGENTA + + # Progress bar colors + bar_good_fg: int = CursesColors.GREEN + bar_bad_fg: int = CursesColors.RED + + # Stats colors + on_gil_fg: int = CursesColors.GREEN + off_gil_fg: int = CursesColors.RED + waiting_gil_fg: int = CursesColors.YELLOW + gc_fg: int = CursesColors.MAGENTA + + # Function display colors + func_total_fg: int = CursesColors.CYAN + func_exec_fg: int = CursesColors.GREEN + func_stack_fg: int = CursesColors.YELLOW + func_shown_fg: int = CursesColors.MAGENTA + + # Table header colors (for sorted column highlight) + sorted_header_fg: int = CursesColors.BLACK + sorted_header_bg: int = CursesColors.CYAN + + # Normal header colors (non-sorted columns) - use reverse video style + normal_header_fg: int = CursesColors.BLACK + normal_header_bg: int = CursesColors.WHITE + + # Data row colors + samples_fg: int = CursesColors.CYAN + file_fg: int = CursesColors.GREEN + func_fg: int = CursesColors.YELLOW + + # Trend indicator colors + trend_up_fg: int = CursesColors.GREEN + trend_down_fg: int = CursesColors.RED + + # Medal colors for top functions + medal_gold_fg: int = CursesColors.RED + medal_silver_fg: int = CursesColors.YELLOW + medal_bronze_fg: int = CursesColors.GREEN + + # Background style: 'dark' or 'light' + background_style: BackgroundStyle = "dark" + + +LiveProfilerLight = LiveProfiler( + # Header colors + title_fg=CursesColors.BLUE, # Blue is more readable than cyan on light bg + + # Status display colors - darker colors for light backgrounds + pid_fg=CursesColors.BLUE, + uptime_fg=CursesColors.BLACK, + time_fg=CursesColors.BLACK, + interval_fg=CursesColors.BLUE, + + # Thread view colors + thread_all_fg=CursesColors.BLACK, + thread_single_fg=CursesColors.BLUE, + + # Stats colors + waiting_gil_fg=CursesColors.RED, + gc_fg=CursesColors.BLUE, + + # Function display colors + func_total_fg=CursesColors.BLUE, + func_exec_fg=CursesColors.BLACK, + func_stack_fg=CursesColors.BLACK, + func_shown_fg=CursesColors.BLUE, + + # Table header colors (for sorted column highlight) + sorted_header_fg=CursesColors.WHITE, + sorted_header_bg=CursesColors.BLUE, + + # Normal header colors (non-sorted columns) + normal_header_fg=CursesColors.WHITE, + normal_header_bg=CursesColors.BLACK, + + # Data row colors - use dark colors readable on white + samples_fg=CursesColors.BLACK, + file_fg=CursesColors.BLACK, + func_fg=CursesColors.BLUE, # Blue is more readable than magenta on light bg + + # Medal colors for top functions + medal_silver_fg=CursesColors.BLUE, + + # Background style + background_style="light", +) + + +@dataclass(frozen=True, kw_only=True) +class ProfilerDump(ThemeSection): + header: str = ANSIColors.BOLD_BLUE + interpreter: str = ANSIColors.GREY + thread: str = ANSIColors.BOLD_CYAN + status: str = ANSIColors.YELLOW + frame_index: str = ANSIColors.GREY + frame: str = ANSIColors.BOLD_GREEN + filename: str = ANSIColors.CYAN + line_no: str = ANSIColors.YELLOW + source: str = ANSIColors.WHITE + source_highlight: str = ANSIColors.BOLD_YELLOW + opcode: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Pickletools(ThemeSection): + annotation: str = ANSIColors.GREY + arg_number: str = ANSIColors.YELLOW + arg_string: str = ANSIColors.GREEN + mark: str = ANSIColors.GREY + op_call: str = ANSIColors.GREEN + op_container: str = ANSIColors.INTENSE_BLUE + op_memo: str = ANSIColors.MAGENTA + op_meta: str = ANSIColors.GREY + op_stack: str = ANSIColors.BOLD_RED + opcode_code: str = ANSIColors.CYAN + position: str = ANSIColors.GREY + proto: str = ANSIColors.YELLOW reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE @@ -186,10 +414,31 @@ class Syntax(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) +class Timeit(ThemeSection): + timing: str = ANSIColors.CYAN + best: str = ANSIColors.BOLD_GREEN + per_loop: str = ANSIColors.GREEN + punctuation: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + warning_worst: str = ANSIColors.MAGENTA + warning_best: str = ANSIColors.GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Tokenize(ThemeSection): + whitespace: str = ANSIColors.GREY + error: str = ANSIColors.BOLD_RED + position: str = ANSIColors.GREY + delimiter: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA + note: str = ANSIColors.CYAN filename: str = ANSIColors.MAGENTA line_no: str = ANSIColors.MAGENTA frame: str = ANSIColors.MAGENTA @@ -198,7 +447,7 @@ class Traceback(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Unittest(ThemeSection): passed: str = ANSIColors.GREEN warn: str = ANSIColors.YELLOW @@ -207,7 +456,7 @@ class Unittest(ThemeSection): reset: str = ANSIColors.RESET -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Theme: """A suite of themes for all sections of Python. @@ -215,7 +464,17 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + ast: Ast = field(default_factory=Ast) + calendar: Calendar = field(default_factory=Calendar) + difflib: Difflib = field(default_factory=Difflib) + fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) + http_server: HttpServer = field(default_factory=HttpServer) + live_profiler: LiveProfiler = field(default_factory=LiveProfiler) + pickletools: Pickletools = field(default_factory=Pickletools) + profiler_dump: ProfilerDump = field(default_factory=ProfilerDump) syntax: Syntax = field(default_factory=Syntax) + timeit: Timeit = field(default_factory=Timeit) + tokenize: Tokenize = field(default_factory=Tokenize) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -223,7 +482,17 @@ def copy_with( self, *, argparse: Argparse | None = None, + ast: Ast | None = None, + calendar: Calendar | None = None, + difflib: Difflib | None = None, + fancycompleter: FancyCompleter | None = None, + http_server: HttpServer | None = None, + live_profiler: LiveProfiler | None = None, + pickletools: Pickletools | None = None, + profiler_dump: ProfilerDump | None = None, syntax: Syntax | None = None, + timeit: Timeit | None = None, + tokenize: Tokenize | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, ) -> Self: @@ -234,7 +503,17 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + ast=ast or self.ast, + calendar=calendar or self.calendar, + difflib=difflib or self.difflib, + fancycompleter=fancycompleter or self.fancycompleter, + http_server=http_server or self.http_server, + live_profiler=live_profiler or self.live_profiler, + pickletools=pickletools or self.pickletools, + profiler_dump=profiler_dump or self.profiler_dump, syntax=syntax or self.syntax, + timeit=timeit or self.timeit, + tokenize=tokenize or self.tokenize, traceback=traceback or self.traceback, unittest=unittest or self.unittest, ) @@ -249,7 +528,17 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + ast=Ast.no_colors(), + calendar=Calendar.no_colors(), + difflib=Difflib.no_colors(), + fancycompleter=FancyCompleter.no_colors(), + http_server=HttpServer.no_colors(), + live_profiler=LiveProfiler.no_colors(), + pickletools=Pickletools.no_colors(), + profiler_dump=ProfilerDump.no_colors(), syntax=Syntax.no_colors(), + timeit=Timeit.no_colors(), + tokenize=Tokenize.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), ) @@ -318,6 +607,9 @@ def _safe_getenv(k: str, fallback: str | None = None) -> str | None: default_theme = Theme() theme_no_color = default_theme.no_colors() +# Convenience theme with light profiler colors (for white/light terminal backgrounds) +light_profiler_theme = default_theme.copy_with(live_profiler=LiveProfilerLight) + def get_theme( *, diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index 60793c391ae..64db87d4fff 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -183,7 +183,6 @@ 'SimpleDialog': 'tkinter.simpledialog', 'DocXMLRPCServer': 'xmlrpc.server', 'SimpleHTTPServer': 'http.server', - 'CGIHTTPServer': 'http.server', # For compatibility with broken pickles saved in old Python 3 versions 'UserDict': 'collections', 'UserList': 'collections', @@ -225,8 +224,6 @@ ('DocXMLRPCServer', 'DocCGIXMLRPCRequestHandler'), ('http.server', 'SimpleHTTPRequestHandler'): ('SimpleHTTPServer', 'SimpleHTTPRequestHandler'), - ('http.server', 'CGIHTTPRequestHandler'): - ('CGIHTTPServer', 'CGIHTTPRequestHandler'), ('_socket', 'socket'): ('socket', '_socketobject'), }) @@ -251,6 +248,7 @@ REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') PYTHON3_IMPORTERROR_EXCEPTIONS = ( + 'ImportCycleError', 'ModuleNotFoundError', ) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 5c9a0812646..9ae1e3a06e9 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -191,12 +191,27 @@ def evaluate( arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): + resolved = _sentinel if arg in locals: - return locals[arg] + resolved = locals[arg] elif arg in globals: - return globals[arg] + resolved = globals[arg] elif hasattr(builtins, arg): - return getattr(builtins, arg) + resolved = getattr(builtins, arg) + + if resolved is not _sentinel: + if isinstance(resolved, types.LazyImportType): + # We try reifying the lazy object. If this fails, we propagate + # the error in the VALUE format and leave the + # ForwardRef unresolved in the FORWARDREF format. + try: + return resolved.resolve() + except Exception: + if not is_forwardref_format: + raise + return self + else: + return resolved elif is_forwardref_format: return self else: diff --git a/Lib/ast.py b/Lib/ast.py index 2f11683ecf7..4f88a554344 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -24,7 +24,7 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +44,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): @@ -57,73 +58,93 @@ def literal_eval(node_or_string): Caution: A complex expression can overflow the C stack and cause a crash. """ if isinstance(node_or_string, str): - node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval') - if isinstance(node_or_string, Expression): + node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval').body + elif isinstance(node_or_string, Expression): node_or_string = node_or_string.body - def _raise_malformed_node(node): - msg = "malformed node or string" - if lno := getattr(node, 'lineno', None): - msg += f' on line {lno}' - raise ValueError(msg + f': {node!r}') - def _convert_num(node): - if not isinstance(node, Constant) or type(node.value) not in (int, float, complex): - _raise_malformed_node(node) + return _convert_literal(node_or_string) + + +def _convert_literal(node): + """ + Used by `literal_eval` to convert an AST node into a value. + """ + if isinstance(node, Constant): return node.value - def _convert_signed_num(node): - if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): - operand = _convert_num(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - return _convert_num(node) - def _convert(node): - if isinstance(node, Constant): - return node.value - elif isinstance(node, Tuple): - return tuple(map(_convert, node.elts)) - elif isinstance(node, List): - return list(map(_convert, node.elts)) - elif isinstance(node, Set): - return set(map(_convert, node.elts)) - elif (isinstance(node, Call) and isinstance(node.func, Name) and - node.func.id == 'set' and node.args == node.keywords == []): - return set() - elif isinstance(node, Dict): - if len(node.keys) != len(node.values): - _raise_malformed_node(node) - return dict(zip(map(_convert, node.keys), - map(_convert, node.values))) - elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): - left = _convert_signed_num(node.left) - right = _convert_num(node.right) - if isinstance(left, (int, float)) and isinstance(right, complex): - if isinstance(node.op, Add): - return left + right - else: - return left - right - return _convert_signed_num(node) - return _convert(node_or_string) + if isinstance(node, Dict) and len(node.keys) == len(node.values): + return dict(zip( + map(_convert_literal, node.keys), + map(_convert_literal, node.values), + )) + if isinstance(node, Tuple): + return tuple(map(_convert_literal, node.elts)) + if isinstance(node, List): + return list(map(_convert_literal, node.elts)) + if isinstance(node, Set): + return set(map(_convert_literal, node.elts)) + if ( + isinstance(node, Call) and isinstance(node.func, Name) + and node.func.id == 'set' and node.args == node.keywords == [] + ): + return set() + if ( + isinstance(node, UnaryOp) + and isinstance(node.op, (UAdd, USub)) + and isinstance(node.operand, Constant) + and type(operand := node.operand.value) in (int, float, complex) + ): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + if ( + isinstance(node, BinOp) + and isinstance(node.op, (Add, Sub)) + and isinstance(node.left, (Constant, UnaryOp)) + and isinstance(node.right, Constant) + and type(left := _convert_literal(node.left)) in (int, float) + and type(right := _convert_literal(node.right)) is complex + ): + if isinstance(node.op, Add): + return left + right + else: + return left - right + msg = "malformed node or string" + if lno := getattr(node, 'lineno', None): + msg += f' on line {lno}' + raise ValueError(msg + f': {node!r}') def dump( node, annotate_fields=True, include_attributes=False, *, - indent=None, show_empty=False, + color=False, indent=None, show_empty=False, ): """ Return a formatted dump of the tree in node. This is mainly useful for - debugging purposes. If annotate_fields is true (by default), - the returned string will show the names and the values for fields. - If annotate_fields is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - include_attributes can be set to true. If indent is a non-negative - integer or string, then the tree will be pretty-printed with that indent - level. None (the default) selects the single line representation. + debugging purposes. + + If annotate_fields is true (by default), the returned string will show the + names and the values for fields. If annotate_fields is false, the result + string will be more compact by omitting unambiguous field names. + + Attributes such as line numbers and column offsets are not dumped by default. + If this is wanted, include_attributes can be set to true. + + If color is true, the returned string is syntax highlighted using ANSI + escape sequences. If color is false (the default), colored output is always + disabled. + + If indent is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. If indent is None (the default), + the tree is dumped on a single line. + If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + from _colorize import get_theme + + t = get_theme(force_color=color, force_no_color=not color).ast + def _format(node, level=0): if indent is not None: level += 1 @@ -154,13 +175,21 @@ def _format(node, level=0): if not keywords: args_buffer.append(repr(value)) continue + elif isinstance(value, Load): + field_type = cls._field_types.get(name, object) + if field_type is expr_context: + if not keywords: + args_buffer.append( + f'{t.node}{type(value).__name__}' + f'{t.reset}()') + continue if not keywords: args.extend(args_buffer) args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append('%s=%s' % (name, value)) + args.append(f'{t.field}{name}{t.reset}={value}') else: args.append(value) if include_attributes and node._attributes: @@ -173,14 +202,21 @@ def _format(node, level=0): continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append('%s=%s' % (name, value)) + args.append(f'{t.attribute}{name}{t.reset}={value}') + cls_name = f'{t.node}{cls.__name__}{t.reset}' if allsimple and len(args) <= 3: - return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args - return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False + return f'{cls_name}({", ".join(args)})', not args + return f'{cls_name}({prefix}{sep.join(args)})', False elif isinstance(node, list): if not node: return '[]', True return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False + if isinstance(node, bool) or node is None or node is Ellipsis: + return f'{t.keyword}{node!r}{t.reset}', True + if isinstance(node, (int, float, complex)): + return f'{t.number}{node!r}{t.reset}', True + if isinstance(node, (str, bytes)): + return f'{t.string}{node!r}{t.reset}', True return repr(node), True if not isinstance(node, AST): @@ -628,9 +664,9 @@ def main(args=None): import argparse import sys - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', - help='the file to parse; defaults to stdin') + help='the file to parse; defaults to `stdin`') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), help='specify what kind of code must be parsed') @@ -643,11 +679,11 @@ def main(args=None): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='Python version in the format 3.x ' - '(for example, 3.10)') + help='Python version in the format `3.x` ' + '(for example, `3.10`)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', - help='optimization level for parser (default -1)') + help='optimization level for parser') parser.add_argument('--show-empty', default=False, action='store_true', help='show empty lists and fields in dump output') args = parser.parse_args(args) @@ -673,7 +709,9 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) + from _colorize import can_colorize print(dump(tree, include_attributes=args.include_attributes, + color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) if __name__ == '__main__': diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index f2649aa2d41..98a5e950602 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -14,6 +14,7 @@ BaseException ├── EOFError ├── ExceptionGroup [BaseExceptionGroup] ├── ImportError + │ └── ImportCycleError │ └── ModuleNotFoundError ├── LookupError │ ├── IndexError diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 5087c3ca425..a7d3bca7402 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -654,6 +654,13 @@ def f(x: 'int') -> 'str': ... result = get_annotations(f, eval_str=True) self.assertEqual(result, {'x': int, 'return': str}) + def test_eval_str_wrapped_partial_cycle_self(self): + def f(x: 'int') -> 'str': ... + f.__wrapped__ = functools.partial(f, 0) + # Cycle is detected and broken; globals from f itself are used. + result = get_annotations(f, eval_str=True) + self.assertEqual(result, {'x': int, 'return': str}) + def test_eval_str_wrapped_cycle_mutual(self): # gh-146556: mutual __wrapped__ cycle (a -> b -> a) must not hang. def a(x: 'int'): ... @@ -1862,7 +1869,7 @@ def nested(): self.assertEqual(type_repr(t'''{ 0 & 1 | 2 - }'''), 't"""{ 0\n & 1\n | 2}"""') + }'''), 't"""{ 0\n & 1\n | 2\n }"""') self.assertEqual( type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" ) @@ -1874,6 +1881,10 @@ def nested(): type_repr(Template("hi", Interpolation(42, " "))), "Template('hi', Interpolation(42, ' ', None, ''))", ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42, "4!2"))), + "Template('hi', Interpolation(42, '4!2', None, ''))", + ) # gh138558: perhaps in the future, we can improve this behavior: self.assertEqual(type_repr(Template(Interpolation(42, "99"))), "t'{99}'") @@ -2133,8 +2144,13 @@ def f(a: unknown | str | int | list[str] | tuple[int, ...]): ... self.assertIsNone(fr.__resolved_str_cache__) self.assertEqual(fr.evaluate(format=Format.STRING), "unknown | str | int | list[str] | tuple[int, ...]") + + # Test that the cache is now set correctly self.assertEqual(fr.__resolved_str_cache__, "unknown | str | int | list[str] | tuple[int, ...]") + # Test that future evaluations return the same cache + self.assertIs(fr.evaluate(format=Format.STRING), fr.__resolved_str_cache__) + def test_evaluate_forwardref_format(self): fr = ForwardRef("undef") evaluated = fr.evaluate(format=Format.FORWARDREF) @@ -2163,6 +2179,118 @@ def test_evaluate_forwardref_format(self): support.EqualToForwardRef('"a" + 1'), ) + def test_evaluate_lazy_import(self): + ns = {} + exec( + textwrap.dedent( + """ + lazy from test.test_lazy_import.data.basic2 import x + lazy from test.test_lazy_import.data.broken_module import y + + class A: + a: x + + class B: + b: y + """ + ), + ns, + ) + self.addCleanup( + import_helper.unload, "test.test_lazy_import.data.basic2" + ) + self.addCleanup( + import_helper.unload, "test.test_lazy_import.data.broken_module" + ) + self.assertIs(type(ns["x"]), types.LazyImportType) + self.assertIs(type(ns["y"]), types.LazyImportType) + + # The lazy import resolves successfully: + for format in (Format.VALUE, Format.FORWARDREF): + with self.subTest(format=format): + self.assertEqual( + ForwardRef("x").evaluate(globals=ns, format=format), 42 + ) + self.assertEqual( + ForwardRef("x").evaluate(locals=ns, format=format), 42 + ) + self.assertEqual( + get_annotations(ns["A"], format=Format.FORWARDREF), {"a": 42} + ) + + # The lazy import fails to resolve: + fr = ForwardRef("y") + with self.assertRaisesRegex(ValueError, "always fails to import"): + fr.evaluate(globals=ns, format=Format.VALUE) + with self.assertRaisesRegex(ValueError, "always fails to import"): + fr.evaluate(locals=ns, format=Format.VALUE) + self.assertIs(fr.evaluate(globals=ns, format=Format.FORWARDREF), fr) + self.assertIs(fr.evaluate(locals=ns, format=Format.FORWARDREF), fr) + + annos = get_annotations(ns["B"], format=Format.FORWARDREF) + self.assertEqual( + annos, + {"b": support.EqualToForwardRef("y", is_class=True, owner=ns["B"])}, + ) + with self.assertRaisesRegex(ValueError, "always fails to import"): + annos["b"].evaluate(format=Format.VALUE) + with self.assertRaisesRegex(ValueError, "always fails to import"): + get_annotations(ns["B"], format=Format.VALUE) + + def test_get_annotations_lazy_import(self): + ns = {} + exec( + textwrap.dedent( + """ + lazy from test.test_lazy_import.data.basic2 import x + lazy from test.test_lazy_import.data.broken_module import y + + class A: + a: object.fail + b: x + + class B: + a: object.fail + b: y + """ + ), + ns, + ) + self.addCleanup( + import_helper.unload, "test.test_lazy_import.data.basic2" + ) + self.addCleanup( + import_helper.unload, "test.test_lazy_import.data.broken_module" + ) + self.assertIs(type(ns["x"]), types.LazyImportType) + self.assertIs(type(ns["y"]), types.LazyImportType) + + annos = get_annotations(ns["A"], format=Format.FORWARDREF) + self.assertEqual( + annos, + { + "a": support.EqualToForwardRef( + "object.fail", is_class=True, owner=ns["A"] + ), + "b": 42, + }, + ) + + annos = get_annotations(ns["B"], format=Format.FORWARDREF) + self.assertEqual( + annos, + { + "a": support.EqualToForwardRef( + "object.fail", is_class=True, owner=ns["B"] + ), + "b": support.EqualToForwardRef( + "y", is_class=True, owner=ns["B"] + ), + }, + ) + with self.assertRaisesRegex(ValueError, "always fails to import"): + annos["b"].evaluate(format=Format.VALUE) + def test_evaluate_notimplemented_format(self): class C: x: alias diff --git a/Lib/test/test_ast/data/ast_repr.txt b/Lib/test/test_ast/data/ast_repr.txt index 1c1985519cd..cc6accd766b 100644 --- a/Lib/test/test_ast/data/ast_repr.txt +++ b/Lib/test/test_ast/data/ast_repr.txt @@ -69,10 +69,14 @@ Module(body=[Try(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='ex Module(body=[TryStar(body=[Pass()], handlers=[ExceptHandler(type=Name(...), name='exc', body=[Pass(...)])], orelse=[Pass()], finalbody=[Pass()])], type_ignores=[]) Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=None)], type_ignores=[]) Module(body=[Assert(test=Name(id='v', ctx=Load(...)), msg=Constant(value='message', kind=None))], type_ignores=[]) -Module(body=[Import(names=[alias(name='sys', asname=None)])], type_ignores=[]) -Module(body=[Import(names=[alias(name='foo', asname='bar')])], type_ignores=[]) -Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0)], type_ignores=[]) -Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0)], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)], is_lazy=0)], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')], is_lazy=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0, is_lazy=0)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0, is_lazy=0)], type_ignores=[]) +Module(body=[Import(names=[alias(name='sys', asname=None)], is_lazy=1)], type_ignores=[]) +Module(body=[Import(names=[alias(name='foo', asname='bar')], is_lazy=1)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='x', asname='y')], level=0, is_lazy=1)], type_ignores=[]) +Module(body=[ImportFrom(module='sys', names=[alias(name='v', asname=None)], level=0, is_lazy=1)], type_ignores=[]) Module(body=[Global(names=['v'])], type_ignores=[]) Module(body=[Expr(value=Constant(value=1, kind=None))], type_ignores=[]) Module(body=[Pass()], type_ignores=[]) diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py index b76f98901d2..a565ed10f8b 100644 --- a/Lib/test/test_ast/snippets.py +++ b/Lib/test/test_ast/snippets.py @@ -118,6 +118,12 @@ # ImportFrom "from sys import x as y", "from sys import v", + # Lazy Import + "lazy import sys", + "lazy import foo as bar", + # Lazy ImportFrom + "lazy from sys import x as y", + "lazy from sys import v", # Global "global v", # Expr @@ -460,10 +466,14 @@ def main(): ('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), ('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), ('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), -('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), +('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)], 0)], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')], 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0, 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0, 0)], []), +('Module', [('Import', (1, 0, 1, 15), [('alias', (1, 12, 1, 15), 'sys', None)], 1)], []), +('Module', [('Import', (1, 0, 1, 22), [('alias', (1, 12, 1, 22), 'foo', 'bar')], 1)], []), +('Module', [('ImportFrom', (1, 0, 1, 27), 'sys', [('alias', (1, 21, 1, 27), 'x', 'y')], 0, 1)], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 21, 1, 22), 'v', None)], 0, 1)], []), ('Module', [('Global', (1, 0, 1, 8), ['v'])], []), ('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), ('Module', [('Pass', (1, 0, 1, 4))], []), diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index b006e513fe4..68255d17932 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1,4 +1,5 @@ import _ast_unparse +import _ast import ast import builtins import contextlib @@ -13,6 +14,7 @@ import textwrap import types import unittest +import warnings import weakref from io import StringIO from pathlib import Path @@ -86,7 +88,9 @@ def _assertTrueorder(self, ast_node, parent_pos): self.assertEqual(ast_node._fields, ast_node.__match_args__) def test_AST_objects(self): - x = ast.AST() + # Directly instantiating abstract node class AST is allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.AST() self.assertEqual(x._fields, ()) x.foobar = 42 self.assertEqual(x.foobar, 42) @@ -95,7 +99,7 @@ def test_AST_objects(self): with self.assertRaises(AttributeError): x.vararg - with self.assertRaises(TypeError): + with self.assertRaises(TypeError), self.assertWarns(DeprecationWarning): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) @@ -111,15 +115,21 @@ def cleanup(): msg = "type object 'ast.AST' has no attribute '_fields'" # Both examples used to crash: - with self.assertRaisesRegex(AttributeError, msg): + with ( + self.assertRaisesRegex(AttributeError, msg), + self.assertWarns(DeprecationWarning), + ): ast.AST(arg1=123) - with self.assertRaisesRegex(AttributeError, msg): + with ( + self.assertRaisesRegex(AttributeError, msg), + self.assertWarns(DeprecationWarning), + ): ast.AST() - def test_AST_garbage_collection(self): + def test_node_garbage_collection(self): class X: pass - a = ast.AST() + a = ast.Module() a.x = X() a.x.a = a ref = weakref.ref(a.x) @@ -133,7 +143,7 @@ def test_snippets(self): (eval_tests, eval_results, "eval")): for i, o in zip(input, output): with self.subTest(action="parsing", input=i): - ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) + ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST, optimize=False) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) with self.subTest(action="compiling", input=i, kind=kind): @@ -143,7 +153,7 @@ def test_ast_validation(self): # compile() is the only function that calls PyAST_Validate snippets_to_validate = exec_tests + single_tests + eval_tests for snippet in snippets_to_validate: - tree = ast.parse(snippet) + tree = ast.parse(snippet, optimize=False) compile(tree, '', 'exec') def test_parse_invalid_ast(self): @@ -152,6 +162,15 @@ def test_parse_invalid_ast(self): self.assertRaises(TypeError, ast.parse, ast.Constant(42), optimize=optval) + def test_parse_ast_func_type(self): + # see gh-156689 + tree = ast.parse('(int, str) -> bool', mode='func_type') + self.assertEqual(ast.dump(ast.parse(tree, mode='func_type')), + ast.dump(tree)) + self.assertRaises(TypeError, ast.parse, ast.Constant(42), + mode='func_type') + self.assertRaises(TypeError, ast.parse, tree, mode='exec') + def test_optimization_levels__debug__(self): cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] for (optval, expected) in cases: @@ -419,6 +438,17 @@ def test_field_attr_existence(self): if isinstance(x, ast.AST): self.assertIs(type(x._fields), tuple) + def test_dynamic_attr(self): + for name, item in ast.__dict__.items(): + # constructor has a different signature + if name == 'Index': + continue + if self._is_ast_node(name, item): + x = self._construct_ast_class(item) + # Custom attribute assignment is allowed + x.foo = 5 + self.assertEqual(x.foo, 5) + def _construct_ast_class(self, cls): kwargs = {} for name, typ in cls.__annotations__.items(): @@ -429,7 +459,15 @@ def _construct_ast_class(self, cls): elif typ is object: kwargs[name] = b'capybara' elif isinstance(typ, type) and issubclass(typ, ast.AST): - kwargs[name] = self._construct_ast_class(typ) + if _ast._is_abstract(typ): + # Use an arbitrary concrete subclass + concrete = next((sub for sub in typ.__subclasses__() + if not _ast._is_abstract(sub)), None) + msg = f"abstract node class {typ} has no concrete subclasses" + self.assertIsNotNone(concrete, msg) + else: + concrete = typ + kwargs[name] = self._construct_ast_class(concrete) return cls(**kwargs) def test_arguments(self): @@ -568,6 +606,10 @@ def test_nodeclasses(self): with self.assertRaisesRegex(TypeError, re.escape(msg)): ast.BinOp(1, 2, 3, foobarbaz=42) + # Directly instantiating abstract node types is allowed (but deprecated) + self.assertWarns(DeprecationWarning, ast.stmt) + self.assertWarns(DeprecationWarning, ast.expr_context) + def test_no_fields(self): # this used to fail because Sub._fields was None x = ast.Sub() @@ -575,7 +617,10 @@ def test_no_fields(self): def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) + with self.assertWarns(DeprecationWarning): + # Creating instances of ast.expr is deprecated + e = ast.expr(**pos) + m = ast.Module([ast.Expr(e, **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") self.assertIn("but got expr()", str(cm.exception)) @@ -942,6 +987,34 @@ def test_constant_as_name(self): with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): compile(expr, "", "eval") + def test_constant_in_identifier_fields(self): + # gh-85260: an identifier field holding a constant name used to + # crash the compiler + for statement in [ + "def x(): pass", + "async def x(): pass", + "class x: pass", + "from a import x", + "from a import b as x", + "from a import b, c, d as x", + "import x", + "import a, b, x", + "try: pass\nexcept A as x: pass", + "try: pass\nexcept A as b: pass\nexcept B as x: pass\n", + ]: + for constant in "True", "False", "None": + with self.subTest(statement=statement, constant=constant): + tree = ast.parse(statement) + for node in ast.walk(tree): + for field, value in ast.iter_fields(node): + if value == "x": + setattr(node, field, constant) + with self.assertRaisesRegex( + ValueError, + f"identifier field can't represent " + f"'{constant}' constant"): + compile(tree, "", "exec") + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised def test_constant_as_unicode_name(self): constants = [ @@ -985,8 +1058,7 @@ def next(self): enum._test_simple_enum(_Precedence, _ast_unparse._Precedence) @support.cpython_only - @support.run_with_limited_c_stack( - 100_000 if sys.platform == "android" else 500_000) + @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000) @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): @@ -1048,7 +1120,7 @@ def test_repr(self) -> None: snapshots = AST_REPR_DATA_FILE.read_text().split("\n") for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): with self.subTest(test_input=test): - self.assertEqual(repr(ast.parse(test)), snapshot) + self.assertEqual(repr(ast.parse(test, optimize=False)), snapshot) def test_repr_large_input_crash(self): # gh-125010: Fix use-after-free in ast repr() @@ -1069,13 +1141,28 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_optimization_levels_const_folding(self): - return super().test_optimization_levels_const_folding() - - @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 - def test_constant_subclasses_deprecated(self): - return super().test_constant_subclasses_deprecated() + def test_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + ast.parse(source) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, '') + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=r'') + ast.parse(source, filename, module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) class CopyTests(unittest.TestCase): @@ -1085,14 +1172,19 @@ class CopyTests(unittest.TestCase): def iter_ast_classes(): """Iterate over the (native) subclasses of ast.AST recursively. - This excludes the special class ast.Index since its constructor - returns an integer. + This excludes: + * abstract AST nodes + * the special class ast.Index, since its constructor returns + an integer. """ def do(cls): if cls.__module__ != 'ast': return if cls is ast.Index: return + # Don't attempt to create instances of abstract AST nodes + if _ast._is_abstract(cls): + return yield cls for sub in cls.__subclasses__(): @@ -1359,14 +1451,14 @@ def test_replace_ignore_known_custom_instance_fields(self): self.assertRaises(AttributeError, getattr, repl, 'extra') def test_replace_reject_missing_field(self): - # case: warn if deleted field is not replaced + # case: raise if deleted field is not replaced node = ast.parse('x').body[0].value context = node.ctx del node.id self.assertRaises(AttributeError, getattr, node, 'id') self.assertIs(node.ctx, context) - msg = "Name.__replace__ missing 1 keyword argument: 'id'." + msg = "ast.Name.__init__ missing 1 required positional argument: 'id'" with self.assertRaisesRegex(TypeError, re.escape(msg)): copy.replace(node) # assert that there is no side-effect @@ -1403,7 +1495,7 @@ def test_replace_reject_known_custom_instance_fields_commits(self): # explicit rejection of known instance fields self.assertHasAttr(node, 'extra') - msg = "Name.__replace__ got an unexpected keyword argument 'extra'." + msg = "ast.Name.__init__ got an unexpected keyword argument 'extra'" with self.assertRaisesRegex(TypeError, re.escape(msg)): copy.replace(node, extra=1) # assert that there is no side-effect @@ -1417,7 +1509,7 @@ def test_replace_reject_unknown_instance_fields(self): # explicit rejection of unknown extra fields self.assertRaises(AttributeError, getattr, node, 'unknown') - msg = "Name.__replace__ got an unexpected keyword argument 'unknown'." + msg = "ast.Name.__init__ got an unexpected keyword argument 'unknown'" with self.assertRaisesRegex(TypeError, re.escape(msg)): copy.replace(node, unknown=1) # assert that there is no side-effect @@ -1452,17 +1544,17 @@ def test_parse_in_error(self): def test_dump(self): node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])" + "Module(body=[Expr(value=Call(func=Name(id='spam'), " + "args=[Name(id='eggs'), Constant(value='and cheese')]))])" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " + "Module([Expr(Call(Name('spam'), [Name('eggs'), " "Constant('and cheese')]))])" ) self.assertEqual(ast.dump(node, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "Module(body=[Expr(value=Call(func=Name(id='spam', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " - "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " + "args=[Name(id='eggs', lineno=1, col_offset=5, " "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " @@ -1476,18 +1568,18 @@ def test_dump_indent(self): body=[ Expr( value=Call( - func=Name(id='spam', ctx=Load()), + func=Name(id='spam'), args=[ - Name(id='eggs', ctx=Load()), + Name(id='eggs'), Constant(value='and cheese')]))])""") self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ Module( \t[ \t\tExpr( \t\t\tCall( -\t\t\t\tName('spam', Load()), +\t\t\t\tName('spam'), \t\t\t\t[ -\t\t\t\t\tName('eggs', Load()), +\t\t\t\t\tName('eggs'), \t\t\t\t\tConstant('and cheese')]))])""") self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ Module( @@ -1496,7 +1588,6 @@ def test_dump_indent(self): value=Call( func=Name( id='spam', - ctx=Load(), lineno=1, col_offset=0, end_lineno=1, @@ -1504,7 +1595,6 @@ def test_dump_indent(self): args=[ Name( id='eggs', - ctx=Load(), lineno=1, col_offset=5, end_lineno=1, @@ -1534,23 +1624,23 @@ def test_dump_incomplete(self): ) node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) self.assertEqual(ast.dump(node), - "Raise(exc=Name(id='e', ctx=Load()))" + "Raise(exc=Name(id='e'))" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(Name('e', Load()))" + "Raise(Name('e'))" ) self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" + "Raise(exc=Name(id='e'), lineno=3, col_offset=4)" ) self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)" + "Raise(Name('e'), lineno=3, col_offset=4)" ) node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) self.assertEqual(ast.dump(node), - "Raise(cause=Name(id='e', ctx=Load()))" + "Raise(cause=Name(id='e'))" ) self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(cause=Name('e', Load()))" + "Raise(cause=Name('e'))" ) # Arguments: node = ast.arguments(args=[ast.arg("x")]) @@ -1582,10 +1672,10 @@ def test_dump_incomplete(self): [ast.Name('dataclass', ctx=ast.Load())], ) self.assertEqual(ast.dump(node), - "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass')])", ) self.assertEqual(ast.dump(node, annotate_fields=False), - "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass')])", ) def test_dump_show_empty(self): @@ -1613,7 +1703,7 @@ def check_text(code, empty, full, **kwargs): check_node( # Corner case: there are no real `Name` instances with `id=''`: ast.Name(id='', ctx=ast.Load()), - empty="Name(id='', ctx=Load())", + empty="Name(id='')", full="Name(id='', ctx=Load())", ) @@ -1661,35 +1751,45 @@ def check_text(code, empty, full, **kwargs): check_text( "def a(b: int = 0, *, c): ...", - empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int'))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", ) check_text( "def a(b: int = 0, *, c): ...", - empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", include_attributes=True, ) check_text( 'spam(eggs, "and cheese")', - empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + empty="Module(body=[Expr(value=Call(func=Name(id='spam'), args=[Name(id='eggs'), Constant(value='and cheese')]))])", full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", ) check_text( 'spam(eggs, text="and cheese")', - empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + empty="Module(body=[Expr(value=Call(func=Name(id='spam'), args=[Name(id='eggs')], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", ) check_text( "import _ast as ast; from module import sub", - empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])", - full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", + empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')], is_lazy=0), ImportFrom(module='module', names=[alias(name='sub')], level=0, is_lazy=0)])", + full="Module(body=[Import(names=[alias(name='_ast', asname='ast')], is_lazy=0), ImportFrom(module='module', names=[alias(name='sub')], level=0, is_lazy=0)], type_ignores=[])", ) + def test_dump_with_color(self): + node = ast.parse("x = 1") + self.assertNotIn("\x1b[", ast.dump(node)) + self.assertNotIn("\x1b[", ast.dump(node, color=False)) + self.assertIn("\x1b[", ast.dump(node, color=True)) + + node = ast.Constant(value="\x1b[31m") + self.assertEqual(ast.dump(node), "Constant(value='\\x1b[31m')") + self.assertIn("'\\x1b[31m'", ast.dump(node, color=True)) + def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') src.body.right = ast.copy_location(ast.Constant(2), src.body.right) @@ -1714,12 +1814,12 @@ def test_fix_missing_locations(self): self.assertEqual(src, ast.fix_missing_locations(src)) self.maxDiff = None self.assertEqual(ast.dump(src, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " + "Module(body=[Expr(value=Call(func=Name(id='write', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "end_col_offset=13), Expr(value=Call(func=Name(id='spam', " "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " @@ -1780,22 +1880,22 @@ def test_iter_child_nodes(self): ) def test_get_docstring(self): - node = ast.parse('"""line one\n line two"""') + node = ast.parse('"""line one\n line two"""', optimize=False) self.assertEqual(ast.get_docstring(node), 'line one\nline two') - node = ast.parse('class foo:\n """line one\n line two"""') + node = ast.parse('class foo:\n """line one\n line two"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0]), 'line one\nline two') - node = ast.parse('def foo():\n """line one\n line two"""') + node = ast.parse('def foo():\n """line one\n line two"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0]), 'line one\nline two') - node = ast.parse('async def foo():\n """spam\n ham"""') + node = ast.parse('async def foo():\n """spam\n ham"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') - node = ast.parse('async def foo():\n """spam\n ham"""') + node = ast.parse('async def foo():\n """spam\n ham"""', optimize=False) self.assertEqual(ast.get_docstring(node.body[0], clean=False), 'spam\n ham') node = ast.parse('x') @@ -1834,7 +1934,8 @@ def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): 'def foo():\n """line one\n line two"""\n\n' ' def bar():\n """line one\n line two"""\n' ' """line one\n line two"""\n' - '"""line one\nline two"""\n\n' + '"""line one\nline two"""\n\n', + optimize=False ) self.assertEqual(node.body[0].col_offset, 0) self.assertEqual(node.body[0].lineno, 1) @@ -2407,9 +2508,9 @@ def test_stdlib_validates(self): fn = os.path.join(STDLIB, module) with open(fn, "r", encoding="utf-8") as fp: source = fp.read() - mod = ast.parse(source, fn) + mod = ast.parse(source, fn, optimize=False) compile(mod, fn, "exec") - mod2 = ast.parse(source, fn) + mod2 = ast.parse(source, fn, optimize=False) self.assertTrue(ast.compare(mod, mod2)) constant_1 = ast.Constant(1) @@ -2623,16 +2724,17 @@ def test_assign_to_constant(self): "to in Store context") def test_get_docstring(self): - tree = ast.parse("'docstring'\nx = 1") + tree = ast.parse("'docstring'\nx = 1", optimize=False) self.assertEqual(ast.get_docstring(tree), 'docstring') def get_load_const(self, tree): # Compile to bytecode, disassemble and get parameter of LOAD_CONST - # instructions + # and LOAD_COMMON_CONSTANT instructions co = compile(tree, '', 'exec') consts = [] for instr in dis.get_instructions(co): - if instr.opcode in dis.hasconst: + if instr.opcode in dis.hasconst or \ + instr.opname == 'LOAD_COMMON_CONSTANT': consts.append(instr.argval) return consts @@ -3055,8 +3157,8 @@ def test_source_segment_missing_info(self): class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, transformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) + code, expected_code): + initial_ast = ast.parse(dedent(code)) expected_ast = ast.parse(dedent(expected_code)) transformer = transformer_class() @@ -3392,6 +3494,7 @@ def test_subinterpreter(self): self.assertEqual(res, 0) +@support.force_not_colorized_test_class class CommandLineTests(unittest.TestCase): def setUp(self): self.filename = tempfile.mktemp() @@ -3464,7 +3567,7 @@ def test_exec_mode_flag(self): body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)], type_ignores=[ @@ -3492,7 +3595,7 @@ def test_eval_mode_flag(self): expect = ''' Expression( body=Call( - func=Name(id='print', ctx=Load()), + func=Name(id='print'), args=[ Constant(value=1), Constant(value=2), @@ -3508,12 +3611,11 @@ def test_func_type_mode_flag(self): expect = ''' FunctionType( argtypes=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], + Name(id='int'), + Name(id='str')], returns=Subscript( - value=Name(id='list', ctx=Load()), - slice=Name(id='int', ctx=Load()), - ctx=Load())) + value=Name(id='list'), + slice=Name(id='int'))) ''' for flag in ('-m=func_type', '--mode=func_type'): with self.subTest(flag=flag): @@ -3527,7 +3629,7 @@ def test_no_type_comments_flag(self): body=[ AnnAssign( target=Name(id='x', ctx=Store()), - annotation=Name(id='bool', ctx=Load()), + annotation=Name(id='bool'), value=Constant(value=1), simple=1)]) ''' @@ -3572,7 +3674,7 @@ def test_feature_version_flag(self): Module( body=[ Match( - subject=Name(id='x', ctx=Load()), + subject=Name(id='x'), cases=[ match_case( pattern=MatchValue( @@ -3595,7 +3697,7 @@ def test_no_optimize_flag(self): Module( body=[ Match( - subject=Name(id='a', ctx=Load()), + subject=Name(id='a'), cases=[ match_case( pattern=MatchValue( @@ -3621,7 +3723,7 @@ def test_optimize_flag(self): Module( body=[ Match( - subject=Name(id='a', ctx=Load()), + subject=Name(id='a'), cases=[ match_case( pattern=MatchValue( @@ -3776,46 +3878,6 @@ def test_match_case_not_folded_in_unoptimized_ast(self): for optval in (0, 1, 2): self.assertIn(folded if optval else unfolded, ast.dump(ast.parse(src, optimize=optval))) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_binop(self): - return super().test_folding_binop() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_comparator(self): - return super().test_folding_comparator() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_iter(self): - return super().test_folding_iter() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_not(self): - return super().test_folding_not() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_subscript(self): - return super().test_folding_subscript() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_tuple(self): - return super().test_folding_tuple() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_type_param_in_class_def(self): - return super().test_folding_type_param_in_class_def() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_type_param_in_function_def(self): - return super().test_folding_type_param_in_function_def() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_type_param_in_type_alias(self): - return super().test_folding_type_param_in_type_alias() - - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags - def test_folding_unaryop(self): - return super().test_folding_unaryop() - if __name__ == '__main__': if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py new file mode 100644 index 00000000000..4c116ca0c0a --- /dev/null +++ b/Lib/test/test_lazy_import/__init__.py @@ -0,0 +1,2826 @@ +"""Tests for PEP 810 lazy imports.""" + +import io +import dis +import subprocess +import sys +import textwrap +import threading +import types +import unittest +import tempfile +import os +import contextlib + +from test import support +from test.support.script_helper import assert_python_ok, assert_python_failure + +try: + import _testcapi +except ImportError: + _testcapi = None + + +class LazyImportTestCase(unittest.TestCase): + def setUp(self): + self.lazy_imports_filter = sys.get_lazy_imports_filter() + self.lazy_imports = sys.get_lazy_imports() + + def tearDown(self): + """Clean up any test modules from sys.modules.""" + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + + sys.set_lazy_imports_filter(self.lazy_imports_filter) + sys.set_lazy_imports(self.lazy_imports) + sys.lazy_modules.clear() + + +class LazyImportTests(LazyImportTestCase): + """Tests for basic lazy import functionality.""" + + def test_basic_unused(self): + """Lazy imported module should not be loaded if never accessed.""" + import test.test_lazy_import.data.basic_unused + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) + + def test_sys_lazy_modules(self): + try: + import test.test_lazy_import.data.basic_from_unused + except ImportError as e: + self.fail('lazy import failed') + + self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules) + # The package is already loaded, so it is not a pending import. + self.assertNotIn("test.test_lazy_import.data", sys.lazy_modules) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) + test.test_lazy_import.data.basic_from_unused.basic2 + self.assertNotIn("test.test_import.data", sys.lazy_modules) + + def test_basic_unused_use_externally(self): + """Lazy import should load module when accessed from outside.""" + from test.test_lazy_import.data import basic_unused + + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + x = basic_unused.test.test_lazy_import.data.basic2 + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_basic_from_unused_use_externally(self): + """Lazy 'from' import should load when accessed from outside.""" + from test.test_lazy_import.data import basic_from_unused + + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + x = basic_from_unused.basic2 + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_basic_unused_dir(self): + """dir() on module should not trigger lazy import reification.""" + import test.test_lazy_import.data.basic_unused + + x = dir(test.test_lazy_import.data.basic_unused) + self.assertIn("test", x) + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_basic_dir(self): + """dir() at module scope should not trigger lazy import reification.""" + from test.test_lazy_import.data import basic_dir + + self.assertIn("test", basic_dir.x) + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_basic_used(self): + """Lazy import should load when accessed within the module.""" + import test.test_lazy_import.data.basic_used + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + @support.requires_subprocess() + def test_from_import_with_module_getattr(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_raising(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import raising_attr + + try: + raising_attr + except ValueError as exc: + assert str(exc) == 'from_getattr', exc + else: + assert False, f'ValueError is not raised: {raising_attr}' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_missing(self): + """Lazy from import should respect module-level __getattr__.""" + for attr in ("missing_attr", "import_error_attr"): + with self.subTest(attr=attr): + code = textwrap.dedent(f""" + lazy from test.test_lazy_import.data.module_with_getattr import {attr} + + try: + {attr} + except ImportError as exc: + assert '{attr}' in str(exc), exc + assert exc.__cause__ is not None + else: + assert False, ('ImportError is not raised', {attr}) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_warning(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + import warnings + + with warnings.catch_warnings(record=True) as log: + lazy from test.test_lazy_import.data.module_with_getattr import warning_attr + + assert log == [] + + with warnings.catch_warnings(record=True) as log: + warning_attr + assert warning_attr == 'from_warning_attr', warning_attr + assert len(log) == 1, log + assert isinstance(log[0].message, UserWarning), log + assert str(log[0].message) == 'from_getattr', log + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_imported_module_getattr(self): + """Lazy from import should not shadow an imported module's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.module_with_getattr as mod + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + assert mod.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + +class GlobalLazyImportModeTests(LazyImportTestCase): + """Tests for sys.set_lazy_imports() global mode control.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_global_off_rejected(self): + """Mode 'none' is not supported.""" + with self.assertRaises(ValueError): + sys.set_lazy_imports("none") + + def test_global_on(self): + """Mode 'all' should make regular imports lazy.""" + import test.test_lazy_import.data.global_on + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_global_filter(self): + """Filter returning False should prevent lazy loading.""" + import test.test_lazy_import.data.global_filter + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_global_filter_true(self): + """Filter returning True should allow lazy loading.""" + import test.test_lazy_import.data.global_filter_true + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_global_filter_from(self): + """Filter should work with 'from' imports.""" + import test.test_lazy_import.data.global_filter_from + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_global_filter_from_true(self): + """Filter returning True should allow lazy 'from' imports.""" + import test.test_lazy_import.data.global_filter_from_true + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + +class CompatibilityModeTests(LazyImportTestCase): + """Tests for __lazy_modules__ compatibility mode.""" + + def test_compatibility_mode(self): + """__lazy_modules__ should enable lazy imports for listed modules.""" + import test.test_lazy_import.data.basic_compatibility_mode + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_compatibility_mode_used(self): + """Using a lazy import from __lazy_modules__ should load the module.""" + import test.test_lazy_import.data.basic_compatibility_mode_used + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_compatibility_mode_func(self): + """Imports inside functions should be eager even in compatibility mode.""" + import test.test_lazy_import.data.compatibility_mode_func + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_compatibility_mode_try_except(self): + """Imports in try/except should be eager even in compatibility mode.""" + import test.test_lazy_import.data.compatibility_mode_try_except + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_compatibility_mode_relative(self): + """__lazy_modules__ should work with relative imports.""" + import test.test_lazy_import.data.basic_compatibility_mode_relative + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + +class ModuleIntrospectionTests(LazyImportTestCase): + """Tests for module dict and getattr behavior with lazy imports.""" + + def test_modules_dict(self): + """Accessing module.__dict__ should not trigger reification.""" + import test.test_lazy_import.data.modules_dict + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_modules_getattr(self): + """Module __getattr__ for lazy import name should trigger reification.""" + import test.test_lazy_import.data.modules_getattr + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_modules_getattr_other(self): + """Module __getattr__ for other names should not trigger reification.""" + import test.test_lazy_import.data.modules_getattr_other + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + +class LazyImportTypeTests(LazyImportTestCase): + """Tests for the LazyImportType and its resolve() method.""" + + def test_lazy_value_resolve(self): + """resolve() method should force the lazy import to load.""" + import test.test_lazy_import.data.lazy_get_value + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_lazy_import_type_exposed(self): + """LazyImportType should be exposed in types module.""" + self.assertHasAttr(types, 'LazyImportType') + self.assertEqual(types.LazyImportType.__name__, 'lazy_import') + + def test_lazy_import_type_cant_construct(self): + """LazyImportType should not be directly constructible.""" + self.assertRaises(TypeError, types.LazyImportType, {}, "module") + + @support.requires_subprocess() + def test_lazy_import_type_attributes_accessible(self): + """Check that static PyLazyImport_Type is initialized at startup.""" + code = textwrap.dedent(""" + lazy import json + print(globals()["json"].resolve) + """) + proc = assert_python_ok("-c", code) + self.assertIn(b" initial_count + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +@support.requires_subprocess() +class ErrorHandlingTests(LazyImportTestCase): + """Tests for error handling during lazy import reification.""" + + def test_missing_lazy_submodule_raises_module_not_found_error(self): + """Accessing a nonexistent lazy submodule via parent attr raises ModuleNotFoundError.""" + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.nonexistent_module + + try: + _ = test.test_lazy_import.data.nonexistent_module + except ModuleNotFoundError: + pass + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_non_package_lazily_imported(self): + """Accessing a nonexistent lazy name via parent attr raises ModuleNotFoundError.""" + code = textwrap.dedent(""" + lazy import math.pi + + try: + _ = math.pi + except ModuleNotFoundError: + pass + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_non_package_lazily_imported_as(self): + """A dotted lazy import as raises when the name is not a module.""" + # gh-157757: the eager statement raises, so the lazy one raises too. + code = textwrap.dedent(""" + lazy import math.pi as pi + + try: + pi + except ModuleNotFoundError: + pass + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_missing_attribute_raises_import_error(self): + """Accessing a nonexistent lazy name via from import raises ImportError.""" + code = textwrap.dedent(""" + lazy from sys import doesnotexist + + try: + _ = doesnotexist + except ImportError: + pass + else: + raise AssertionError("ImportError was not raised") + """) + assert_python_ok("-c", code) + + def test_missing_lazy_from_import_shows_chained_traceback(self): + """Accessing missing attribute from lazy from-import should chain errors.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.basic2 import nonexistent_name + + try: + _ = nonexistent_name + except ImportError as e: + assert e.__cause__ is not None, "Expected chained exception" + else: + raise AssertionError("ImportError was not raised") + """) + assert_python_ok("-c", code) + + @support.subTests('name', ( + 'test.test_lazy_import.data.broken_module_chained_cause', + 'test.test_lazy_import.data.broken_module_chained_context', + 'test.test_lazy_import.data.broken_module_chained_suppressed', + )) + def test_chained_exception_import_shows_notes(self, name): + """Accessing missing attribute from lazy from-import should chain errors.""" + code = textwrap.dedent(f""" + lazy import {name} + + try: + _ = {name} + except ValueError as e: + assert any( + note.startswith("lazy import of '{name}' declared in ") + for note in e.__notes__ + ), e.__notes__ + else: + raise AssertionError("ImportError was not raised") + """) + assert_python_ok("-c", code) + + def test_reification_retries_on_failure(self): + """Failed reification should allow retry on subsequent access. + + PEP 810: "If reification fails, the lazy object is not reified or replaced. + Subsequent uses of the lazy object will re-try the reification." + """ + code = textwrap.dedent(""" + import sys + + lazy import test.test_lazy_import.data.broken_module + + try: + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") + + assert "test.test_lazy_import.data.broken_module" not in sys.modules + + try: + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") + """) + assert_python_ok("-c", code) + + def test_lazy_submodule_traceback_hides_importlib_frames(self): + code = textwrap.dedent(""" + import traceback + + lazy import test.test_lazy_import.data.broken_module + + try: + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + frames = traceback.extract_tb(exc.__traceback__) + assert [frame.name for frame in frames] == ["", ""], frames + assert frames[0].filename == "", frames + assert frames[1].filename.endswith("broken_module.py"), frames + else: + raise AssertionError("ValueError was not raised") + """) + assert_python_ok("-c", code) + + def test_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.missing_dependency + + try: + _ = test.test_lazy_import.data.missing_dependency + except ModuleNotFoundError as exc: + assert exc.name == "missing_dependency_for_lazy_import_test", exc.name + assert str(exc) == "No module named 'missing_dependency_for_lazy_import_test'", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_self_named_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.self_named_module_not_found + + try: + _ = test.test_lazy_import.data.self_named_module_not_found + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.self_named_module_not_found", exc.name + assert str(exc) == "boom", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_none_in_sys_modules_during_submodule_resolution_propagates(self): + code = textwrap.dedent(""" + import sys + + sys.modules["test.test_lazy_import.data.blocked_module"] = None + lazy import test.test_lazy_import.data.blocked_module + + try: + _ = test.test_lazy_import.data.blocked_module + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.blocked_module", exc.name + assert str(exc) == ( + "import of test.test_lazy_import.data.blocked_module " + "halted; None in sys.modules" + ), exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_circular_lazy_import_does_not_crash_for_gh_144727(self): + with tempfile.TemporaryDirectory() as tmpdir: + a_path = os.path.join(tmpdir, "a.py") + b_path = os.path.join(tmpdir, "b.py") + + with open(a_path, "w") as f: + f.write(textwrap.dedent("""\ + lazy import b + + def something(): + b.hello() + + something() + """)) + + with open(b_path, "w") as f: + f.write(textwrap.dedent("""\ + lazy import a + + def hello(): + print(a) + """)) + + result = subprocess.run( + [sys.executable, a_path], + capture_output=True, + text=True, + cwd=tmpdir, + ) + # Should get a proper Python error, not a crash + self.assertEqual(result.returncode, 1) + self.assertIn("Error", result.stderr) + + +@support.requires_subprocess() +class GlobalsAndDictTests(LazyImportTestCase): + """Tests for globals() and __dict__ behavior with lazy imports. + + PEP 810: "Calling globals() or accessing a module's __dict__ does not trigger + reification – they return the module's dictionary, and accessing lazy objects + through that dictionary still returns lazy proxy objects." + """ + + def test_globals_returns_lazy_proxy_when_accessed_from_function(self): + """globals() accessed from a function should return lazy proxy without reification. + + Note: At module level, accessing globals()['name'] triggers LOAD_NAME which + automatically resolves lazy imports. Inside a function, accessing globals()['name'] + uses BINARY_SUBSCR which returns the lazy proxy without resolution. + """ + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_lazy_import.data.basic2 import x + + # Check that module is not yet loaded + assert 'test.test_lazy_import.data.basic2' not in sys.modules + + def check_lazy(): + # Access through globals() from inside a function + g = globals() + lazy_obj = g['x'] + return type(lazy_obj) is types.LazyImportType + + # Inside function, should get lazy proxy + is_lazy = check_lazy() + assert is_lazy, "Expected LazyImportType from function scope" + + # Module should STILL not be loaded + assert 'test.test_lazy_import.data.basic2' not in sys.modules + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_globals_dict_access_returns_lazy_proxy_inline(self): + """Accessing globals()['name'] inline should return lazy proxy. + + Note: Assigning g['name'] to a local variable at module level triggers + reification due to STORE_NAME bytecode. Inline access preserves laziness. + """ + code = textwrap.dedent(""" + import sys + import types + lazy import json + # Inline access without assignment to local variable preserves lazy proxy + assert type(globals()['json']) is types.LazyImportType + assert 'json' not in sys.modules + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_module_dict_returns_lazy_proxy_without_reifying(self): + """module.__dict__ access should not trigger reification.""" + import test.test_lazy_import.data.globals_access + + # Module not loaded yet via direct dict access + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + # Access via get_from_globals should return lazy proxy + lazy_obj = test.test_lazy_import.data.globals_access.get_from_globals() + self.assertEqual(type(lazy_obj), types.LazyImportType) + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_direct_access_triggers_reification(self): + """Direct name access (not through globals()) should trigger reification.""" + import test.test_lazy_import.data.globals_access + + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + # Direct access should reify + result = test.test_lazy_import.data.globals_access.get_direct() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_resolve_method_forces_reification(self): + """Calling resolve() on lazy proxy should force reification. + + Note: Must access lazy proxy from within a function to avoid automatic + reification by LOAD_NAME at module level. + """ + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_lazy_import.data.basic2 import x + + assert 'test.test_lazy_import.data.basic2' not in sys.modules + + def test_resolve(): + g = globals() + lazy_obj = g['x'] + assert type(lazy_obj) is types.LazyImportType, f"Expected lazy proxy, got {type(lazy_obj)}" + + resolved = lazy_obj.resolve() + + # Now module should be loaded + assert 'test.test_lazy_import.data.basic2' in sys.modules + assert resolved == 42 # x is 42 in basic2.py + return True + + assert test_resolve() + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_add_lazy_to_globals(self): + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_lazy_import.data import basic2 + + assert 'test.test_lazy_import.data.basic2' not in sys.modules + + class C: pass + sneaky = C() + sneaky.x = 1 + + def f(): + t = 0 + for _ in range(5): + t += sneaky.x + return t + + f() + globals()["sneaky"] = globals()["basic2"] + assert f() == 210 + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_add_lazy_to_exec_globals_after_specialization(self): + code = textwrap.dedent(""" + source = ''' + import sys + import types + + lazy from test.test_lazy_import.data import basic2 + + assert 'test.test_lazy_import.data.basic2' not in sys.modules + + class C: pass + sneaky = C() + sneaky.x = 1 + + def f(): + t = 0 + for _ in range(5): + t += sneaky.x + return t + + f() + globals()["sneaky"] = globals()["basic2"] + assert f() == 210 + print("OK") + ''' + ns = {"__name__": "lazy_exec_globals"} + exec(source, ns) + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_add_lazy_to_exec_builtins_after_specialization(self): + code = textwrap.dedent(""" + import builtins + source = ''' + import sys + import types + + lazy from test.test_lazy_import.data import basic2 + + assert 'test.test_lazy_import.data.basic2' not in sys.modules + + class C: pass + sneaky = C() + sneaky.x = 1 + __builtins__["sneaky"] = sneaky + del sneaky + + def f(): + t = 0 + for _ in range(5): + t += sneaky.x + return t + + f() + __builtins__["sneaky"] = globals()["basic2"] + assert f() == 210 + print("OK") + ''' + ns = {"__name__": "lazy_exec_builtins", "__builtins__": builtins.__dict__.copy()} + exec(source, ns) + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +@support.requires_subprocess() +class MultipleNameFromImportTests(LazyImportTestCase): + """Tests for lazy from ... import with multiple names. + + PEP 810: "When using lazy from ... import, each imported name is bound to a + lazy proxy object. The first access to any of these names triggers loading + of the entire module and reifies only that specific name to its actual value. + Other names remain as lazy proxies until they are accessed." + """ + + def test_accessing_one_name_leaves_others_as_proxies(self): + """Accessing one name from multi-name import should leave others lazy.""" + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_lazy_import.data.basic2 import f, x + + # Neither should be loaded yet + assert 'test.test_lazy_import.data.basic2' not in sys.modules + + g = globals() + assert type(g['f']) is types.LazyImportType + assert type(g['x']) is types.LazyImportType + + # Access 'x' - this loads the module and reifies only 'x' + value = x + assert value == 42 + + # Module is now loaded + assert 'test.test_lazy_import.data.basic2' in sys.modules + + # 'x' should be reified (int), 'f' should still be lazy proxy + assert type(g['x']) is int, f"Expected int, got {type(g['x'])}" + assert type(g['f']) is types.LazyImportType, f"Expected LazyImportType, got {type(g['f'])}" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_accessing_one_name_imports_only_its_submodule(self): + """Accessing one name should not import the other names' submodules.""" + code = textwrap.dedent(""" + import sys + + lazy from test.test_lazy_import.data.pkg import b, bar, broken + + # Importing bar prints, and importing broken raises. + b.foo() + + assert "test.test_lazy_import.data.pkg.bar" not in sys.modules + assert "test.test_lazy_import.data.pkg.broken" not in sys.modules + """) + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(out, b"") + + def test_all_names_reified_after_all_accessed(self): + """All names should be reified after each is accessed.""" + code = textwrap.dedent(""" + import sys + import types + + lazy from test.test_lazy_import.data.basic2 import f, x + + g = globals() + + # Access both + _ = x + _ = f + + # Both should be reified now + assert type(g['x']) is int + assert callable(g['f']) + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +@support.requires_subprocess() +class SysLazyModulesTrackingTests(LazyImportTestCase): + """Tests for sys.lazy_modules tracking behavior. + + PEP 810: "When the module is reified, it's removed from sys.lazy_modules" + """ + + def test_module_added_to_lazy_modules_on_lazy_import(self): + """Module should be added to sys.lazy_modules when lazily imported.""" + # PEP 810 states lazy_modules tracks modules that have been lazily imported + # Note: The current implementation keeps modules in lazy_modules even after + # reification (primarily for diagnostics and introspection) + code = textwrap.dedent(""" + import sys + + initial_count = len(sys.lazy_modules) + + lazy import test.test_lazy_import.data.basic2 + + # Should be in lazy_modules after lazy import + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules + assert len(sys.lazy_modules) > initial_count + + # Trigger reification + _ = test.test_lazy_import.data.basic2.x + + # Module should still be tracked (for diagnostics per PEP 810) + assert "test.test_lazy_import.data.basic2" not in sys.lazy_modules + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_lazy_modules_is_per_interpreter(self): + """Each interpreter should have independent sys.lazy_modules.""" + # Basic test that sys.lazy_modules exists and is a set + self.assertIsInstance(sys.lazy_modules, set) + + def test_lazy_module_without_children_is_tracked(self): + code = textwrap.dedent(""" + import sys + lazy import json + assert "json" in sys.lazy_modules, ( + f"expected 'json' in sys.lazy_modules, got {set(sys.lazy_modules)}" + ) + print("OK") + """) + assert_python_ok("-c", code) + + def test_already_loaded_module_is_not_tracked(self): + """A lazy import of an already loaded module should not be tracked.""" + code = textwrap.dedent(""" + import sys + + # Loaded by a regular import. + import json + lazy import json as lazy_json + assert "json" not in sys.lazy_modules, ( + f"expected 'json' not in sys.lazy_modules, got {sys.lazy_modules}" + ) + + # Loaded by reifying an earlier lazy import. + lazy import base64 + _ = base64.b64encode + lazy import base64 as lazy_base64 + assert "base64" not in sys.lazy_modules, ( + f"expected 'base64' not in sys.lazy_modules, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_already_loaded_submodule_is_not_tracked(self): + """`lazy from` a loaded submodule should not be tracked either.""" + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg.b + lazy from test.test_lazy_import.data.pkg import b + assert "test.test_lazy_import.data.pkg.b" not in sys.lazy_modules, ( + f"expected 'pkg.b' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_attribute_entry_removed_on_reification(self): + """`lazy from x import attr` should untrack "x.attr" once resolved.""" + code = textwrap.dedent(""" + import sys + lazy from test.test_lazy_import.data.basic2 import x + assert "test.test_lazy_import.data.basic2.x" in sys.lazy_modules, ( + f"expected 'basic2.x' tracked, got {sys.lazy_modules}" + ) + _ = x + assert "test.test_lazy_import.data.basic2.x" not in sys.lazy_modules, ( + f"expected 'basic2.x' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_already_loaded_attribute_is_not_tracked(self): + code = textwrap.dedent(""" + import sys + import math + + lazy from math import pi + assert "math.pi" not in sys.lazy_modules, sys.lazy_modules + assert pi == math.pi + """) + assert_python_ok("-c", code) + + def test_cached_lazy_attribute_tracks_its_target(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.basic_from_unused + + holder = "test.test_lazy_import.data.basic_from_unused" + target = "test.test_lazy_import.data.basic2" + + namespace = {"__name__": "cached_import_test"} + exec(f"lazy from {holder} import basic2", namespace) + assert holder + ".basic2" not in sys.lazy_modules, sys.lazy_modules + assert target in sys.lazy_modules, sys.lazy_modules + exec("assert basic2.x == 42", namespace) + assert target not in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_cached_attribute_keeps_pending_module_tracked(self): + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as pkg + pkg.b = 42 + + lazy import test.test_lazy_import.data.pkg.b as pending + lazy from test.test_lazy_import.data.pkg import b + name = "test.test_lazy_import.data.pkg.b" + assert b == 42, b + assert name not in sys.modules, sys.modules + assert name in sys.lazy_modules, sys.lazy_modules + assert pending.foo() == "foo" + assert name not in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_failed_reification_stays_tracked(self): + """A lazy import that fails to resolve must stay tracked.""" + code = textwrap.dedent(""" + import sys + lazy import test.test_lazy_import.data.broken_module + try: + _ = test.test_lazy_import.data.broken_module + except ValueError: + pass + else: + raise AssertionError("ValueError was not raised") + assert "test.test_lazy_import.data.broken_module" in sys.lazy_modules, ( + f"failed reification must stay tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_blocked_module_is_still_tracked(self): + """A ``None`` entry in sys.modules must not count as loaded.""" + code = textwrap.dedent(""" + import sys + sys.modules['test.test_lazy_import.data.basic2'] = None + lazy import test.test_lazy_import.data.basic2 + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules, ( + f"blocked module must stay tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_initializing_module_is_still_tracked(self): + """A module that is still executing must not count as loaded.""" + code = textwrap.dedent(""" + import sys + name = "test.test_lazy_import.data.init_fails" + try: + import test.test_lazy_import.data.init_fails + except ValueError: + pass + else: + raise AssertionError("ValueError was not raised") + assert name not in sys.modules, "failed import left a module behind" + assert name in sys.lazy_modules, ( + f"expected {name!r} tracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_module_spec_descriptor_is_not_run(self): + """Checking whether a module is loaded must not run its descriptors.""" + code = textwrap.dedent(""" + import sys + import types + + class RaisingSpec(types.ModuleType): + @property + def __spec__(self): + raise RuntimeError("__spec__ descriptor was run") + + sys.modules["raising_spec"] = RaisingSpec("raising_spec") + lazy import raising_spec + assert "raising_spec" not in sys.lazy_modules, ( + f"expected 'raising_spec' untracked, got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + def test_spec_initializing_descriptor_is_not_run(self): + code = textwrap.dedent(""" + import sys + import types + + class Spec: + @property + def _initializing(self): + raise RuntimeError("_initializing descriptor was run") + + @property + def __dict__(self): + raise RuntimeError("__dict__ descriptor was run") + + class SlottedSpec: + __slots__ = () + + @property + def _initializing(self): + raise RuntimeError("_initializing descriptor was run") + + for spec in (Spec(), SlottedSpec()): + module = types.ModuleType("custom_spec") + module.__spec__ = spec + sys.modules["custom_spec"] = module + lazy import custom_spec + """) + assert_python_ok("-c", code) + + def test_cached_attribute_does_not_check_spec_twice(self): + code = textwrap.dedent(""" + import sys + import types + + class Spec: + def __init__(self): + self.calls = 0 + + @property + def _initializing(self): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("_initializing was read twice") + return False + + module = types.ModuleType("cached_spec") + module.__spec__ = Spec() + module.attr = 1 + sys.modules["cached_spec"] = module + lazy from cached_spec import attr + assert attr == 1, attr + """) + assert_python_ok("-c", code) + + def test_spec_initializing_truth_conversion_is_not_run(self): + code = textwrap.dedent(""" + import sys + import types + from importlib.machinery import ModuleSpec + + class Flag: + def __bool__(self): + raise RuntimeError("_initializing truth conversion was run") + + released = memoryview(b"") + released.release() + for flag in (Flag(), released): + sys.lazy_modules.discard("custom_spec") + module = types.ModuleType("custom_spec") + module.__spec__ = ModuleSpec("custom_spec", None) + module.__spec__._initializing = flag + sys.modules["custom_spec"] = module + lazy import custom_spec + assert "custom_spec" in sys.lazy_modules, sys.lazy_modules + """) + assert_python_ok("-c", code) + + def test_pending_submodule_is_still_tracked(self): + """`lazy from` a submodule that is not loaded must stay tracked.""" + code = textwrap.dedent(""" + import sys + lazy from test.test_lazy_import.data.pkg import b + assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, ( + f"expected 'pkg.b' tracked, got {sys.lazy_modules}" + ) + import test.test_lazy_import.data.pkg + assert "test.test_lazy_import.data.pkg.b" not in sys.modules, ( + "loading the package must not load the submodule" + ) + assert "test.test_lazy_import.data.pkg.b" in sys.lazy_modules, ( + f"loading the package must not untrack the submodule, " + f"got {sys.lazy_modules}" + ) + """) + assert_python_ok("-c", code) + + +@support.requires_subprocess() +class CommandLineAndEnvVarTests(unittest.TestCase): + """Tests for command-line and environment variable control. + + PEP 810: The global lazy imports flag can be controlled through: + - The -X lazy_imports= command-line option + - The PYTHON_LAZY_IMPORTS= environment variable + """ + + def test_cli_lazy_imports_all_makes_regular_imports_lazy(self): + """-X lazy_imports=all should make all imports potentially lazy.""" + code = textwrap.dedent(""" + import sys + # In 'all' mode, regular imports become lazy + import json + # json should not be in sys.modules yet (lazy) + # Actually accessing it triggers reification + if 'json' not in sys.modules: + print("LAZY") + else: + print("EAGER") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=all", "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("LAZY", result.stdout) + + def test_cli_lazy_imports_none_is_rejected(self): + """-X lazy_imports=none should be rejected.""" + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=none", "-c", "pass"], + capture_output=True, + text=True + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("-X lazy_imports: invalid value", result.stderr) + self.assertIn("expected 'all' or 'normal'", result.stderr) + + def test_cli_lazy_imports_normal_respects_lazy_keyword_only(self): + """-X lazy_imports=normal should respect lazy keyword only.""" + # Note: Use test modules instead of stdlib modules to avoid + # modules already loaded by the interpreter startup + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.basic2 # Should be eager + lazy import test.test_lazy_import.data.pkg.b # Should be lazy + + eager_loaded = 'test.test_lazy_import.data.basic2' in sys.modules + lazy_loaded = 'test.test_lazy_import.data.pkg.b' in sys.modules + + if eager_loaded and not lazy_loaded: + print("OK") + else: + print(f"FAIL: eager={eager_loaded}, lazy={lazy_loaded}") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=normal", "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_env_var_lazy_imports_all_enables_global_lazy(self): + """PYTHON_LAZY_IMPORTS=all should enable global lazy imports.""" + code = textwrap.dedent(""" + import sys + import json + if 'json' not in sys.modules: + print("LAZY") + else: + print("EAGER") + """) + import os + env = os.environ.copy() + env["PYTHON_LAZY_IMPORTS"] = "all" + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("LAZY", result.stdout) + + def test_env_var_lazy_imports_none_is_rejected(self): + """PYTHON_LAZY_IMPORTS=none should be rejected.""" + import os + env = os.environ.copy() + env["PYTHON_LAZY_IMPORTS"] = "none" + result = subprocess.run( + [sys.executable, "-c", "pass"], + capture_output=True, + text=True, + env=env + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("PYTHON_LAZY_IMPORTS: invalid value", result.stderr) + self.assertIn("expected 'all' or 'normal'", result.stderr) + + def test_cli_overrides_env_var(self): + """Command-line option should take precedence over environment variable.""" + # PEP 810: -X lazy_imports takes precedence over PYTHON_LAZY_IMPORTS + code = textwrap.dedent(""" + import sys + import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + import os + env = os.environ.copy() + env["PYTHON_LAZY_IMPORTS"] = "all" # env says all imports are lazy + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=normal", "-c", code], + capture_output=True, + text=True, + env=env + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + # CLI should win, so a regular import should stay eager. + self.assertIn("EAGER", result.stdout) + + def test_sys_set_lazy_imports_overrides_cli(self): + """sys.set_lazy_imports() should take precedence over CLI option.""" + code = textwrap.dedent(""" + import sys + sys.set_lazy_imports("normal") # Override CLI + import json + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + result = subprocess.run( + [sys.executable, "-X", "lazy_imports=all", "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + + +@support.requires_subprocess() +class FilterFunctionSignatureTests(LazyImportTestCase): + """Tests for the filter function signature per PEP 810. + + PEP 810: func(importer: str, name: str, fromlist: tuple[str, ...] | None) -> bool + """ + + def _run_subprocess_with_modules(self, code, files): + with tempfile.TemporaryDirectory() as tmpdir: + for relpath, contents in files.items(): + path = os.path.join(tmpdir, relpath) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as file: + file.write(textwrap.dedent(contents)) + + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join( + entry for entry in (tmpdir, env.get("PYTHONPATH")) if entry + ) + env["PYTHON_LAZY_IMPORTS"] = "normal" + + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + capture_output=True, + cwd=tmpdir, + env=env, + text=True, + ) + return result + + def _assert_subprocess_ok(self, code, files): + result = self._run_subprocess_with_modules(code, files) + self.assertEqual( + result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}" + ) + return result + + def test_filter_receives_correct_arguments_for_import(self): + """Filter should receive (importer, name, fromlist=None) for 'import x'.""" + code = textwrap.dedent(""" + import sys + + received_args = [] + + def my_filter(importer, name, fromlist): + received_args.append((importer, name, fromlist)) + return True + + sys.set_lazy_imports_filter(my_filter) + + lazy import json + + assert len(received_args) == 1, f"Expected 1 call, got {len(received_args)}" + importer, name, fromlist = received_args[0] + assert name == "json", f"Expected name='json', got {name!r}" + assert fromlist is None, f"Expected fromlist=None, got {fromlist!r}" + assert isinstance(importer, str), f"Expected str importer, got {type(importer)}" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_filter_receives_fromlist_for_from_import(self): + """Filter should receive fromlist tuple for 'from x import y, z'.""" + code = textwrap.dedent(""" + import sys + + received_args = [] + + def my_filter(importer, name, fromlist): + received_args.append((importer, name, fromlist)) + return True + + sys.set_lazy_imports_filter(my_filter) + + lazy from json import dumps, loads + + assert len(received_args) == 1, f"Expected 1 call, got {len(received_args)}" + importer, name, fromlist = received_args[0] + assert name == "json", f"Expected name='json', got {name!r}" + assert fromlist == ("dumps", "loads"), f"Expected ('dumps', 'loads'), got {fromlist!r}" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_filter_returning_false_forces_eager_import(self): + """Filter returning False should make import eager.""" + code = textwrap.dedent(""" + import sys + + def deny_filter(importer, name, fromlist): + return False + + sys.set_lazy_imports_filter(deny_filter) + + lazy import json + + # Should be eager due to filter + if 'json' in sys.modules: + print("EAGER") + else: + print("LAZY") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") + self.assertIn("EAGER", result.stdout) + + def test_filter_distinguishes_absolute_and_relative_from_imports(self): + """Relative imports should pass resolved module names to the filter.""" + files = { + "target.py": """ + VALUE = "absolute" + """, + "pkg/__init__.py": "", + "pkg/target.py": """ + VALUE = "relative" + """, + "pkg/runner.py": """ + import sys + + seen = [] + + def my_filter(importer, name, fromlist): + seen.append((importer, name, fromlist)) + return True + + sys.set_lazy_imports_filter(my_filter) + + lazy from target import VALUE as absolute_value + lazy from .target import VALUE as relative_value + + assert seen == [ + (__name__, "target", ("VALUE",)), + (__name__, "pkg.target", ("VALUE",)), + ], seen + """, + } + + result = self._assert_subprocess_ok( + """ + import pkg.runner + print("OK") + """, + files, + ) + self.assertIn("OK", result.stdout) + + def test_filter_receives_resolved_name_for_relative_package_import(self): + """'lazy from . import x' should report the resolved package name.""" + files = { + "pkg/__init__.py": "", + "pkg/sibling.py": """ + VALUE = 1 + """, + "pkg/runner.py": """ + import sys + + seen = [] + + def my_filter(importer, name, fromlist): + seen.append((importer, name, fromlist)) + return True + + sys.set_lazy_imports_filter(my_filter) + + lazy from . import sibling + + assert seen == [ + (__name__, "pkg", ("sibling",)), + ], seen + """, + } + + result = self._assert_subprocess_ok( + """ + import pkg.runner + print("OK") + """, + files, + ) + self.assertIn("OK", result.stdout) + + def test_filter_receives_resolved_name_for_parent_relative_import(self): + """Parent relative imports should also use the resolved module name.""" + files = { + "pkg/__init__.py": "", + "pkg/target.py": """ + VALUE = 1 + """, + "pkg/sub/__init__.py": "", + "pkg/sub/runner.py": """ + import sys + + seen = [] + + def my_filter(importer, name, fromlist): + seen.append((importer, name, fromlist)) + return True + + sys.set_lazy_imports_filter(my_filter) + + lazy from ..target import VALUE + + assert seen == [ + (__name__, "pkg.target", ("VALUE",)), + ], seen + """, + } + + result = self._assert_subprocess_ok( + """ + import pkg.sub.runner + print("OK") + """, + files, + ) + self.assertIn("OK", result.stdout) + + def test_filter_can_force_eager_only_for_resolved_relative_target(self): + """Resolved names should let filters treat relative and absolute imports differently.""" + files = { + "target.py": """ + VALUE = "absolute" + """, + "pkg/__init__.py": "", + "pkg/target.py": """ + VALUE = "relative" + """, + "pkg/runner.py": """ + import sys + + def my_filter(importer, name, fromlist): + return name != "pkg.target" + + sys.set_lazy_imports_filter(my_filter) + + lazy from target import VALUE as absolute_value + lazy from .target import VALUE as relative_value + + assert "pkg.target" in sys.modules, sorted( + name for name in sys.modules + if name in {"target", "pkg.target"} + ) + assert "target" not in sys.modules, sorted( + name for name in sys.modules + if name in {"target", "pkg.target"} + ) + assert relative_value == "relative", relative_value + """, + } + + result = self._assert_subprocess_ok( + """ + import pkg.runner + print("OK") + """, + files, + ) + self.assertIn("OK", result.stdout) + + +class AdditionalSyntaxRestrictionTests(LazyImportTestCase): + """Additional syntax restriction tests per PEP 810.""" + + def test_lazy_import_inside_class_raises_syntax_error(self): + """lazy import inside class body should raise SyntaxError.""" + # PEP 810: "The soft keyword is only allowed at the global (module) level, + # not inside functions, class bodies, try blocks, or import *" + with self.assertRaises(SyntaxError): + import test.test_lazy_import.data.badsyntax.lazy_class_body + + +@support.requires_subprocess() +class MixedLazyEagerImportTests(LazyImportTestCase): + """Tests for mixing lazy and eager imports of the same module. + + PEP 810: "If module foo is imported both lazily and eagerly in the same + program, the eager import takes precedence and both bindings resolve to + the same module object." + """ + + def test_eager_import_before_lazy_resolves_to_same_module(self): + """Eager import before lazy should make lazy resolve to same module.""" + code = textwrap.dedent(""" + import sys + import json # Eager import first + + lazy import json as lazy_json # Lazy import same module + + # lazy_json should resolve to the same object + assert json is lazy_json, "Lazy and eager imports should resolve to same module" + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_lazy_import_before_eager_resolves_to_same_module(self): + """Lazy import followed by eager should both point to same module.""" + code = textwrap.dedent(""" + import sys + + lazy import json as lazy_json + + # Lazy not loaded yet + assert 'json' not in sys.modules + + import json # Eager import triggers load + + # Both should be the same object + assert json is lazy_json + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class RelativeImportTests(LazyImportTestCase): + """Tests for relative imports with lazy keyword.""" + + def test_relative_lazy_import(self): + """lazy from . import submodule should work.""" + from test.test_lazy_import.data import relative_lazy + + # basic2 should not be loaded yet + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + # Access triggers reification + result = relative_lazy.get_basic2() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_relative_lazy_from_import(self): + """lazy from .module import name should work.""" + from test.test_lazy_import.data import relative_lazy_from + + # basic2 should not be loaded yet + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + # Access triggers reification + result = relative_lazy_from.get_x() + self.assertEqual(result, 42) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + +class LazyModulesCompatibilityFromImportTests(LazyImportTestCase): + """Tests for __lazy_modules__ with from imports. + + PEP 810: "When a module is made lazy this way, from-imports using that + module are also lazy" + """ + + def test_lazy_modules_makes_from_imports_lazy(self): + """__lazy_modules__ should make from imports of listed modules lazy.""" + from test.test_lazy_import.data import lazy_compat_from + + # basic2 should not be loaded yet because it's in __lazy_modules__ + self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) + + # Access triggers reification + result = lazy_compat_from.get_x() + self.assertEqual(result, 42) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + +@support.requires_subprocess() +class ImportStateAtReificationTests(LazyImportTestCase): + """Tests for import system state at reification time. + + PEP 810: "Reification still calls __import__ to resolve the import, which uses + the state of the import system (e.g. sys.path, sys.meta_path, sys.path_hooks + and __import__) at reification time, not the state when the lazy import + statement was evaluated." + """ + + def test_sys_path_at_reification_time_is_used(self): + """sys.path changes after lazy import should affect reification.""" + code = textwrap.dedent(""" + import sys + import tempfile + import os + + # Create a temporary module + with tempfile.TemporaryDirectory() as tmpdir: + mod_path = os.path.join(tmpdir, "dynamic_test_module.py") + with open(mod_path, "w") as f: + f.write("VALUE = 'from_temp_dir'\\n") + + # Lazy import before adding to path + lazy import dynamic_test_module + + # Module cannot be found yet + try: + _ = dynamic_test_module + print("FAIL - should not find module") + except ModuleNotFoundError: + pass + + # Now add temp dir to path + sys.path.insert(0, tmpdir) + + # Now reification should succeed using current sys.path + assert dynamic_test_module.VALUE == 'from_temp_dir' + print("OK") + + sys.path.remove(tmpdir) + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +@support.requires_subprocess() +class ThreadSafetyTests(LazyImportTestCase): + """Tests for thread-safety of lazy imports.""" + + def test_concurrent_lazy_import_reification(self): + """Multiple threads racing to reify the same lazy import should succeed.""" + from test.test_lazy_import.data import basic_unused + + num_threads = 10 + results = [None] * num_threads + errors = [] + barrier = threading.Barrier(num_threads) + + def access_lazy_import(idx): + try: + barrier.wait() + module = basic_unused.test.test_lazy_import.data.basic2 + results[idx] = module + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=access_lazy_import, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + self.assertEqual(errors, [], f"Errors occurred: {errors}") + self.assertTrue(all(r is not None for r in results)) + first_module = results[0] + for r in results[1:]: + self.assertIs(r, first_module) + + def test_concurrent_reification_multiple_modules(self): + """Multiple threads reifying different lazy imports concurrently.""" + code = textwrap.dedent(""" + import sys + import threading + + sys.set_lazy_imports("all") + + lazy import json + lazy import os + lazy import io + lazy import re + + num_threads = 8 + results = {} + errors = [] + barrier = threading.Barrier(num_threads) + + def access_modules(idx): + try: + barrier.wait() + mods = [json, os, io, re] + results[idx] = [type(m).__name__ for m in mods] + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=access_modules, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + for idx, mods in results.items(): + assert all(m == 'module' for m in mods), f"Thread {idx} got: {mods}" + + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_concurrent_lazy_modules_dict_updates(self): + """Multiple threads creating lazy imports should safely update sys.lazy_modules.""" + code = textwrap.dedent(""" + import sys + import threading + + num_threads = 16 + iterations = 50 + errors = [] + barrier = threading.Barrier(num_threads) + + def create_lazy_imports(idx): + try: + barrier.wait() + for i in range(iterations): + exec(f"lazy import json as json_{idx}_{i}", globals()) + exec(f"lazy import os as os_{idx}_{i}", globals()) + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=create_lazy_imports, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + assert isinstance(sys.lazy_modules, set), "sys.lazy_modules is not a dict" + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_concurrent_reification_same_module_high_contention(self): + """High contention: many threads reifying the exact same lazy import.""" + code = textwrap.dedent(""" + import sys + import threading + import types + + sys.set_lazy_imports("all") + + lazy import json + + num_threads = 20 + results = [None] * num_threads + errors = [] + barrier = threading.Barrier(num_threads) + + def access_json(idx): + try: + barrier.wait() + for _ in range(100): + _ = json.dumps + _ = json.loads + results[idx] = json + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=access_json, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + assert all(r is not None for r in results), "Some threads got None" + first = results[0] + assert all(r is first for r in results), "Inconsistent module objects" + assert not isinstance(first, types.LazyImportType), "Got lazy import instead of module" + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_concurrent_reification_with_module_attribute_access(self): + """Threads racing to reify and immediately access module attributes.""" + code = textwrap.dedent(""" + import sys + import threading + + sys.set_lazy_imports("all") + + lazy import collections + lazy import functools + lazy import itertools + + num_threads = 12 + results = {} + errors = [] + barrier = threading.Barrier(num_threads) + + def stress_lazy_imports(idx): + try: + barrier.wait() + for _ in range(50): + _ = collections.OrderedDict + _ = functools.partial + _ = itertools.chain + _ = collections.defaultdict + _ = functools.lru_cache + _ = itertools.islice + results[idx] = ( + type(collections).__name__, + type(functools).__name__, + type(itertools).__name__, + ) + except Exception as e: + errors.append((idx, e)) + + threads = [ + threading.Thread(target=stress_lazy_imports, args=(i,)) + for i in range(num_threads) + ] + + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors: {errors}" + for idx, types_tuple in results.items(): + assert all(t == 'module' for t in types_tuple), f"Thread {idx}: {types_tuple}" + print("OK") + """) + + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + +class LazyImportDisTests(unittest.TestCase): + def test_lazy_import_dis(self): + """dis should properly show lazy import""" + code = compile("lazy import foo", "exec", "exec") + f = io.StringIO() + dis.dis(code, file=f) + self.assertIn("foo + lazy", f.getvalue()) + + def test_normal_import_dis(self): + """non lazy imports should just show the name""" + code = compile("import foo", "exec", "exec") + f = io.StringIO() + dis.dis(code, file=f) + for line in f.getvalue().split('\n'): + if "IMPORT_NAME" in line: + self.assertIn("(foo)", line) + break + else: + self.assertFail("IMPORT_NAME not found") + + +@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +class LazyCApiTests(LazyImportTestCase): + def test_set_matches_sys(self): + self.assertEqual(_testcapi.PyImport_GetLazyImportsMode(), sys.get_lazy_imports()) + for mode in ("normal", "all"): + _testcapi.PyImport_SetLazyImportsMode(mode) + self.assertEqual(_testcapi.PyImport_GetLazyImportsMode(), sys.get_lazy_imports()) + self.assertRaises(ValueError, _testcapi.PyImport_SetLazyImportsMode, "none") + + def test_filter_matches_sys(self): + self.assertEqual(_testcapi.PyImport_GetLazyImportsFilter(), sys.get_lazy_imports_filter()) + + def filter(*args): + pass + + _testcapi.PyImport_SetLazyImportsFilter(filter) + self.assertEqual(_testcapi.PyImport_GetLazyImportsFilter(), sys.get_lazy_imports_filter()) + + def test_set_bad_filter(self): + self.assertRaises(ValueError, _testcapi.PyImport_SetLazyImportsFilter, 42) + + def test_dunder_lazy_import_without_frame(self): + # gh-151510: __lazy_import__() called with no globals and no running + # Python frame must raise TypeError instead of crashing. + with self.assertRaisesRegex( + TypeError, + r"__lazy_import__\(\) missing globals when called without a frame", + ): + _testcapi.lazy_import_without_frame( + "test.test_lazy_import.data.basic2" + ) + + +class DottedLazyImportTests(unittest.TestCase): + + def test_lazy_import_as_wins_over_variable(self): + """A dotted lazy import as imports the submodule the variable hides.""" + # gh-157757: importing pkg.b rebinds pkg.b from the variable to the + # module, eagerly and lazily alike. + code = textwrap.dedent(""" + import sys + import test.test_lazy_import.data.pkg as pkg + pkg.b = "hides the b submodule" + + lazy import test.test_lazy_import.data.pkg.b as b + lazy import xml.dom.minidom as minidom + + assert b is sys.modules["test.test_lazy_import.data.pkg.b"], b + assert minidom is sys.modules["xml.dom.minidom"], minidom + """) + assert_python_ok("-c", code) + + + def test_dotted_as_of_loaded_module(self): + """A dotted lazy import as binds the module, not a same-named attribute.""" + # importlib.metadata is already loaded and has a `metadata` attribute. + code = textwrap.dedent(""" + import importlib.metadata + import importlib.metadata as eager + + lazy import importlib.metadata as lazily + + assert lazily is eager, lazily + """) + assert_python_ok("-c", code) + + + def test_dotted_as_replays_lookups_on_custom_placeholder(self): + """A dotted lazy import as looks up its names on what the hook returned.""" + code = textwrap.dedent(""" + import builtins + import xml.dom + + # In a list, so the hook reading it does not resolve it. + placeholder = [__lazy_import__("xml")] + default = builtins.__lazy_import__ + builtins.__lazy_import__ = lambda *args: placeholder[0] + lazy import fake.dom as dom + builtins.__lazy_import__ = default + + assert dom is xml.dom, dom + """) + assert_python_ok("-c", code) + + + def test_empty_fromlist_placeholder_matches_no_fromlist(self): + """An empty fromlist behaves like None.""" + code = textwrap.dedent(""" + expected = "" + # In lists, so reading them does not resolve them. + for fromlist in (None, ()): + same = [__lazy_import__("xml.dom", fromlist=fromlist)] + assert repr(same[0]) == expected, (fromlist, repr(same[0])) + bare = [__lazy_import__("xml.dom")] + assert repr(bare[0]) == expected, repr(bare[0]) + """) + assert_python_ok("-c", code) + + + def test_empty_fromlist_preserved_for_custom_import(self): + code = textwrap.dedent(""" + import builtins + import types + + value = object() + module = types.SimpleNamespace(dom=value) + placeholder = [__lazy_import__("xml.dom", fromlist=())] + default_import = builtins.__import__ + default_lazy_import = builtins.__lazy_import__ + calls = [] + + def import_hook(name, globals, locals, fromlist, level): + assert name == "xml.dom", name + assert fromlist == (), fromlist + calls.append(fromlist) + return module + + builtins.__import__ = import_hook + assert placeholder[0].resolve() is module + builtins.__lazy_import__ = lambda *args: placeholder[0] + lazy import fake.dom as dom + assert dom is value + builtins.__import__ = default_import + builtins.__lazy_import__ = default_lazy_import + + assert calls == [(), ()], calls + """) + assert_python_ok("-c", code) + + + def test_dotted_as_replays_lookups_on_dotted_placeholder(self): + """A dotted lazy import as replays its names on the hook's package.""" + # importlib.metadata has a `metadata` attribute of its own, which the + # placeholder for importlib must not answer with. + for target in ("xml.dom", "importlib.metadata"): + with self.subTest(target=target): + leaf = target.rpartition(".")[2] + code = textwrap.dedent(f""" + import builtins + import sys + import {target} + + # In a list, so the hook reading it does not resolve it. + placeholder = [__lazy_import__("{target}", fromlist=())] + default = builtins.__lazy_import__ + builtins.__lazy_import__ = lambda *args: placeholder[0] + lazy import fake.{leaf} as {leaf} + builtins.__lazy_import__ = default + + name = repr(globals()["{leaf}"]) + assert name == "", name + assert {leaf} is sys.modules["{target}"], {leaf} + """) + assert_python_ok("-c", code) + + +class SubmoduleLazinessTests(unittest.TestCase): + """Tests that module-level lazy imports remain lazy until accessed.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_unaccessed_imports_stay_lazy(self): + """Imports in 'all' mode should stay lazy until accessed.""" + sys.set_lazy_imports("all") + from test.test_lazy_import.data.metasyntactic import names + self.assertIsInstance(names.__dict__["Foo"], types.LazyImportType) + self.assertNotIn( + "test.test_lazy_import.data.metasyntactic.foo", sys.modules + ) + _ = names.Foo + self.assertEqual(names.Foo, "Foo") + self.assertIn( + "test.test_lazy_import.data.metasyntactic.foo", sys.modules + ) + self.assertIsInstance(names.__dict__["Ack"], types.LazyImportType) + self.assertNotIn( + "test.test_lazy_import.data.metasyntactic.foo.ack", sys.modules + ) + + +class AttributeSideEffectTests(unittest.TestCase): + """Tests that submodule imports don't overwrite parent attributes.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_version_submodule_does_not_overwrite(self): + """A __version__ submodule should not overwrite the parent's + __version__ attribute imported in __init__.py.""" + import test.test_lazy_import.data.versioned as versioned + self.assertEqual(versioned.__version__, "1.0") + self.assertEqual( + versioned.__copyright__, + "Copyright (c) 2001-2022 Python Software Foundation.", + ) + + +class ModuleVariableNameCollisionTests(unittest.TestCase): + """Tests for name collision between a submodule and a variable.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_variable_after_import_wins(self): + """Variable assigned after import should overwrite the submodule.""" + from test.test_lazy_import.data import module_same_name_var_order1 + self.assertEqual(module_same_name_var_order1.bar, "Blah") + + def test_import_after_variable_wins(self): + """Import after variable assignment should overwrite the variable.""" + from test.test_lazy_import.data import module_same_name_var_order2 + bar_mod = sys.modules[ + "test.test_lazy_import.data.module_same_name_var_order2.bar" + ] + self.assertIs(module_same_name_var_order2.bar, bar_mod) + + +class DeletedModuleReimportTests(unittest.TestCase): + """Tests for reimporting after module deletion from sys.modules.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_reimport_creates_new_module(self): + """Deleting and reimporting should create a new module object.""" + import test.test_lazy_import.data.metasyntactic.foo + import test.test_lazy_import.data.metasyntactic.foo.bar.baz + + first_bar = test.test_lazy_import.data.metasyntactic.foo.bar + + del sys.modules[ + "test.test_lazy_import.data.metasyntactic.foo.bar" + ] + + import test.test_lazy_import.data.metasyntactic.foo.bar.thud + + second_bar = test.test_lazy_import.data.metasyntactic.foo.bar + + self.assertIsNot(first_bar, second_bar) + self.assertIn("baz", dir(first_bar)) + self.assertNotIn("thud", dir(first_bar)) + self.assertIn("thud", dir(second_bar)) + self.assertNotIn("baz", dir(second_bar)) + + +@support.requires_subprocess() +class CircularImportLazyTests(unittest.TestCase): + """Tests that lazy imports can break circular import patterns.""" + + def test_succeeds_with_lazy(self): + """Same-level circular imports should succeed with lazy mode.""" + proc = assert_python_ok( + "-X", "lazy_imports=all", "-c", + "import test.test_lazy_import.data.circular_import_pkg.main;" + "print('OK')", + ) + self.assertIn(b"OK", proc.out) + + def test_fails_without_lazy(self): + """Same-level circular imports should fail without lazy mode.""" + result = subprocess.run( + [sys.executable, "-c", + "import test.test_lazy_import.data.circular_import_pkg.main"], + capture_output=True, text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("ImportError", result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_lazy_import/__main__.py b/Lib/test/test_lazy_import/__main__.py new file mode 100644 index 00000000000..d6c94efaf30 --- /dev/null +++ b/Lib/test/test_lazy_import/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_lazy_import') diff --git a/Lib/test/test_lazy_import/data/badsyntax/lazy_class_body.py b/Lib/test/test_lazy_import/data/badsyntax/lazy_class_body.py new file mode 100644 index 00000000000..e154b78f6cd --- /dev/null +++ b/Lib/test/test_lazy_import/data/badsyntax/lazy_class_body.py @@ -0,0 +1,3 @@ +# SyntaxError: lazy import inside class body is not allowed +class Foo: + lazy import json diff --git a/Lib/test/test_lazy_import/data/badsyntax/lazy_future_import.py b/Lib/test/test_lazy_import/data/badsyntax/lazy_future_import.py new file mode 100644 index 00000000000..8bd258b76b4 --- /dev/null +++ b/Lib/test/test_lazy_import/data/badsyntax/lazy_future_import.py @@ -0,0 +1 @@ +lazy from __future__ import annotations diff --git a/Lib/test/test_lazy_import/data/badsyntax/lazy_import_func.py b/Lib/test/test_lazy_import/data/badsyntax/lazy_import_func.py new file mode 100644 index 00000000000..af758b51127 --- /dev/null +++ b/Lib/test/test_lazy_import/data/badsyntax/lazy_import_func.py @@ -0,0 +1,2 @@ +def f(): + lazy import foo diff --git a/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except.py b/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except.py new file mode 100644 index 00000000000..e58d1f929ca --- /dev/null +++ b/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except.py @@ -0,0 +1,4 @@ +try: + lazy import foo +except: + pass diff --git a/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except_from.py b/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except_from.py new file mode 100644 index 00000000000..8c97553f486 --- /dev/null +++ b/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except_from.py @@ -0,0 +1,4 @@ +try: + lazy from foo import bar +except: + pass diff --git a/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except_from_star.py b/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except_from_star.py new file mode 100644 index 00000000000..b2370eaae77 --- /dev/null +++ b/Lib/test/test_lazy_import/data/badsyntax/lazy_try_except_from_star.py @@ -0,0 +1 @@ +lazy from foo import * diff --git a/Lib/test/test_lazy_import/data/basic2.py b/Lib/test/test_lazy_import/data/basic2.py new file mode 100644 index 00000000000..f93ec89d5ab --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic2.py @@ -0,0 +1,4 @@ +def f(): + pass + +x = 42 diff --git a/Lib/test/test_lazy_import/data/basic_compatibility_mode.py b/Lib/test/test_lazy_import/data/basic_compatibility_mode.py new file mode 100644 index 00000000000..33eeab8879c --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic_compatibility_mode.py @@ -0,0 +1,2 @@ +__lazy_modules__ = ['test.test_lazy_import.data.basic2'] +import test.test_lazy_import.data.basic2 diff --git a/Lib/test/test_lazy_import/data/basic_compatibility_mode_relative.py b/Lib/test/test_lazy_import/data/basic_compatibility_mode_relative.py new file mode 100644 index 00000000000..15c1ea77630 --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic_compatibility_mode_relative.py @@ -0,0 +1,2 @@ +__lazy_modules__ = ['test.test_lazy_import.data.basic2'] +lazy from .basic2 import f diff --git a/Lib/test/test_lazy_import/data/basic_compatibility_mode_used.py b/Lib/test/test_lazy_import/data/basic_compatibility_mode_used.py new file mode 100644 index 00000000000..78bb4ffa523 --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic_compatibility_mode_used.py @@ -0,0 +1,3 @@ +__lazy_modules__ = ['test.test_lazy_import.data.basic2'] +import test.test_lazy_import.data.basic2 +test.test_lazy_import.data.basic2.f() diff --git a/Lib/test/test_lazy_import/data/basic_dir.py b/Lib/test/test_lazy_import/data/basic_dir.py new file mode 100644 index 00000000000..a88c8ed80a1 --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic_dir.py @@ -0,0 +1,2 @@ +lazy import test.test_lazy_import.data.basic2 +x = dir() diff --git a/Lib/test/test_lazy_import/data/basic_from_unused.py b/Lib/test/test_lazy_import/data/basic_from_unused.py new file mode 100644 index 00000000000..62fa04bc8fa --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic_from_unused.py @@ -0,0 +1 @@ +lazy from test.test_lazy_import.data import basic2 diff --git a/Lib/test/test_lazy_import/data/basic_unused.py b/Lib/test/test_lazy_import/data/basic_unused.py new file mode 100644 index 00000000000..f3e502c59e7 --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic_unused.py @@ -0,0 +1 @@ +lazy import test.test_lazy_import.data.basic2 diff --git a/Lib/test/test_lazy_import/data/basic_used.py b/Lib/test/test_lazy_import/data/basic_used.py new file mode 100644 index 00000000000..7234bd2fe02 --- /dev/null +++ b/Lib/test/test_lazy_import/data/basic_used.py @@ -0,0 +1,3 @@ +lazy import test.test_lazy_import.data.basic2 as basic2 + +basic2.f() diff --git a/Lib/test/test_lazy_import/data/broken_attr_module.py b/Lib/test/test_lazy_import/data/broken_attr_module.py new file mode 100644 index 00000000000..a60bca2bad0 --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_attr_module.py @@ -0,0 +1,3 @@ +# Module that exists but doesn't have expected attributes +x = 42 +# No 'nonexistent_attr' here diff --git a/Lib/test/test_lazy_import/data/broken_module.py b/Lib/test/test_lazy_import/data/broken_module.py new file mode 100644 index 00000000000..b49d4a4a15f --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_module.py @@ -0,0 +1,2 @@ +# Module that raises an error during import +raise ValueError("This module always fails to import") diff --git a/Lib/test/test_lazy_import/data/broken_module_chained_cause.py b/Lib/test/test_lazy_import/data/broken_module_chained_cause.py new file mode 100644 index 00000000000..786607b7bb5 --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_module_chained_cause.py @@ -0,0 +1,3 @@ +# Module that raises an exception with explicit cause during import +cause = ValueError("Cause of failure") +raise ValueError("This module always fails to import") from cause diff --git a/Lib/test/test_lazy_import/data/broken_module_chained_context.py b/Lib/test/test_lazy_import/data/broken_module_chained_context.py new file mode 100644 index 00000000000..82640856e3e --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_module_chained_context.py @@ -0,0 +1,5 @@ +# Module that raises an exception with context during import +try: + raise ValueError("Cause of failure") +except: + raise ValueError("This module always fails to import") diff --git a/Lib/test/test_lazy_import/data/broken_module_chained_suppressed.py b/Lib/test/test_lazy_import/data/broken_module_chained_suppressed.py new file mode 100644 index 00000000000..5c34654afc1 --- /dev/null +++ b/Lib/test/test_lazy_import/data/broken_module_chained_suppressed.py @@ -0,0 +1,2 @@ +# Module that raises an exception with suppressed context during import +raise ValueError("This module always fails to import") from None diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/__init__.py b/Lib/test/test_lazy_import/data/circular_import_pkg/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/main.py b/Lib/test/test_lazy_import/data/circular_import_pkg/main.py new file mode 100644 index 00000000000..0b897b377e2 --- /dev/null +++ b/Lib/test/test_lazy_import/data/circular_import_pkg/main.py @@ -0,0 +1,2 @@ +from .x import X2 +X2() diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/x.py b/Lib/test/test_lazy_import/data/circular_import_pkg/x.py new file mode 100644 index 00000000000..b44d5185c9e --- /dev/null +++ b/Lib/test/test_lazy_import/data/circular_import_pkg/x.py @@ -0,0 +1,7 @@ +def X1(): + return "X" + +from .y import Y1 + +def X2(): + return Y1() diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/y.py b/Lib/test/test_lazy_import/data/circular_import_pkg/y.py new file mode 100644 index 00000000000..d2385f7e67c --- /dev/null +++ b/Lib/test/test_lazy_import/data/circular_import_pkg/y.py @@ -0,0 +1,7 @@ +def Y1(): + return "Y" + +from .x import X2 + +def Y2(): + return X2() diff --git a/Lib/test/test_lazy_import/data/compatibility_mode_func.py b/Lib/test/test_lazy_import/data/compatibility_mode_func.py new file mode 100644 index 00000000000..9405d2f832b --- /dev/null +++ b/Lib/test/test_lazy_import/data/compatibility_mode_func.py @@ -0,0 +1,5 @@ +__lazy_modules__ = ['test.test_lazy_import.data.basic2'] +def f(): + import test.test_lazy_import.data.basic2 + +f() diff --git a/Lib/test/test_lazy_import/data/compatibility_mode_try_except.py b/Lib/test/test_lazy_import/data/compatibility_mode_try_except.py new file mode 100644 index 00000000000..a746c4a34a5 --- /dev/null +++ b/Lib/test/test_lazy_import/data/compatibility_mode_try_except.py @@ -0,0 +1,5 @@ +__lazy_modules__ = ['test.test_lazy_import.data.basic2'] +try: + import test.test_lazy_import.data.basic2 +except: + pass diff --git a/Lib/test/test_lazy_import/data/dunder_lazy_import.py b/Lib/test/test_lazy_import/data/dunder_lazy_import.py new file mode 100644 index 00000000000..c4f4d61623f --- /dev/null +++ b/Lib/test/test_lazy_import/data/dunder_lazy_import.py @@ -0,0 +1 @@ +basic = __lazy_import__('test.test_lazy_import.data.basic2') diff --git a/Lib/test/test_lazy_import/data/dunder_lazy_import_builtins.py b/Lib/test/test_lazy_import/data/dunder_lazy_import_builtins.py new file mode 100644 index 00000000000..8b6dbd7e3f6 --- /dev/null +++ b/Lib/test/test_lazy_import/data/dunder_lazy_import_builtins.py @@ -0,0 +1,14 @@ +import sys + +def myimport(*args): + return sys.modules[__name__] + + +new_globals = dict(globals()) +new_globals["__builtins__"] = { + "__import__": myimport, +} +basic2 = 42 +basic = __lazy_import__("test.test_lazy_import.data.basic2", + globals=new_globals) +basic diff --git a/Lib/test/test_lazy_import/data/dunder_lazy_import_used.py b/Lib/test/test_lazy_import/data/dunder_lazy_import_used.py new file mode 100644 index 00000000000..42c026994b1 --- /dev/null +++ b/Lib/test/test_lazy_import/data/dunder_lazy_import_used.py @@ -0,0 +1,3 @@ +basic = __lazy_import__('test.test_lazy_import.data', + fromlist=("basic2", )) +basic diff --git a/Lib/test/test_lazy_import/data/eager_import_func.py b/Lib/test/test_lazy_import/data/eager_import_func.py new file mode 100644 index 00000000000..1f7130209d4 --- /dev/null +++ b/Lib/test/test_lazy_import/data/eager_import_func.py @@ -0,0 +1,3 @@ +def f(): + import test.test_lazy_import.data.basic2 as basic2 + return basic2 diff --git a/Lib/test/test_lazy_import/data/global_filter.py b/Lib/test/test_lazy_import/data/global_filter.py new file mode 100644 index 00000000000..0e2311842df --- /dev/null +++ b/Lib/test/test_lazy_import/data/global_filter.py @@ -0,0 +1,10 @@ +import sys + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_lazy_import.data.basic2" + return False + +sys.set_lazy_imports_filter(filter) + +lazy import test.test_lazy_import.data.basic2 as basic2 diff --git a/Lib/test/test_lazy_import/data/global_filter_from.py b/Lib/test/test_lazy_import/data/global_filter_from.py new file mode 100644 index 00000000000..da9d22e8727 --- /dev/null +++ b/Lib/test/test_lazy_import/data/global_filter_from.py @@ -0,0 +1,11 @@ +import sys + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_lazy_import.data.basic2" + assert from_list == ('f',) + return False + +sys.set_lazy_imports_filter(filter) + +lazy from test.test_lazy_import.data.basic2 import f diff --git a/Lib/test/test_lazy_import/data/global_filter_from_true.py b/Lib/test/test_lazy_import/data/global_filter_from_true.py new file mode 100644 index 00000000000..2d8b1de4c7c --- /dev/null +++ b/Lib/test/test_lazy_import/data/global_filter_from_true.py @@ -0,0 +1,12 @@ +import sys + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_lazy_import.data.basic2" + assert from_list == ('f',) + return True + +sys.set_lazy_imports("normal") +sys.set_lazy_imports_filter(filter) + +lazy from test.test_lazy_import.data.basic2 import f diff --git a/Lib/test/test_lazy_import/data/global_filter_true.py b/Lib/test/test_lazy_import/data/global_filter_true.py new file mode 100644 index 00000000000..da4abeacf87 --- /dev/null +++ b/Lib/test/test_lazy_import/data/global_filter_true.py @@ -0,0 +1,11 @@ +import sys + +def filter(module_name, imported_name, from_list): + assert module_name == __name__ + assert imported_name == "test.test_lazy_import.data.basic2" + return True + +sys.set_lazy_imports("normal") +sys.set_lazy_imports_filter(filter) + +lazy import test.test_lazy_import.data.basic2 as basic2 diff --git a/Lib/test/test_lazy_import/data/global_on.py b/Lib/test/test_lazy_import/data/global_on.py new file mode 100644 index 00000000000..ddacab5468b --- /dev/null +++ b/Lib/test/test_lazy_import/data/global_on.py @@ -0,0 +1,5 @@ +import sys + +sys.set_lazy_imports("all") + +import test.test_lazy_import.data.basic2 as basic2 diff --git a/Lib/test/test_lazy_import/data/globals_access.py b/Lib/test/test_lazy_import/data/globals_access.py new file mode 100644 index 00000000000..af6fc40115b --- /dev/null +++ b/Lib/test/test_lazy_import/data/globals_access.py @@ -0,0 +1,9 @@ +# Test that globals() returns lazy proxy objects without reifying +lazy import test.test_lazy_import.data.basic2 as basic2 + +def get_from_globals(): + g = globals() + return g['basic2'] + +def get_direct(): + return basic2 diff --git a/Lib/test/test_lazy_import/data/init_fails.py b/Lib/test/test_lazy_import/data/init_fails.py new file mode 100644 index 00000000000..16e7ad363b0 --- /dev/null +++ b/Lib/test/test_lazy_import/data/init_fails.py @@ -0,0 +1,4 @@ +# Imported by test_initializing_module_is_still_tracked. The module it imports +# lazily imports this one back while this one is still initializing. +import test.test_lazy_import.data.lazy_on_init_fails +raise ValueError("initialization failed") diff --git a/Lib/test/test_lazy_import/data/lazy_compat_from.py b/Lib/test/test_lazy_import/data/lazy_compat_from.py new file mode 100644 index 00000000000..d6f7ed9a785 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_compat_from.py @@ -0,0 +1,6 @@ +# Test __lazy_modules__ with from imports +__lazy_modules__ = ['test.test_lazy_import.data.basic2'] +from test.test_lazy_import.data.basic2 import x, f + +def get_x(): + return x diff --git a/Lib/test/test_lazy_import/data/lazy_get_value.py b/Lib/test/test_lazy_import/data/lazy_get_value.py new file mode 100644 index 00000000000..f216dc52f29 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_get_value.py @@ -0,0 +1,7 @@ +lazy import test.test_lazy_import.data.basic2 as basic2 + +def f(): + x = globals() + return x['basic2'].resolve() + +f() diff --git a/Lib/test/test_lazy_import/data/lazy_import_pkg.py b/Lib/test/test_lazy_import/data/lazy_import_pkg.py new file mode 100644 index 00000000000..f237b50fd42 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_import_pkg.py @@ -0,0 +1,2 @@ +lazy import test.test_lazy_import.data.pkg.bar +x = test.test_lazy_import.data.pkg.bar.f diff --git a/Lib/test/test_lazy_import/data/lazy_on_init_fails.py b/Lib/test/test_lazy_import/data/lazy_on_init_fails.py new file mode 100644 index 00000000000..7203fcbb4aa --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_on_init_fails.py @@ -0,0 +1 @@ +lazy import test.test_lazy_import.data.init_fails diff --git a/Lib/test/test_lazy_import/data/lazy_with.py b/Lib/test/test_lazy_import/data/lazy_with.py new file mode 100644 index 00000000000..30cf9d37762 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_with.py @@ -0,0 +1,3 @@ +import contextlib +with contextlib.nullcontext(): + lazy import test.test_lazy_import.data.basic2 diff --git a/Lib/test/test_lazy_import/data/lazy_with_from.py b/Lib/test/test_lazy_import/data/lazy_with_from.py new file mode 100644 index 00000000000..794ba33f828 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazy_with_from.py @@ -0,0 +1,3 @@ +import contextlib +with contextlib.nullcontext(): + lazy import test.test_lazy_import.data.basic2 as basic2 diff --git a/Lib/test/test_lazy_import/data/lazypkg/__init__.py b/Lib/test/test_lazy_import/data/lazypkg/__init__.py new file mode 100644 index 00000000000..276b51823fe --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazypkg/__init__.py @@ -0,0 +1 @@ +lazy from . import bar diff --git a/Lib/test/test_lazy_import/data/lazypkg/bar.py b/Lib/test/test_lazy_import/data/lazypkg/bar.py new file mode 100644 index 00000000000..b8d8b60b886 --- /dev/null +++ b/Lib/test/test_lazy_import/data/lazypkg/bar.py @@ -0,0 +1,2 @@ +print("BAR_MODULE_LOADED") +def f(): pass diff --git a/Lib/test/test_lazy_import/data/metasyntactic/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/__init__.py new file mode 100644 index 00000000000..632a9fb87ac --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/__init__.py @@ -0,0 +1 @@ +Foo = "Foo" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/ack/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/ack/__init__.py new file mode 100644 index 00000000000..c4c249f3b65 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/ack/__init__.py @@ -0,0 +1 @@ +Ack = "Ack" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/__init__.py new file mode 100644 index 00000000000..03faa550258 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/__init__.py @@ -0,0 +1 @@ +Bar = "Bar" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/__init__.py new file mode 100644 index 00000000000..62d8366b440 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/__init__.py @@ -0,0 +1 @@ +Baz = "Baz" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/qux/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/qux/__init__.py new file mode 100644 index 00000000000..12389d76a28 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/qux/__init__.py @@ -0,0 +1 @@ +Qux = "Qux" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/thud/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/thud/__init__.py new file mode 100644 index 00000000000..a4dd8ab0fae --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/thud/__init__.py @@ -0,0 +1 @@ +Thud = "Thud" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/names.py b/Lib/test/test_lazy_import/data/metasyntactic/names.py new file mode 100644 index 00000000000..b2edf13e850 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/names.py @@ -0,0 +1,9 @@ +from .foo import Foo +from .foo.ack import Ack +from .foo.bar import Bar +from .foo.bar.baz import Baz +from .foo.bar.thud import Thud +from .waldo import Waldo +from .waldo.fred import Fred +from .plugh import Plugh +Metasyntactic = "Metasyntactic" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/plugh/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/plugh/__init__.py new file mode 100644 index 00000000000..5db053204fc --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/plugh/__init__.py @@ -0,0 +1 @@ +Plugh = "Plugh" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/waldo/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/waldo/__init__.py new file mode 100644 index 00000000000..39f5a1b1838 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/waldo/__init__.py @@ -0,0 +1 @@ +Waldo = "Waldo" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/waldo/fred/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/waldo/fred/__init__.py new file mode 100644 index 00000000000..8c9dce1b3cf --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/waldo/fred/__init__.py @@ -0,0 +1 @@ +Fred = "Fred" diff --git a/Lib/test/test_lazy_import/data/missing_dependency.py b/Lib/test/test_lazy_import/data/missing_dependency.py new file mode 100644 index 00000000000..aa2f2d0f65f --- /dev/null +++ b/Lib/test/test_lazy_import/data/missing_dependency.py @@ -0,0 +1 @@ +import missing_dependency_for_lazy_import_test diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order1/__init__.py b/Lib/test/test_lazy_import/data/module_same_name_var_order1/__init__.py new file mode 100644 index 00000000000..b815b2a7d6f --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order1/__init__.py @@ -0,0 +1,3 @@ +from .bar import Bar +bar = "Blah" +Bar diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order1/bar.py b/Lib/test/test_lazy_import/data/module_same_name_var_order1/bar.py new file mode 100644 index 00000000000..03faa550258 --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order1/bar.py @@ -0,0 +1 @@ +Bar = "Bar" diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order2/__init__.py b/Lib/test/test_lazy_import/data/module_same_name_var_order2/__init__.py new file mode 100644 index 00000000000..008d199c73e --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order2/__init__.py @@ -0,0 +1,3 @@ +bar = "Blah" +from .bar import Bar +Bar diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order2/bar.py b/Lib/test/test_lazy_import/data/module_same_name_var_order2/bar.py new file mode 100644 index 00000000000..03faa550258 --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order2/bar.py @@ -0,0 +1 @@ +Bar = "Bar" diff --git a/Lib/test/test_lazy_import/data/module_with_getattr.py b/Lib/test/test_lazy_import/data/module_with_getattr.py new file mode 100644 index 00000000000..db3a2301075 --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_with_getattr.py @@ -0,0 +1,12 @@ +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/modules_dict.py b/Lib/test/test_lazy_import/data/modules_dict.py new file mode 100644 index 00000000000..562a3b9a62f --- /dev/null +++ b/Lib/test/test_lazy_import/data/modules_dict.py @@ -0,0 +1,5 @@ +lazy import test.test_lazy_import.data.basic2 as basic2 + +import sys +mod = sys.modules[__name__] +x = mod.__dict__ diff --git a/Lib/test/test_lazy_import/data/modules_getattr.py b/Lib/test/test_lazy_import/data/modules_getattr.py new file mode 100644 index 00000000000..f0b8e2bd061 --- /dev/null +++ b/Lib/test/test_lazy_import/data/modules_getattr.py @@ -0,0 +1,5 @@ +lazy import test.test_lazy_import.data.basic2 as basic2 + +import sys +mod = sys.modules[__name__] +x = mod.basic2 diff --git a/Lib/test/test_lazy_import/data/modules_getattr_other.py b/Lib/test/test_lazy_import/data/modules_getattr_other.py new file mode 100644 index 00000000000..642840c2872 --- /dev/null +++ b/Lib/test/test_lazy_import/data/modules_getattr_other.py @@ -0,0 +1,5 @@ +lazy import test.test_lazy_import.data.basic2 as basic2 + +import sys +mod = sys.modules[__name__] +x = mod.__name__ diff --git a/Lib/test/test_lazy_import/data/multi_from_import.py b/Lib/test/test_lazy_import/data/multi_from_import.py new file mode 100644 index 00000000000..dfd875975dd --- /dev/null +++ b/Lib/test/test_lazy_import/data/multi_from_import.py @@ -0,0 +1,5 @@ +# Test that lazy from import with multiple names only reifies accessed names +lazy from test.test_lazy_import.data.basic2 import f, x + +def get_globals(): + return globals() diff --git a/Lib/test/test_lazy_import/data/pkg/__init__.py b/Lib/test/test_lazy_import/data/pkg/__init__.py new file mode 100644 index 00000000000..5f7b8662596 --- /dev/null +++ b/Lib/test/test_lazy_import/data/pkg/__init__.py @@ -0,0 +1,14 @@ +x = 42 + +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/pkg/b.py b/Lib/test/test_lazy_import/data/pkg/b.py new file mode 100644 index 00000000000..a266b7c7c0d --- /dev/null +++ b/Lib/test/test_lazy_import/data/pkg/b.py @@ -0,0 +1,2 @@ +def foo(): + return 'foo' diff --git a/Lib/test/test_lazy_import/data/pkg/bar.py b/Lib/test/test_lazy_import/data/pkg/bar.py new file mode 100644 index 00000000000..b8d8b60b886 --- /dev/null +++ b/Lib/test/test_lazy_import/data/pkg/bar.py @@ -0,0 +1,2 @@ +print("BAR_MODULE_LOADED") +def f(): pass diff --git a/Lib/test/test_lazy_import/data/pkg/broken.py b/Lib/test/test_lazy_import/data/pkg/broken.py new file mode 100644 index 00000000000..6499aa9e150 --- /dev/null +++ b/Lib/test/test_lazy_import/data/pkg/broken.py @@ -0,0 +1,2 @@ +# Submodule that raises an error during import +raise ValueError("This module always fails to import") diff --git a/Lib/test/test_lazy_import/data/pkg/c.py b/Lib/test/test_lazy_import/data/pkg/c.py new file mode 100644 index 00000000000..0bb03119864 --- /dev/null +++ b/Lib/test/test_lazy_import/data/pkg/c.py @@ -0,0 +1,4 @@ +lazy from . import b, x + +def get_globals(): + return globals() diff --git a/Lib/test/test_lazy_import/data/relative_lazy.py b/Lib/test/test_lazy_import/data/relative_lazy.py new file mode 100644 index 00000000000..6273d3883ab --- /dev/null +++ b/Lib/test/test_lazy_import/data/relative_lazy.py @@ -0,0 +1,5 @@ +# Test relative imports with lazy keyword +lazy from . import basic2 + +def get_basic2(): + return basic2 diff --git a/Lib/test/test_lazy_import/data/relative_lazy_from.py b/Lib/test/test_lazy_import/data/relative_lazy_from.py new file mode 100644 index 00000000000..1bae2d6853d --- /dev/null +++ b/Lib/test/test_lazy_import/data/relative_lazy_from.py @@ -0,0 +1,8 @@ +# Test relative from imports with lazy keyword +lazy from .basic2 import x, f + +def get_x(): + return x + +def get_f(): + return f diff --git a/Lib/test/test_lazy_import/data/self_named_module_not_found.py b/Lib/test/test_lazy_import/data/self_named_module_not_found.py new file mode 100644 index 00000000000..941230d5da2 --- /dev/null +++ b/Lib/test/test_lazy_import/data/self_named_module_not_found.py @@ -0,0 +1 @@ +raise ModuleNotFoundError("boom", name=__name__) diff --git a/Lib/test/test_lazy_import/data/try_except_eager.py b/Lib/test/test_lazy_import/data/try_except_eager.py new file mode 100644 index 00000000000..90b4bc62898 --- /dev/null +++ b/Lib/test/test_lazy_import/data/try_except_eager.py @@ -0,0 +1,4 @@ +try: + import test.test_lazy_import.data.basic2 +except: + pass diff --git a/Lib/test/test_lazy_import/data/try_except_eager_from.py b/Lib/test/test_lazy_import/data/try_except_eager_from.py new file mode 100644 index 00000000000..1e6c650d0d1 --- /dev/null +++ b/Lib/test/test_lazy_import/data/try_except_eager_from.py @@ -0,0 +1,4 @@ +try: + from test.test_lazy_import.data.basic2 import f +except: + pass diff --git a/Lib/test/test_lazy_import/data/versioned/__init__.py b/Lib/test/test_lazy_import/data/versioned/__init__.py new file mode 100644 index 00000000000..be5bb3f8e60 --- /dev/null +++ b/Lib/test/test_lazy_import/data/versioned/__init__.py @@ -0,0 +1,2 @@ +from .__version__ import __version__ +from .__version__ import __copyright__ diff --git a/Lib/test/test_lazy_import/data/versioned/__version__.py b/Lib/test/test_lazy_import/data/versioned/__version__.py new file mode 100644 index 00000000000..37a10438735 --- /dev/null +++ b/Lib/test/test_lazy_import/data/versioned/__version__.py @@ -0,0 +1,2 @@ +__version__ = "1.0" +__copyright__ = "Copyright (c) 2001-2022 Python Software Foundation." diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 6649de68285..252a3a77981 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1233,6 +1233,20 @@ def test_multiline_non_ascii_fstring_with_expr(self): FSTRING_END \'"\' (2, 2) (2, 3) """) + def test_ineq_tokens(self): + self.check_tokenize("1 != 2", """\ + NUMBER '1' (1, 0) (1, 1) + OP '!=' (1, 2) (1, 4) + NUMBER '2' (1, 5) (1, 6) +""") + self.check_tokenize("1 <> 2", """\ + NUMBER '1' (1, 0) (1, 1) + OP '<' (1, 2) (1, 3) + OP '>' (1, 3) (1, 4) + NUMBER '2' (1, 5) (1, 6) +""") + + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. @@ -3332,6 +3346,7 @@ def test_newline_at_the_end_of_buffer(self): run_test_script(file_name) +@support.force_not_colorized_test_class class CommandLineTest(unittest.TestCase): def setUp(self): self.filename = tempfile.mktemp() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 88e39145c9b..fa5f1e30355 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -9,18 +9,21 @@ import builtins import unittest import unittest.mock +import os import re import tempfile import random import string +import importlib.machinery +import sysconfig from test import support import shutil from test.support import (Error, captured_output, cpython_only, ALWAYS_EQ, requires_debug_ranges, has_no_debug_ranges, - requires_subprocess) + requires_subprocess, os_helper) from test.support.os_helper import TESTFN, temp_dir, unlink from test.support.script_helper import assert_python_ok, assert_python_failure, make_script -from test.support.import_helper import forget +from test.support.import_helper import ensure_lazy_imports, forget from test.support import force_not_colorized, force_not_colorized_test_class import json @@ -37,7 +40,7 @@ test_frame = namedtuple('frame', ['f_code', 'f_globals', 'f_locals']) test_tb = namedtuple('tb', ['tb_frame', 'tb_lineno', 'tb_next', 'tb_lasti']) -color_overrides = {"reset": "z", "filename": "fn", "error_highlight": "E"} +color_overrides = {"reset": "z", "filename": "fn", "error_highlight": "E", "note": "n"} colors = { color_overrides.get(k, k[0].lower()): v for k, v in _colorize.default_theme.traceback.items() @@ -88,6 +91,12 @@ def syntax_error_bad_indentation2(self): def tokenizer_error_with_caret_range(self): compile("blech ( ", "?", "exec") + def syntax_error_with_caret_wide_char(self): + compile("女女女=1; 女女女/", "?", "exec") + + def syntax_error_with_caret_wide_char_range(self): + compile("f(x, 女女女 for 女女女 in range(30), z)", "?", "exec") + def test_caret(self): err = self.get_exception_format(self.syntax_error_with_caret, SyntaxError) @@ -125,6 +134,20 @@ def test_caret(self): self.assertEqual(err[1].find("("), err[2].find("^")) # in the right place self.assertEqual(err[2].count("^"), 1) + def test_caret_wide_char(self): + err = self.get_exception_format(self.syntax_error_with_caret_wide_char, + SyntaxError) + self.assertIn("^", err[2]) + # "女女女=1; 女女女/" has display width 17 + self.assertEqual(err[2].find("^"), 4 + 17) + + err = self.get_exception_format(self.syntax_error_with_caret_wide_char_range, + SyntaxError) + self.assertIn("^", err[2]) + self.assertEqual(err[2].find("^"), 4 + 5) + # "女女女 for 女女女 in range(30)" has display width 30 + self.assertEqual(err[2].count("^"), 30) + def test_nocaret(self): exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) err = traceback.format_exception_only(SyntaxError, exc) @@ -1767,6 +1790,7 @@ def f(): ] self.assertEqual(result_lines, expected) + class TestKeywordTypoSuggestions(unittest.TestCase): TYPO_CASES = [ ("with block ad something:\n pass", "and"), @@ -1811,6 +1835,23 @@ def test_keyword_suggestions_from_command_string(self): stderr_text = stderr.decode('utf-8') self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + def test_no_keyword_suggestion_for_comma_errors(self): + # When the parser identifies a missing comma, don't suggest + # bogus keyword replacements like 'print' -> 'not' + code = '''\ +import sys +print( + "line1" + "line2" + file=sys.stderr +) +''' + source = textwrap.dedent(code).strip() + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn("Perhaps you forgot a comma", stderr_text) + self.assertNotIn("Did you mean", stderr_text) + @requires_debug_ranges() @force_not_colorized_test_class class PurePythonTracebackErrorCaretTests( @@ -3852,7 +3893,7 @@ def f(): def test_dont_swallow_cause_or_context_of_falsey_exception(self): # see gh-132308: Ensure that __cause__ or __context__ attributes of exceptions # that evaluate as falsey are included in the output. For falsey term, - # see https://docs.python.org/3/library/stdtypes.html#truth-value-testing. + # see https://docs.python.org/3/builtins/stdtypes.html#truth-value-testing. try: raise FalseyException from KeyError @@ -4071,7 +4112,7 @@ def test_comparison(self): def test_dont_swallow_subexceptions_of_falsey_exceptiongroup(self): # see gh-132308: Ensure that subexceptions of exception groups # that evaluate as falsey are displayed in the output. For falsey term, - # see https://docs.python.org/3/library/stdtypes.html#truth-value-testing. + # see https://docs.python.org/3/builtins/stdtypes.html#truth-value-testing. try: raise FalseyExceptionGroup("Gih", (KeyError(), NameError())) @@ -4092,11 +4133,13 @@ def test_dont_swallow_subexceptions_of_falsey_exceptiongroup(self): global_for_suggestions = None -class SuggestionFormattingTestBase: +class SuggestionFormattingTestMixin: + attr_function = getattr + def get_suggestion(self, obj, attr_name=None): if attr_name is not None: def callable(): - getattr(obj, attr_name) + self.attr_function(obj, attr_name) else: callable = obj @@ -4105,7 +4148,9 @@ def callable(): ) return result_lines[0] - def test_getattr_suggestions(self): + +class BaseSuggestionTests(SuggestionFormattingTestMixin): + def test_suggestions(self): class Substitution: noise = more_noise = a = bc = None blech = None @@ -4136,40 +4181,41 @@ class CaseChangeOverSubstitution: BLuch = None for cls, suggestion in [ - (Addition, "'bluchin'?"), - (Substitution, "'blech'?"), - (Elimination, "'blch'?"), - (Addition, "'bluchin'?"), - (SubstitutionOverElimination, "'blach'?"), - (SubstitutionOverAddition, "'blach'?"), - (EliminationOverAddition, "'bluc'?"), - (CaseChangeOverSubstitution, "'BLuch'?"), + (Addition, "'.bluchin'"), + (Substitution, "'.blech'"), + (Elimination, "'.blch'"), + (Addition, "'.bluchin'"), + (SubstitutionOverElimination, "'.blach'"), + (SubstitutionOverAddition, "'.blach'"), + (EliminationOverAddition, "'.bluc'"), + (CaseChangeOverSubstitution, "'.BLuch'"), ]: actual = self.get_suggestion(cls(), 'bluch') - self.assertIn(suggestion, actual) + self.assertIn('Did you mean ' + suggestion, actual) - def test_getattr_suggestions_underscored(self): + def test_suggestions_underscored(self): class A: bluch = None - self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) - self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) - self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + self.assertIn("'.bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'.bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'.bluch'", self.get_suggestion(A(), '_bluch')) + attr_function = self.attr_function class B: _bluch = None def method(self, name): - getattr(self, name) + attr_function(self, name) - self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) - self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) - self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + self.assertIn("'._bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'._bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'._bluch'", self.get_suggestion(B(), 'bluch')) - self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) - self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) - self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + self.assertIn("'._bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'._bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'._bluch'", self.get_suggestion(partial(B().method, 'bluch'))) - def test_getattr_suggestions_with_custom___dir__(self): + def test_suggestions_with_custom___dir__(self): class M(type): def __dir__(cls): return [None, "fox"] @@ -4181,29 +4227,30 @@ def __dir__(self): class C1(C0, metaclass=M): pass - self.assertNotIn("'bluch'", self.get_suggestion(C0, "blach")) - self.assertIn("'bluch'", self.get_suggestion(C0(), "blach")) + self.assertNotIn("'.bluch'", self.get_suggestion(C0, "blach")) + self.assertIn("'.bluch'", self.get_suggestion(C0(), "blach")) - self.assertIn("'fox'", self.get_suggestion(C1, "foo")) - self.assertNotIn("'fox'", self.get_suggestion(C1(), "foo")) + self.assertIn("'.fox'", self.get_suggestion(C1, "foo")) + self.assertNotIn("'.fox'", self.get_suggestion(C1(), "foo")) - self.assertNotIn("'bluch'", self.get_suggestion(C1, "blach")) - self.assertIn("'bluch'", self.get_suggestion(C1(), "blach")) + self.assertNotIn("'.bluch'", self.get_suggestion(C1, "blach")) + self.assertIn("'.bluch'", self.get_suggestion(C1(), "blach")) - def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): + + def test_do_not_trigger_for_long_attributes(self): class A: blech = None actual = self.get_suggestion(A(), 'somethingverywrong') self.assertNotIn("blech", actual) - def test_getattr_error_bad_suggestions_do_not_trigger_for_small_names(self): + def test_do_not_trigger_for_small_names(self): class MyClass: vvv = mom = w = id = pytho = None for name in ("b", "v", "m", "py"): with self.subTest(name=name): - actual = self.get_suggestion(MyClass, name) + actual = self.get_suggestion(MyClass(), name) self.assertNotIn("Did you mean", actual) self.assertNotIn("'vvv", actual) self.assertNotIn("'mom'", actual) @@ -4211,7 +4258,7 @@ class MyClass: self.assertNotIn("'w'", actual) self.assertNotIn("'pytho'", actual) - def test_getattr_suggestions_do_not_trigger_for_big_dicts(self): + def test_do_not_trigger_for_big_dicts(self): class A: blech = None # A class with a very big __dict__ will not be considered @@ -4222,7 +4269,36 @@ class A: actual = self.get_suggestion(A(), 'bluch') self.assertNotIn("blech", actual) - def test_getattr_suggestions_no_args(self): + def test_suggestions_for_same_name(self): + class A: + def __dir__(self): + return ['blech'] + actual = self.get_suggestion(A(), 'blech') + self.assertNotIn("Did you mean", actual) + + def test_suggestions_not_normalized(self): + class A: + analization = None + fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = None + + suggestion = self.get_suggestion(A(), 'fiⁿₐˡᵢᶻₐᵗᵢᵒₙ') + self.assertIn("'.finalization'", suggestion) + self.assertNotIn("analization", suggestion) + + class B: + attr_a = None + attr_µ = None # attr_\xb5 + + suggestion = self.get_suggestion(B(), 'attr_\xb5') + self.assertIn( + "'.attr_\u03bc' ('attr_\\u03bc') " + "instead of '.attr_\xb5' ('attr_\\xb5')", + suggestion) + self.assertNotIn("attr_a", suggestion) + + +class GetattrSuggestionTests(BaseSuggestionTests): + def test_suggestions_no_args(self): class A: blech = None def __getattr__(self, attr): @@ -4239,7 +4315,7 @@ def __getattr__(self, attr): actual = self.get_suggestion(A(), 'bluch') self.assertIn("blech", actual) - def test_getattr_suggestions_invalid_args(self): + def test_suggestions_invalid_args(self): class NonStringifyClass: __str__ = None __repr__ = None @@ -4263,13 +4339,12 @@ def __getattr__(self, attr): actual = self.get_suggestion(cls(), 'bluch') self.assertIn("blech", actual) - def test_getattr_suggestions_for_same_name(self): - class A: - def __dir__(self): - return ['blech'] - actual = self.get_suggestion(A(), 'blech') - self.assertNotIn("Did you mean", actual) +class DelattrSuggestionTests(BaseSuggestionTests): + attr_function = delattr + + +class SuggestionFormattingTestBase(SuggestionFormattingTestMixin): def test_attribute_error_with_failing_dict(self): class T: bluch = 1 @@ -4311,6 +4386,275 @@ def __getattribute__(self, attr): self.assertIn("Did you mean", actual) self.assertIn("bluch", actual) + def test_getattr_nested_attribute_suggestions(self): + # Test that nested attributes are suggested when no direct match + class Inner: + def __init__(self): + self.value = 42 + self.data = "test" + + class Outer: + def __init__(self): + self.inner = Inner() + + # Should suggest 'inner.value' + actual = self.get_suggestion(Outer(), 'value') + self.assertIn("Did you mean '.inner.value' instead of '.value'", actual) + + # Should suggest 'inner.data' + actual = self.get_suggestion(Outer(), 'data') + self.assertIn("Did you mean '.inner.data' instead of '.data'", actual) + + def test_getattr_nested_prioritizes_direct_matches(self): + # Test that direct attribute matches are prioritized over nested ones + class Inner: + def __init__(self): + self.foo = 42 + + class Outer: + def __init__(self): + self.inner = Inner() + self.fooo = 100 # Similar to 'foo' + + # Should suggest 'fooo' (direct) not 'inner.foo' (nested) + actual = self.get_suggestion(Outer(), 'foo') + self.assertIn("Did you mean '.fooo'", actual) + self.assertNotIn("inner.foo", actual) + + def test_getattr_nested_with_property(self): + # Property suggestions should not execute the property getter. + class Inner: + @property + def computed(self): + return missing_name + + class Outer: + def __init__(self): + self.inner = Inner() + + actual = self.get_suggestion(Outer(), 'computed') + self.assertIn( + "Did you mean '.inner.computed' instead of '.computed'", + actual, + ) + + def test_getattr_nested_no_suggestion_for_deep_nesting(self): + # Test that deeply nested attributes (2+ levels) are not suggested + class Deep: + def __init__(self): + self.value = 42 + + class Middle: + def __init__(self): + self.deep = Deep() + + class Outer: + def __init__(self): + self.middle = Middle() + + # Should not suggest 'middle.deep.value' (too deep) + actual = self.get_suggestion(Outer(), 'value') + self.assertNotIn("Did you mean", actual) + + def test_getattr_nested_ignores_private_attributes(self): + # Test that nested suggestions ignore private attributes + class Inner: + def __init__(self): + self.public_value = 42 + + class Outer: + def __init__(self): + self._private_inner = Inner() + + # Should not suggest '_private_inner.public_value' + actual = self.get_suggestion(Outer(), 'public_value') + self.assertNotIn("Did you mean", actual) + + def test_getattr_nested_limits_attribute_checks(self): + # Test that nested suggestions are limited to checking first 20 non-private attributes + class Inner: + def __init__(self): + self.target_value = 42 + + class Outer: + def __init__(self): + # Add many attributes before 'inner' + for i in range(25): + setattr(self, f'attr_{i:02d}', i) + # Add the inner object after 20+ attributes + self.inner = Inner() + + obj = Outer() + # Verify that 'inner' is indeed present but after position 20 + attrs = [x for x in sorted(dir(obj)) if not x.startswith('_')] + inner_position = attrs.index('inner') + self.assertGreater(inner_position, 19, "inner should be after position 20 in sorted attributes") + + # Should not suggest 'inner.target_value' because inner is beyond the first 20 attributes checked + actual = self.get_suggestion(obj, 'target_value') + self.assertNotIn("inner.target_value", actual) + + def test_getattr_nested_returns_first_match_only(self): + # Test that only the first nested match is returned (not multiple) + class Inner1: + def __init__(self): + self.value = 1 + + class Inner2: + def __init__(self): + self.value = 2 + + class Inner3: + def __init__(self): + self.value = 3 + + class Outer: + def __init__(self): + # Multiple inner objects with same attribute + self.a_inner = Inner1() + self.b_inner = Inner2() + self.c_inner = Inner3() + + # Should suggest only the first match (alphabetically) + actual = self.get_suggestion(Outer(), 'value') + self.assertIn("'.a_inner.value'", actual) + # Verify it's a single suggestion, not multiple + self.assertEqual(actual.count("Did you mean"), 1) + + def test_getattr_nested_handles_attribute_access_exceptions(self): + # Test that exceptions raised when accessing attributes don't crash the suggestion system + class ExplodingProperty: + @property + def exploding_attr(self): + raise RuntimeError("BOOM! This property always explodes") + + def __repr__(self): + raise RuntimeError("repr also explodes") + + class SafeInner: + def __init__(self): + self.target = 42 + + class Outer: + def __init__(self): + self.exploder = ExplodingProperty() # Accessing attributes will raise + self.safe_inner = SafeInner() + + # Should still suggest '.safe_inner.target' without crashing + # even though accessing exploder.target would raise an exception + actual = self.get_suggestion(Outer(), 'target') + self.assertIn("'.safe_inner.target'", actual) + + def test_getattr_nested_handles_hasattr_exceptions(self): + # Test that exceptions in hasattr don't crash the system + class WeirdObject: + def __getattr__(self, name): + if name == 'target': + raise RuntimeError("Can't check for target attribute") + raise AttributeError(f"No attribute {name}") + + class NormalInner: + def __init__(self): + self.target = 100 + + class Outer: + def __init__(self): + self.weird = WeirdObject() # hasattr will raise for 'target' + self.normal = NormalInner() + + # Should still find 'normal.target' even though weird.target check fails + actual = self.get_suggestion(Outer(), 'target') + self.assertIn("'.normal.target'", actual) + + @force_not_colorized + def test_cross_language(self): + cases = [ + # (type, attr, hint_attr) + (list, 'push', 'append'), + (list, 'concat', 'extend'), + (list, 'addAll', 'extend'), + (str, 'toUpperCase', 'upper'), + (str, 'toLowerCase', 'lower'), + (str, 'trimStart', 'lstrip'), + (str, 'trimEnd', 'rstrip'), + (dict, 'keySet', 'keys'), + (dict, 'entrySet', 'items'), + (dict, 'entries', 'items'), + (dict, 'putAll', 'update'), + ] + for test_type, attr, hint_attr in cases: + with self.subTest(type=test_type.__name__, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, f"Did you mean '.{hint_attr}'?") + + cases = [ + # (type, attr, hint) + (list, 'contains', "Use 'x in list'."), + (list, 'add', "Did you mean to use a 'set' object?"), + (dict, 'put', "Use d[k] = v."), + ] + for test_type, attr, expected in cases: + with self.subTest(type=test_type, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, expected) + + @force_not_colorized + def test_cross_language_levenshtein_fallback(self): + # When no cross-language entry exists, Levenshtein still works + # (e.g., trim->strip is not in the table but Levenshtein catches it) + actual = self.get_suggestion('', 'trim') + self.assertIn("strip", actual) + + @force_not_colorized + def test_cross_language_no_hint_for_unknown_attr(self): + actual = self.get_suggestion([], 'completely_unknown_method') + self.assertNotIn("Did you mean", actual) + + @force_not_colorized + def test_cross_language_works_for_subclasses(self): + # isinstance() check means subclasses also get hints + class MyList(list): + pass + actual = self.get_suggestion(MyList(), 'push') + self.assertEndsWith(actual, "Did you mean '.append'?") + + class MyDict(dict): + pass + actual = self.get_suggestion(MyDict(), 'keySet') + self.assertEndsWith(actual, "Did you mean '.keys'?") + + @force_not_colorized + def test_cross_language_mutable_on_immutable(self): + # Mutable method on immutable type suggests the mutable counterpart + cases = [ + (tuple, 'append', "Did you mean to use a 'list' object?"), + (tuple, 'extend', "Did you mean to use a 'list' object?"), + (tuple, 'insert', "Did you mean to use a 'list' object?"), + (tuple, 'remove', "Did you mean to use a 'list' object?"), + (frozenset, 'add', "Did you mean to use a 'set' object?"), + (frozenset, 'discard', "Did you mean to use a 'set' object?"), + (frozenset, 'remove', "Did you mean to use a 'set' object?"), + (frozenset, 'update', "Did you mean to use a 'set' object?"), + (frozendict, 'update', "Did you mean to use a 'dict' object?"), + ] + for test_type, attr, expected in cases: + with self.subTest(type=test_type.__name__, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, expected) + + @force_not_colorized + def test_cross_language_float_bitwise(self): + # Bitwise operators on float suggest using int + cases = ['__or__', '__and__', '__xor__', '__lshift__', '__rshift__'] + for attr in cases: + with self.subTest(attr=attr): + actual = self.get_suggestion(1.0, attr) + self.assertIn("'int'", actual) + self.assertIn("Bitwise operators", actual) + def make_module(self, code): tmpdir = Path(tempfile.mkdtemp()) self.addCleanup(shutil.rmtree, tmpdir) @@ -4665,6 +5009,34 @@ def foo(self): actual = self.get_suggestion(instance.foo) self.assertIn("self.blech", actual) + def test_name_error_with_instance_not_normalized(self): + class A: + def __init__(self): + self.fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = None + def foo(self): + analization = 1 + x = fiⁿₐˡᵢᶻₐᵗᵢᵒₙ + + instance = A() + actual = self.get_suggestion(instance.foo) + self.assertIn("self.finalization", actual) + self.assertNotIn("fiⁿₐˡᵢᶻₐᵗᵢᵒₙ", actual) + self.assertNotIn("analization", actual) + + class B: + def __init__(self): + self.attr_µ = None # attr_\xb5 + def foo(self): + attr_a = 1 + x = attr_µ # attr_\xb5 + + instance = B() + actual = self.get_suggestion(instance.foo) + self.assertIn("self.attr_\u03bc", actual) + self.assertIn(r"self.attr_\u03bc", actual) + self.assertNotIn("attr_\xb5", actual) + self.assertNotIn("attr_a", actual) + def test_unbound_local_error_with_instance(self): class A: def __init__(self): @@ -4747,6 +5119,51 @@ class CPythonSuggestionFormattingTests( """ +class PurePythonGetattrSuggestionFormattingTests( + PurePythonExceptionFormattingMixin, + GetattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute access) as above using the pure Python + implementation of traceback printing in traceback.py. + """ + + +class PurePythonDelattrSuggestionFormattingTests( + PurePythonExceptionFormattingMixin, + DelattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute deletion) as above using the pure Python + implementation of traceback printing in traceback.py. + """ + + +@cpython_only +class CPythonGetattrSuggestionFormattingTests( + CAPIExceptionFormattingMixin, + GetattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute access) as above but with Python's + internal traceback printing. + """ + + +@cpython_only +class CPythonDelattrSuggestionFormattingTests( + CAPIExceptionFormattingMixin, + DelattrSuggestionTests, + unittest.TestCase, +): + """ + Same set of tests (for attribute deletion) as above but with Python's + internal traceback printing. + """ + class MiscTest(unittest.TestCase): def test_all(self): @@ -4841,7 +5258,109 @@ class MyList(list): with self.assertRaises(TypeError): _suggestions._generate_suggestions(MyList(), "") + def test_no_site_package_flavour(self): + code = """import boo""" + _, _, stderr = assert_python_failure('-S', '-c', code) + self.assertIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path " + b"or to enable your virtual environment?"), stderr + ) + + code = """ + import sys + sys.stdlib_module_names = sys.stdlib_module_names + ("boo",) + import boo + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertNotIn( + (b"Site initialization is disabled, did you forget to " + b"add the site-packages directory to sys.path " + b"or to enable your virtual environment?"), stderr + ) + + def test_missing_stdlib_module(self): + code = """ + import sys + sys.stdlib_module_names |= {'spam'} + import spam + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertIn(b"Standard library module 'spam' was not found", stderr) + + code = """ + import sys + import traceback + traceback._MISSING_STDLIB_MODULE_MESSAGES = {'spam': "Install 'spam4life' for 'spam'"} + sys.stdlib_module_names |= {'spam'} + import spam + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertIn(b"Install 'spam4life' for 'spam'", stderr) + + @unittest.skipIf(sys.platform == "win32", "Non-Windows test") + def test_windows_only_module_error(self): + try: + import msvcrt # noqa: F401 + except ModuleNotFoundError: + formatted = traceback.format_exc() + self.assertIn("Unsupported platform for Windows-only standard library module 'msvcrt'", formatted) + else: + self.fail("ModuleNotFoundError was not raised") + + @unittest.skipIf(not importlib.machinery.EXTENSION_SUFFIXES, 'Platform does not support extension modules') + def test_find_incompatible_extension_modules(self): + """_find_incompatible_extension_modules assumes the last extension in + importlib.machinery.EXTENSION_SUFFIXES (defined in Python/dynload_*.c) + is untagged (eg. .so, .pyd). + + This test exists to make sure that assumption is correct. + """ + last_extension_suffix = importlib.machinery.EXTENSION_SUFFIXES[-1] + if shlib_suffix := sysconfig.get_config_var('SHLIB_SUFFIX'): + self.assertEqual(last_extension_suffix, shlib_suffix) + else: + before_dot, *extensions = last_extension_suffix.split('.') + expected_prefixes = [''] + if os.name == 'nt': + # Windows puts the debug tag in the module file stem (eg. foo_d.pyd) + expected_prefixes.append('_d') + self.assertIn(before_dot, expected_prefixes, msg=( + f'Unexpected prefix {before_dot!r} in extension module ' + f'suffix {last_extension_suffix!r}. ' + 'traceback._find_incompatible_extension_module needs to be ' + 'updated to take this into account!' + )) + # if SHLIB_SUFFIX is not define, we assume the native + # shared library suffix only contains one extension + # (eg. '.so', bad eg. '.cpython-315-x86_64-linux-gnu.so') + self.assertEqual(len(extensions), 1, msg=( + 'The last suffix in importlib.machinery.EXTENSION_SUFFIXES ' + 'contains more than one extension, so it is probably different ' + 'than SHLIB_SUFFIX. It probably contains an ABI tag! ' + 'If this is a false positive, define SHLIB_SUFFIX in sysconfig.' + )) + + @unittest.skipIf(not importlib.machinery.EXTENSION_SUFFIXES, 'Platform does not support extension modules') + def test_incompatible_extension_modules_hint(self): + untagged_suffix = importlib.machinery.EXTENSION_SUFFIXES[-1] + with os_helper.temp_dir() as tmp: + # create a module with a incompatible ABI tag + incompatible_module = f'foo.some-abi{untagged_suffix}' + open(os.path.join(tmp, incompatible_module), "wb").close() + # try importing it + code = f''' + import sys + sys.path.insert(0, {tmp!r}) + import foo + ''' + _, _, stderr = assert_python_failure('-c', code, __cwd=tmp) + hint = f'Although a module with this name was found for a different Python version ({incompatible_module}).' + self.assertIn(hint, stderr.decode()) class TestColorizedTraceback(unittest.TestCase): @@ -4879,6 +5398,23 @@ def bar(): self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) + def test_colorized_exception_notes(self): + def foo(): + raise ValueError() + + try: + foo() + except Exception as e: + e.add_note("First note") + e.add_note("Second note") + exc = traceback.TracebackException.from_exception(e) + + lines = "".join(exc.format(colorize=True)) + note = colors["n"] + reset = colors["z"] + self.assertIn(note + "First note" + reset, lines) + self.assertIn(note + "Second note" + reset, lines) + def test_colorized_syntax_error(self): try: compile("a $ b", "", "exec") @@ -4887,7 +5423,7 @@ def test_colorized_syntax_error(self): e, capture_locals=True ) actual = "".join(exc.format(colorize=True)) - def expected(t, m, fn, l, f, E, e, z): + def expected(t, m, fn, l, f, E, e, z, n): return "".join( [ f' File {fn}""{z}, line {l}1{z}\n', @@ -4914,7 +5450,7 @@ def foo(): actual = tbstderr.getvalue().splitlines() lno_foo = foo.__code__.co_firstlineno - def expected(t, m, fn, l, f, E, e, z): + def expected(t, m, fn, l, f, E, e, z, n): return [ 'Traceback (most recent call last):', f' File {fn}"{__file__}"{z}, ' @@ -4947,7 +5483,7 @@ def foo(): lno_foo = foo.__code__.co_firstlineno actual = "".join(exc.format(colorize=True)).splitlines() - def expected(t, m, fn, l, f, E, e, z): + def expected(t, m, fn, l, f, E, e, z, n): return [ f" + Exception Group Traceback (most recent call last):", f' | File {fn}"{__file__}"{z}, line {l}{lno_foo+9}{z}, in {f}test_colorized_traceback_from_exception_group{z}', @@ -4971,5 +5507,139 @@ def expected(t, m, fn, l, f, E, e, z): ] self.assertEqual(actual, expected(**colors)) + def test_colorized_traceback_unicode(self): + try: + 啊哈=1; 啊哈/0#### + except Exception as e: + exc = traceback.TracebackException.from_exception(e) + + actual = "".join(exc.format(colorize=True)).splitlines() + def expected(t, m, fn, l, f, E, e, z, n): + return [ + f" 啊哈=1; {e}啊哈{z}{E}/{z}{e}0{z}####", + f" {e}~~~~{z}{E}^{z}{e}~{z}", + ] + self.assertEqual(actual[2:4], expected(**colors)) + + try: + ééééé/0 + except Exception as e: + exc = traceback.TracebackException.from_exception(e) + + actual = "".join(exc.format(colorize=True)).splitlines() + def expected(t, m, fn, l, f, E, e, z, n): + return [ + f" {E}ééééé{z}/0", + f" {E}^^^^^{z}", + ] + self.assertEqual(actual[2:4], expected(**colors)) + + def test_colorized_syntax_error_ascii_display_width(self): + """Caret alignment for ASCII edge cases handled by _wlen. + + The old ASCII fast track in _display_width returned the raw character + offset for ASCII strings, which is wrong for CTRL-Z (display width 2) + and ANSI escape sequences (display width 0). + """ + E = colors["E"] + z = colors["z"] + t = colors["t"] + m = colors["m"] + fn = colors["fn"] + l = colors["l"] + + def _make_syntax_error(text, offset, end_offset): + err = SyntaxError("invalid syntax") + err.filename = "" + err.lineno = 1 + err.end_lineno = 1 + err.text = text + err.offset = offset + err.end_offset = end_offset + return err + + # CTRL-Z (\x1a) is ASCII but displayed as ^Z (2 columns). + # Verify caret aligns when CTRL-Z precedes the error. + err = _make_syntax_error("a\x1a$\n", offset=3, end_offset=4) + exc = traceback.TracebackException.from_exception(err) + actual = "".join(exc.format(colorize=True)) + # 'a' (1 col) + '\x1a' (2 cols) = 3 cols before '$' + self.assertIn( + f' File {fn}""{z}, line {l}1{z}\n' + f' a\x1a{E}${z}\n' + f' {" " * 3}{E}^{z}\n' + f'{t}SyntaxError{z}: {m}invalid syntax{z}\n', + actual, + ) + + # CTRL-Z in the highlighted (error) region counts as 2 columns. + err = _make_syntax_error("$\x1a\n", offset=1, end_offset=3) + exc = traceback.TracebackException.from_exception(err) + actual = "".join(exc.format(colorize=True)) + # '$' (1 col) + '\x1a' (2 cols) = 3 columns of carets + self.assertIn( + f' {E}$\x1a{z}\n' + f' {E}{"^" * 3}{z}\n', + actual, + ) + + # ANSI escape sequences are ASCII but take 0 display columns. + err = _make_syntax_error("a\x1b[1mb$\n", offset=7, end_offset=8) + exc = traceback.TracebackException.from_exception(err) + actual = "".join(exc.format(colorize=True)) + # 'a' (1 col) + '\x1b[1m' (0 cols) + 'b' (1 col) = 2 before '$' + self.assertIn( + f' a\x1b[1mb{E}${z}\n' + f' {" " * 2}{E}^{z}\n', + actual, + ) + +class TestLazyImportSuggestions(unittest.TestCase): + """Test that lazy imports are not reified when computing AttributeError suggestions.""" + + def test_attribute_error_does_not_reify_lazy_imports(self): + """Printing an AttributeError should not trigger lazy import reification.""" + # lazypkg.bar prints "BAR_MODULE_LOADED" when imported. + # If lazy import is reified during suggestion computation, we'll see it. + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.lazypkg + test.test_lazy_import.data.lazypkg.nonexistent + """) + rc, stdout, stderr = assert_python_failure('-c', code) + self.assertNotIn(b"BAR_MODULE_LOADED", stdout) + + def test_traceback_formatting_does_not_reify_lazy_imports(self): + """Formatting a traceback should not trigger lazy import reification.""" + code = textwrap.dedent(""" + import traceback + lazy import test.test_lazy_import.data.lazypkg + try: + test.test_lazy_import.data.lazypkg.nonexistent + except AttributeError: + traceback.format_exc() + print("OK") + """) + rc, stdout, stderr = assert_python_ok('-c', code) + self.assertIn(b"OK", stdout) + self.assertNotIn(b"BAR_MODULE_LOADED", stdout) + + def test_suggestion_still_works_for_non_lazy_attributes(self): + """Suggestions should still work for non-lazy module attributes.""" + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.lazypkg + # Typo for __name__ + test.test_lazy_import.data.lazypkg.__nme__ + """) + rc, stdout, stderr = assert_python_failure('-c', code) + self.assertIn(b"__name__", stderr) + self.assertNotIn(b"BAR_MODULE_LOADED", stdout) + + +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("traceback", {"_colorize"}) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index 74653c77c55..1e5f14ca007 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -1,5 +1,9 @@ +import subprocess import unittest +from test import support +from test.support.os_helper import temp_cwd +from test.support.script_helper import assert_python_ok, spawn_python from test.test_string._support import TStringBaseCase, fstring @@ -79,6 +83,44 @@ def upper(self): ) self.assertEqual(fstring(t), "Name: Bob, Age: 30") + @support.requires_subprocess() + def test_expression_in_interactive_after_buffer_resize(self): + expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)" + source = ( + f"result = t'''{{{expression}}}'''\n" + "print(repr(result.interpolations[0].expression))\n" + ) + with spawn_python('-i', '-q', stderr=subprocess.PIPE) as process: + stdout, stderr = process.communicate( + source.encode(), timeout=support.SHORT_TIMEOUT) + self.assertEqual(process.returncode, 0, stderr) + self.assertEqual(stdout.decode().strip(), repr(expression)) + + def test_interpolation_expression_in_file_after_buffer_resize(self): + expression = "(\n" + (" " * 64 + "\n") * 256 + "1\n)" + with temp_cwd(): + script = 'script.py' + source = ( + f"template = t'''{{{expression}}}'''\n" + "interpolation = template.interpolations[0]\n" + f"assert interpolation.expression == {expression!r}\n" + ) + with open(script, 'w') as f: + f.write(source) + assert_python_ok(script) + + @support.requires_resource('cpu') + def test_many_tstrings_in_module(self): + fields = ''.join(f'{{x{i}}}' for i in range(100)) + source = ''.join( + f"value_{i} = t'{fields}'\n" for i in range(1_000) + ) + namespace = {f'x{i}': str(i) for i in range(100)} + expected = ''.join(str(i) for i in range(100)) + exec(source, namespace) + self.assertEqual(fstring(namespace['value_0']), expected) + self.assertEqual(fstring(namespace['value_999']), expected) + def test_format_specifiers(self): # Test basic format specifiers value = 3.14159 @@ -88,6 +130,14 @@ def test_format_specifiers(self): ) self.assertEqual(fstring(t), "Pi: 3.14") + a = 3 + b = 4 + t = t"{a!=b:>10}" + self.assertTStringEqual( + t, ("", ""), [(a != b, "a!=b", None, ">10")] + ) + self.assertEqual(fstring(t), " 1") + def test_conversions(self): # Test !s conversion (str) obj = object() @@ -136,10 +186,82 @@ def test_debug_specifier(self): # Test white space in debug specifier t = t"Value: {value = }" self.assertTStringEqual( - t, ("Value: value = ", ""), [(value, "value", "r")] + t, ("Value: value = ", ""), [(value, "value ", "r")] ) self.assertEqual(fstring(t), "Value: value = 42") + # Explicit line continuations after the debug marker are part of + # the debug text, not the interpolation expression. + for template, strings, interpolation, rendered in ( + ( + t"""Value: {value =\ +}""", + ("Value: value =\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n42", + ), + ( + t"""Value: {value =\ +!r}""", + ("Value: value =\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n42", + ), + ( + t"""Value: {value =\ +:04}""", + ("Value: value =\\\n", ""), + (value, "value ", None, "04"), + "Value: value =\\\n0042", + ), + ( + t"""Value: {value =\ +\ +}""", + ("Value: value =\\\n\\\n", ""), + (value, "value ", "r"), + "Value: value =\\\n\\\n42", + ), + ): + with self.subTest(template=template): + self.assertTStringEqual(template, strings, [interpolation]) + self.assertEqual(fstring(template), rendered) + + def test_interpolation_expression_whitespace(self): + x = 42 + for template, expected in ( + (t"{x}", "x"), + (t"{x }", "x "), + (t"{ x}", " x"), + (t"{ x }", " x "), + (t"{ x }", " x "), + (t"""{ + x +}""", "\n x\n"), + (t"{ x !r}", " x "), + (t"{ x :.2f}", " x "), + (t"{ x = }", " x "), + (t"{ x = !r}", " x "), + (t"{ x = :.2f}", " x "), + (t"{x == 42 = }", "x == 42 "), + ): + with self.subTest(template=template): + self.assertEqual( + template.interpolations[0].expression, + expected, + ) + + def test_interpolation_expression_with_reconstructed_metadata(self): + regular = t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")}''' + debug = t'''{( + 1, # Force lexer metadata reconstruction. + "\"#")=}''' + expected = '(\n 1, \n "\\"#")' + self.assertEqual(regular.interpolations[0].expression, expected) + self.assertEqual(debug.interpolations[0].expression, expected) + def test_raw_tstrings(self): path = r"C:\Users" t = rt"{path}\Documents" diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 57b12609f71..7ca329ff76d 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -46,7 +46,6 @@ def clear_typing_caches(): class TypesTests(unittest.TestCase): - @unittest.expectedFailureIf(not hasattr(c_types, "LazyImportType"), "TODO: RUSTPYTHON; PEP 810 LazyImportType is not implemented") def test_names(self): c_only_names = {'CapsuleType', 'LazyImportType'} ignored = {'new_class', 'resolve_bases', 'prepare_class', diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 1f31258ce36..3545d92c4f5 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -35,6 +35,7 @@ from token import * from token import EXACT_TOKEN_TYPES import _tokenize +lazy import _colorize cookie_re = re.compile(br'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) @@ -403,7 +404,7 @@ def find_cookie(line): return None encoding = _get_normal_name(match.group(1).decode()) try: - codec = lookup(encoding) + lookup(encoding) except LookupError: # This behaviour mimics the Python interpreter if filename is None: @@ -505,6 +506,56 @@ def generate_tokens(readline): """ return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True) + +def _get_token_colors(syntax, tokenize): + """Map token type numbers to theme colors.""" + return frozendict({ + COMMENT: syntax.comment, + DEDENT: tokenize.whitespace, + ENCODING: tokenize.whitespace, + ENDMARKER: tokenize.whitespace, + ERRORTOKEN: tokenize.error, + FSTRING_START: syntax.string, + FSTRING_MIDDLE: syntax.string, + FSTRING_END: syntax.string, + INDENT: tokenize.whitespace, + NAME: syntax.reset, + NEWLINE: tokenize.whitespace, + NL: tokenize.whitespace, + NUMBER: syntax.number, + OP: syntax.op, + SOFT_KEYWORD: syntax.soft_keyword, + STRING: syntax.string, + TSTRING_START: syntax.string, + TSTRING_MIDDLE: syntax.string, + TSTRING_END: syntax.string, + }) + + +def _format_tokens(tokens, *, color=False, exact=False): + theme = _colorize.get_theme(force_no_color=not color) + s = theme.syntax + t = theme.tokenize + token_colors = _get_token_colors(s, t) + for token in tokens: + token_range = ( + f"{t.position}{token.start[0]}" + f"{t.delimiter},{t.position}{token.start[1]}" + f"{t.delimiter}-" + f"{t.position}{token.end[0]}" + f"{t.delimiter},{t.position}{token.end[1]}" + f"{t.delimiter}:" + ) + token_color = token_colors.get(token.type, s.reset) + token_name = tok_name[token.exact_type if exact else token.type] + visible_range = f"{token.start[0]},{token.start[1]}-{token.end[0]},{token.end[1]}:" + yield ( + f"{token_range}{' ' * (20 - len(visible_range))}" + f"{token_color}{token_name:<15}" + f"{s.reset}{token.string!r:<15}" + ) + + def _main(args=None): import argparse @@ -524,10 +575,10 @@ def error(message, filename=None, location=None): sys.exit(1) # Parse the arguments and options - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument(dest='filename', nargs='?', metavar='filename.py', - help='the file to tokenize; defaults to stdin') + help='the file to tokenize; defaults to `stdin`') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') args = parser.parse_args(args) @@ -545,13 +596,8 @@ def error(message, filename=None, location=None): # Output the tokenization - for token in tokens: - token_type = token.type - if args.exact: - token_type = token.exact_type - token_range = "%d,%d-%d,%d:" % (token.start + token.end) - print("%-20s%-15s%-15r" % - (token_range, tok_name[token_type], token.string)) + for line in _format_tokens(tokens, color=True, exact=args.exact): + print(line) except IndentationError as err: line, column = err.args[1][1:3] error(err.args[0], filename, (line, column)) diff --git a/Lib/traceback.py b/Lib/traceback.py index 227ca8efa17..d16ab468db9 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,18 +1,29 @@ """Extract, format and print information about Python stack traces.""" import collections.abc +import functools import itertools import linecache +import os +import re import sys import textwrap +import types import warnings import codeop import keyword import tokenize import io -import _colorize +import importlib.util +import pathlib from contextlib import suppress +lazy import _colorize + +try: + from _missing_stdlib_info import _MISSING_STDLIB_MODULE_MESSAGES +except ImportError: + _MISSING_STDLIB_MODULE_MESSAGES = {} __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', @@ -21,6 +32,36 @@ 'FrameSummary', 'StackSummary', 'TracebackException', 'walk_stack', 'walk_tb', 'print_list'] + +class _ShutdownTheme: + """Empty stand-in if `_colorize` cannot be imported during late shutdown.""" + def __getattr__(self, _): return self + def __getitem__(self, _): return "" + def __format__(self, _): return "" + def __str__(self): return "" + def __add__(self, other): return other + __radd__ = __add__ + + +_shutdown_theme = _ShutdownTheme() + + +def _safe_get_theme(*, force_color=False, force_no_color=False): + try: + return _colorize.get_theme( + force_color=force_color, force_no_color=force_no_color + ) + except ImportError: + return _shutdown_theme + + +def _safe_can_colorize(*, file=None): + try: + return _colorize.can_colorize(file=file) + except ImportError: + return False + + # # Formatting and printing lists of traceback lines. # @@ -137,9 +178,10 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _print_exception_bltin(exc, /): - file = sys.stderr if sys.stderr is not None else sys.__stderr__ - colorize = _colorize.can_colorize(file=file) +def _print_exception_bltin(exc, file=None, /): + if file is None: + file = sys.stderr if sys.stderr is not None else sys.__stderr__ + colorize = _safe_can_colorize(file=file) return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) @@ -187,9 +229,9 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= valuestr = _safe_string(value, 'exception') end_char = "\n" if insert_final_newline else "" if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback if value is None or not valuestr: line = f"{theme.type}{etype}{theme.reset}{end_char}" else: @@ -205,9 +247,9 @@ def _safe_string(value, what, func=str): # -- -def print_exc(limit=None, file=None, chain=True): +def print_exc(limit=None, file=None, chain=True, **kwargs): """Shorthand for 'print_exception(sys.exception(), limit=limit, file=file, chain=chain)'.""" - print_exception(sys.exception(), limit=limit, file=file, chain=chain) + print_exception(sys.exception(), limit=limit, file=file, chain=chain, **kwargs) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" @@ -286,6 +328,12 @@ class FrameSummary: of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. + - :attr:`end_lineno` The last line number of the source code for this frame. + By default, it is set to lineno and indexation starts from 1. + - :attr:`colno` The column number of the source code for this frame. + By default, it is None and indexation starts from 0. + - :attr:`end_colno` The last column number of the source code for this frame. + By default, it is None and indexation starts from 0. """ __slots__ = ('filename', 'lineno', 'end_lineno', 'colno', 'end_colno', @@ -536,9 +584,9 @@ def format_frame_summary(self, frame_summary, **kwargs): if frame_summary.filename.startswith("'): filename = "" if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback row.append( ' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( theme.filename, @@ -667,12 +715,12 @@ def output_line(lineno): colorized_line_parts = [] colorized_carets_parts = [] - for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): + for color, group in itertools.groupby(_zip_display_width(line, carets), key=lambda x: x[1]): caret_group = list(group) - if color == "^": + if "^" in color: colorized_line_parts.append(theme.error_highlight + "".join(char for char, _ in caret_group) + theme.reset) colorized_carets_parts.append(theme.error_highlight + "".join(caret for _, caret in caret_group) + theme.reset) - elif color == "~": + elif "~" in color: colorized_line_parts.append(theme.error_range + "".join(char for char, _ in caret_group) + theme.reset) colorized_carets_parts.append(theme.error_range + "".join(caret for _, caret in caret_group) + theme.reset) else: @@ -954,7 +1002,54 @@ def setup_positions(expr, force_valid=True): return None -_WIDE_CHAR_SPECIFIERS = "WF" + +def _zip_display_width(line, carets): + carets = iter(carets) + if line.isascii() and '\x1a' not in line: + for char in line: + yield char, next(carets, "") + return + + import unicodedata + for char in unicodedata.iter_graphemes(line): + char = str(char) + char_width = _display_width(char) + yield char, "".join(itertools.islice(carets, char_width)) + + +@functools.cache +def _str_width(c: str) -> int: + # copied from _pyrepl.utils to fix gh-130273 + + if ord(c) < 128: + return 1 + import unicodedata + # gh-139246 for zero-width joiner and combining characters + if unicodedata.combining(c): + return 0 + category = unicodedata.category(c) + if category == "Cf" and c != "\u00ad": + return 0 + w = unicodedata.east_asian_width(c) + if w in ("N", "Na", "H", "A"): + return 1 + return 2 + + +_ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +def _wlen(s: str) -> int: + # copied from _pyrepl.utils to fix gh-130273 + + if len(s) == 1 and s != "\x1a": + return _str_width(s) + length = sum(_str_width(i) for i in s) + # remove lengths of any escape sequences + sequence = _ANSI_ESCAPE_SEQUENCE.findall(s) + ctrl_z_cnt = s.count("\x1a") + return length - sum(len(i) for i in sequence) + ctrl_z_cnt + def _display_width(line, offset=None): """Calculate the amount of width space the given source @@ -962,19 +1057,14 @@ def _display_width(line, offset=None): width output device. Supports wide unicode characters and emojis.""" if offset is None: - offset = len(line) - - # Fast track for ASCII-only strings - if line.isascii(): - return offset + return _wlen(line) - import unicodedata + return _wlen(line[:offset]) - return sum( - 2 if unicodedata.east_asian_width(char) in _WIDE_CHAR_SPECIFIERS else 1 - for char in line[:offset] - ) +def _format_note(note, indent, theme): + for l in note.split("\n"): + yield f"{indent}{theme.note}{l}{theme.reset}\n" class _ExceptionPrintContext: @@ -1098,20 +1188,62 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, wrong_name = getattr(exc_value, "name_from", None) suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) if suggestion: - self._str += f". Did you mean: '{suggestion}'?" - elif exc_type and issubclass(exc_type, (NameError, AttributeError)) and \ + if suggestion.isascii(): + self._str += f". Did you mean: '{suggestion}'?" + else: + self._str += f". Did you mean: '{suggestion}' ({suggestion!a})?" + elif exc_type and issubclass(exc_type, ModuleNotFoundError): + module_name = getattr(exc_value, "name", None) + if module_name in sys.stdlib_module_names: + message = _MISSING_STDLIB_MODULE_MESSAGES.get( + module_name, + f"Standard library module {module_name!r} was not found" + ) + self._str = message + elif sys.flags.no_site: + self._str += (". Site initialization is disabled, did you forget to " + + "add the site-packages directory to sys.path " + + "or to enable your virtual environment?") + elif abi_tag := _find_incompatible_extension_module(module_name): + self._str += ( + ". Although a module with this name was found for a " + f"different Python version ({abi_tag})." + ) + else: + suggestion = _compute_suggestion_error(exc_value, exc_traceback, module_name) + if suggestion: + self._str += f". Did you mean: '{suggestion}'?" + elif exc_type and issubclass(exc_type, AttributeError) and \ + getattr(exc_value, "name", None) is not None: + wrong_name = getattr(exc_value, "name", None) + # Check cross-language/wrong-type hints first (more specific), + # then fall back to Levenshtein distance suggestions. + hint = None + if hasattr(exc_value, 'obj'): + hint = _get_cross_language_hint(exc_value.obj, wrong_name) + if hint: + self._str += f". {hint}" + else: + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + if suggestion.isascii(): + self._str += f". Did you mean '.{suggestion}' instead of '.{wrong_name}'?" + else: + self._str += f". Did you mean '.{suggestion}' ({suggestion!a}) instead of '.{wrong_name}' ({wrong_name!a})?" + elif exc_type and issubclass(exc_type, NameError) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) if suggestion: - self._str += f". Did you mean: '{suggestion}'?" - if issubclass(exc_type, NameError): - wrong_name = getattr(exc_value, "name", None) - if wrong_name is not None and wrong_name in sys.stdlib_module_names: - if suggestion: - self._str += f" Or did you forget to import '{wrong_name}'?" - else: - self._str += f". Did you forget to import '{wrong_name}'?" + if suggestion.isascii(): + self._str += f". Did you mean: '{suggestion}'?" + else: + self._str += f". Did you mean: '{suggestion}' ({suggestion!a})?" + if wrong_name is not None and wrong_name in sys.stdlib_module_names: + if suggestion: + self._str += f" Or did you forget to import '{wrong_name}'?" + else: + self._str += f". Did you forget to import '{wrong_name}'?" if lookup_lines: self._load_lines() self.__suppress_context__ = \ @@ -1240,6 +1372,10 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): well, recursively, with indentation relative to their nesting depth. """ colorize = kwargs.get("colorize", False) + if colorize: + theme = _safe_get_theme(force_color=True).traceback + else: + theme = _safe_get_theme(force_no_color=True).traceback indent = 3 * _depth * ' ' if not self._have_exc_type: @@ -1268,9 +1404,10 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): ): for note in self.__notes__: note = _safe_string(note, 'note') - yield from [indent + l + '\n' for l in note.split('\n')] + yield from _format_note(note, indent, theme) elif self.__notes__ is not None: - yield indent + "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) + note = _safe_string(self.__notes__, '__notes__', func=repr) + yield from _format_note(note, indent, theme) if self.exceptions and show_group: for ex in self.exceptions: @@ -1315,6 +1452,15 @@ def _find_keyword_typos(self): if len(error_code) > 1024: return + # If the original code doesn't raise SyntaxError, we can't validate + # that a keyword replacement actually fixes anything + try: + codeop.compile_command(error_code, symbol="exec", flags=codeop.PyCF_ONLY_AST) + except SyntaxError: + pass # Good - the original code has a syntax error we might fix + else: + return # Original code compiles or is incomplete - can't validate fixes + error_lines = error_code.splitlines() tokens = tokenize.generate_tokens(io.StringIO(error_code).readline) tokens_left_to_process = 10 @@ -1377,9 +1523,9 @@ def _format_syntax_error(self, stype, **kwargs): # Show exactly where the problem was found. colorize = kwargs.get("colorize", False) if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback filename_suffix = '' if self.lineno is not None: yield ' File {}"{}"{}, line {}{}{}\n'.format( @@ -1430,10 +1576,11 @@ def _format_syntax_error(self, stype, **kwargs): # Convert 1-based column offset to 0-based index into stripped text colno = offset - 1 - spaces end_colno = end_offset - 1 - spaces - caretspace = ' ' if colno >= 0: - # non-space whitespace (likes tabs) must be kept for alignment - caretspace = ((c if c.isspace() else ' ') for c in ltext[:colno]) + # Calculate display width to account for wide characters + dp_colno = _display_width(ltext, colno) + highlighted = ltext[colno:end_colno] + caret_count = _display_width(highlighted) if highlighted else (end_colno - colno) start_color = end_color = "" if colorize: # colorize from colno to end_colno @@ -1446,9 +1593,9 @@ def _format_syntax_error(self, stype, **kwargs): end_color = theme.reset yield ' {}\n'.format(ltext) yield ' {}{}{}{}\n'.format( - "".join(caretspace), + ' ' * dp_colno, start_color, - ('^' * (end_colno - colno)), + '^' * caret_count, end_color, ) else: @@ -1579,6 +1726,62 @@ def print(self, *, file=None, chain=True, **kwargs): _MOVE_COST = 2 _CASE_COST = 1 +# Cross-language method suggestions for builtin types. +# Consulted as a fallback when Levenshtein-based suggestions find no match. +# +# Inclusion criteria: +# +# 1. Must have evidence of real cross-language confusion (Stack Overflow +# traffic, bug reports in production repos, developer survey data). +# 2. Must not be catchable by Levenshtein distance (too different from +# the correct Python method name). +# +# Each entry maps a wrong method name to a list of (type, suggestion, is_raw) +# tuples. The lookup checks isinstance() so subclasses are also matched. +# If is_raw is False, the suggestion is wrapped in "Did you mean '.X'?". +# If is_raw is True, the suggestion is rendered as-is. +# +# See https://github.com/python/cpython/issues/146406. +_CROSS_LANGUAGE_HINTS = frozendict({ + # list -- JavaScript/Ruby equivalents + "push": ((list, "append", False),), + "concat": ((list, "extend", False),), + # list -- Java/C# equivalents + "addAll": ((list, "extend", False),), + "contains": ((list, "Use 'x in list'.", True),), + # list -- wrong-type suggestion (user expected a set) + "add": ((list, "Did you mean to use a 'set' object?", True), + (frozenset, "Did you mean to use a 'set' object?", True)), + # str -- JavaScript equivalents + "toUpperCase": ((str, "upper", False),), + "toLowerCase": ((str, "lower", False),), + "trimStart": ((str, "lstrip", False),), + "trimEnd": ((str, "rstrip", False),), + # dict -- Java/JavaScript equivalents + "keySet": ((dict, "keys", False),), + "entrySet": ((dict, "items", False),), + "entries": ((dict, "items", False),), + "putAll": ((dict, "update", False),), + "put": ((dict, "Use d[k] = v.", True),), + # tuple -- mutable method on immutable type (user expected a list) + "append": ((tuple, "Did you mean to use a 'list' object?", True),), + "extend": ((tuple, "Did you mean to use a 'list' object?", True),), + "insert": ((tuple, "Did you mean to use a 'list' object?", True),), + "remove": ((tuple, "Did you mean to use a 'list' object?", True), + (frozenset, "Did you mean to use a 'set' object?", True)), + # frozenset -- mutable method on immutable type (user expected a set) + "discard": ((frozenset, "Did you mean to use a 'set' object?", True),), + # frozendict -- mutable method on immutable type (user expected a dict) + "update": ((frozenset, "Did you mean to use a 'set' object?", True), + (frozendict, "Did you mean to use a 'dict' object?", True)), + # float -- bitwise operators belong to int + "__or__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__and__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__xor__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__lshift__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__rshift__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), +}) + def _substitution_cost(ch_a, ch_b): if ch_a == ch_b: @@ -1588,19 +1791,100 @@ def _substitution_cost(ch_a, ch_b): return _MOVE_COST +def _is_lazy_import(obj, attr_name): + """Check if attr_name in obj's __dict__ is a lazy import. + + Returns True if obj is a module and the attribute is a LazyImportType, + False otherwise. This avoids triggering module loading when computing + suggestions for AttributeError. + """ + if not isinstance(obj, types.ModuleType): + return False + obj_dict = getattr(obj, '__dict__', None) + if obj_dict is None: + return False + attr_value = obj_dict.get(attr_name) + return isinstance(attr_value, types.LazyImportType) + + +def _check_for_nested_attribute(obj, wrong_name, attrs): + """Check if any attribute of obj has the wrong_name as a nested attribute. + + Returns the first nested attribute suggestion found, or None. + Limited to checking 20 attributes. + Only considers non-descriptor outer attributes to avoid executing + arbitrary code. Checks nested attributes statically so descriptors such + as properties can still be suggested without invoking them. + Skips lazy imports to avoid triggering module loading. + """ + from inspect import getattr_static + + # Check for nested attributes (only one level deep) + attrs_to_check = [x for x in attrs if not x.startswith('_')][:20] # Limit number of attributes to check + for attr_name in attrs_to_check: + with suppress(Exception): + # Check if attr_name is a descriptor - if so, skip it + attr_from_class = getattr_static(type(obj), attr_name, _sentinel) + if attr_from_class is not _sentinel and hasattr(attr_from_class, '__get__'): + continue # Skip descriptors to avoid executing arbitrary code + + # Skip lazy imports to avoid triggering module loading + if _is_lazy_import(obj, attr_name): + continue + + # Safe to get the attribute since it's not a descriptor + attr_obj = getattr(obj, attr_name) + + if _is_lazy_import(attr_obj, wrong_name): + continue + + if getattr_static(attr_obj, wrong_name, _sentinel) is not _sentinel: + return f"{attr_name}.{wrong_name}" + + return None + + +def _get_cross_language_hint(obj, wrong_name): + """Check if wrong_name is a common method name from another language, + a mutable method on an immutable type, or a method tried on None. + + Uses isinstance() so subclasses of builtin types also get hints. + Returns a formatted hint string, or None. + """ + entries = _CROSS_LANGUAGE_HINTS.get(wrong_name) + if entries is None: + return None + for check_type, hint, is_raw in entries: + if isinstance(obj, check_type): + if is_raw: + return hint + return f"Did you mean '.{hint}'?" + return None + + def _get_safe___dir__(obj): # Use obj.__dir__() to avoid a TypeError when calling dir(obj). # See gh-131001 and gh-139933. + # Also filters out lazy imports to avoid triggering module loading. try: d = obj.__dir__() except TypeError: # when obj is a class d = type(obj).__dir__(obj) - return sorted(x for x in d if isinstance(x, str)) + return sorted( + x for x in d if isinstance(x, str) and not _is_lazy_import(obj, x) + ) def _compute_suggestion_error(exc_value, tb, wrong_name): if wrong_name is None or not isinstance(wrong_name, str): return None + not_normalized = False + if not wrong_name.isascii(): + from unicodedata import normalize + normalized_name = normalize('NFKC', wrong_name) + if normalized_name != wrong_name: + not_normalized = True + wrong_name = normalized_name if isinstance(exc_value, AttributeError): obj = exc_value.obj try: @@ -1616,6 +1900,18 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): d = [x for x in d if x[:1] != '_'] except Exception: return None + elif isinstance(exc_value, ModuleNotFoundError): + try: + if parent_name := wrong_name.rpartition('.')[0]: + parent = importlib.util.find_spec(parent_name) + else: + parent = None + d = [] + for finder in sys.meta_path: + if discover := getattr(finder, 'discover', None): + d += [spec.name for spec in discover(parent)] + except Exception: + return None elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) @@ -1638,6 +1934,8 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): + list(frame.f_builtins) ) d = [x for x in d if isinstance(x, str)] + if not_normalized and wrong_name in d: + return wrong_name # Check first if we are in a method and the instance # has the wrong name as attribute @@ -1650,12 +1948,16 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): if has_wrong_name: return f"self.{wrong_name}" + if not_normalized and wrong_name in d: + return wrong_name try: import _suggestions except ImportError: pass else: - return _suggestions._generate_suggestions(d, wrong_name) + suggestion = _suggestions._generate_suggestions(d, wrong_name) + if suggestion: + return suggestion # Compute closest match @@ -1680,6 +1982,14 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): if not suggestion or current_distance < best_distance: suggestion = possible_name best_distance = current_distance + + # If no direct attribute match found, check for nested attributes + if not suggestion and isinstance(exc_value, AttributeError): + with suppress(Exception): + nested_suggestion = _check_for_nested_attribute(exc_value.obj, wrong_name, d) + if nested_suggestion: + return nested_suggestion + return suggestion @@ -1742,3 +2052,32 @@ def _levenshtein_distance(a, b, max_cost): # Everything in this row is too big, so bail early. return max_cost + 1 return result + + +def _find_incompatible_extension_module(module_name): + import importlib.machinery + import importlib.resources.readers + + if not module_name or not importlib.machinery.EXTENSION_SUFFIXES: + return + + # We assume the last extension is untagged (eg. .so, .pyd)! + # tests.test_traceback.MiscTest.test_find_incompatible_extension_modules + # tests that assumption. + untagged_suffix = importlib.machinery.EXTENSION_SUFFIXES[-1] + # On Windows the debug tag is part of the module file stem, instead of the + # extension (eg. foo_d.pyd), so let's remove it and just look for .pyd. + if os.name == 'nt': + untagged_suffix = untagged_suffix.removeprefix('_d') + + parent, _, child = module_name.rpartition('.') + if parent: + traversable = importlib.resources.files(parent) + else: + traversable = importlib.resources.readers.MultiplexedPath( + *map(pathlib.Path, filter(os.path.isdir, sys.path)) + ) + + for entry in traversable.iterdir(): + if entry.name.startswith(child + '.') and entry.name.endswith(untagged_suffix): + return entry.name diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index 7d43a502c01..fb561db6d93 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -3683,6 +3683,19 @@ impl<'warnings> Compiler<'warnings> { }) } + fn import_policy(&mut self, explicit: bool, star: bool) -> u32 { + if explicit { + 1 + } else if star + || self.current_symbol_table().typ != CompilerScope::Module + || self.in_exception_handler() + { + 2 + } else { + 0 + } + } + fn compile_statement(&mut self, statement: &ast::Stmt) -> CompileResult<()> { trace!("Compiling {statement:?}"); let prev_source_range = self.current_source_span; @@ -3703,7 +3716,7 @@ impl<'warnings> Compiler<'warnings> { } match &statement { - ast::Stmt::Import(ast::StmtImport { names, .. }) => { + ast::Stmt::Import(ast::StmtImport { names, is_lazy, .. }) => { // import a, b, c as d for name in names { let name = &name; @@ -3711,15 +3724,7 @@ impl<'warnings> Compiler<'warnings> { value: num_traits::Zero::zero(), }); self.emit_load_const(ConstantData::None); - // force eager import in try/except block - let low = if self.in_exception_handler() - || self.current_symbol_table().typ != CompilerScope::Module - { - 2 - } else { - 0 - }; - let namei = (self.name(&name.name) << 2) | low; + let namei = (self.name(&name.name) << 2) | self.import_policy(*is_lazy, false); emit!(self, Instruction::ImportName { namei }); if let Some(alias) = &name.asname { let parts: Vec<&str> = name.name.split('.').skip(1).collect(); @@ -3744,6 +3749,7 @@ impl<'warnings> Compiler<'warnings> { level, module, names, + is_lazy, .. }) => { let import_star = names.first().is_some_and(|n| &n.name == "*"); @@ -3764,16 +3770,8 @@ impl<'warnings> Compiler<'warnings> { }); let module_name = module.as_ref().map_or("", |s| s.as_str()); - // forced non-lazy import due to try/except or import * - let low = if self.in_exception_handler() - || self.current_symbol_table().typ != CompilerScope::Module - || import_star - { - 2 - } else { - 0 - }; - let module_idx = (self.name(module_name) << 2) | low; + let module_idx = + (self.name(module_name) << 2) | self.import_policy(*is_lazy, import_star); emit!(self, Instruction::ImportName { namei: module_idx }); if import_star { @@ -13821,18 +13819,13 @@ impl<'warnings> Compiler<'warnings> { self.compile_expression(&interp.expression)?; - let expr_range = interp.expression.range(); - let expr_source = if interp.range.start() < expr_range.start() - && interp.range.end() >= expr_range.end() - { - let after_brace = interp.range.start() + TextSize::new(1); - self.source_file - .source_text() - .slice(TextRange::new(after_brace, expr_range.end())) - } else { - self.source_file.source_text().slice(expr_range) - } - .to_string(); + let expr_source = crate::interpolation_expression_text(&self.source_file, interp) + .unwrap_or_else(|| { + self.source_file + .source_text() + .slice(interp.expression.range()) + .to_owned() + }); self.set_source_node(interp.range, &interp.node_index); self.emit_load_const(ConstantData::Str { value: expr_source.into(), @@ -17530,10 +17523,6 @@ class C[T]: pass } #[test] - #[expect( - clippy::literal_string_with_formatting_args, - reason = "the literal is the expected t-string annotation" - )] fn future_tstring_annotation_preserves_interpolation_source_like_cpython() { let code = compile_exec( "from __future__ import annotations\nx: t'{a + b}'\ny: t'{ a + b }'\nz: f'{a + b =}'\nu: t'{a + b =}'\nv: t'{a + b =:>10}'\np: t'{(a)}'\nq: t'{((a))!r}'\nr: t'{ ((a)) = !r:>10}'\ns: t'{(a)=}'\nt: t'{a == b = }'\na1: t'''{a= # x=y\n}'''\na2: t'''{a # x=y\n}'''\na3: t'''{(a # x=y\n)}'''\na4: t'''{a # x=y\n!r}'''\na5: t'''{a # x=y\n:>10}'''\na6: t'''{'#'}'''\na7: t'''{('#', a) # c=d\n}'''\n", @@ -17556,22 +17545,22 @@ class C[T]: pass annotation_strings, [ "t'{a + b}'", - "t'{ a + b}'", + "t'{ a + b }'", "f'a + b ={a + b!r}'", - "t'a + b ={a + b!r}'", - "t'a + b ={a + b:>10}'", + "t'a + b ={a + b !r}'", + "t'a + b ={a + b :>10}'", "t'{(a)}'", "t'{((a))!r}'", - "t' ((a)) = { ((a))!r:>10}'", + "t' ((a)) = { ((a)) !r:>10}'", "t'(a)={(a)!r}'", - "t'a == b = {a == b!r}'", + "t'a == b = {a == b !r}'", "t'a= \\n{a!r}'", - "t'{a}'", + "t'{a \\n}'", "t'{(a \\n)}'", - "t'{a!r}'", - "t'{a:>10}'", + "t'{a \\n!r}'", + "t'{a \\n:>10}'", "t\"{'#'}\"", - "t\"{('#', a)}\"", + "t\"{('#', a) \\n}\"", ] ); } diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs index 3e3f9e2757a..089f7ab496c 100644 --- a/crates/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -8,7 +8,7 @@ extern crate log; extern crate alloc; -use alloc::{string::String, vec::Vec}; +use alloc::string::String; use rustpython_compiler_core::bytecode::ConstantData; type IndexMap = indexmap::IndexMap; @@ -189,12 +189,14 @@ pub fn interpolation_debug_text( ) -> (String, TextRange) { let leading = debug_text.leading(); let trailing = debug_text.trailing(); - let text = [ - strip_python_comments(leading).as_str(), - source_file.source_text().slice(expression_range), - strip_python_comments(trailing).as_str(), - ] - .concat(); + let text = strip_python_comments( + &[ + leading, + source_file.source_text().slice(expression_range), + trailing, + ] + .concat(), + ); let width = |len: usize| TextSize::new(u32::try_from(len).expect("debug interpolation text too long")); let range = TextRange::new( @@ -204,64 +206,109 @@ pub fn interpolation_debug_text( (text, range) } +/// The source text stored in a template interpolation, including whitespace +/// before its debug marker, conversion, format specifier, or closing brace. +/// Returns `None` for a synthetic node without a source interpolation range. +#[must_use] +pub fn interpolation_expression_text( + source_file: &SourceFile, + interpolation: &ast::InterpolatedElement, +) -> Option { + if let Some(debug_text) = &interpolation.debug_text { + let mut text = strip_python_comments(debug_text.as_str()); + let end = interpolation_suffix_end(&text); + if text[..end].ends_with('=') { + text.truncate(end - 1); + } + return Some(text); + } + + let expression_range = interpolation.expression.range(); + if interpolation.range.start() >= expression_range.start() + || interpolation.range.end() <= expression_range.end() + { + return None; + } + let start = interpolation.range.start() + TextSize::new(1); + let end = interpolation + .format_spec + .as_ref() + .map_or_else( + || interpolation.range.end(), + |format_spec| format_spec.range.start(), + ) + .checked_sub(TextSize::new(1))?; + if start > expression_range.start() || end < expression_range.end() { + return None; + } + let mut text = + strip_python_comments(source_file.source_text().slice(TextRange::new(start, end))); + if interpolation.conversion != ast::ConversionFlag::None { + let end = interpolation_suffix_end(&text); + let conversion_start = end.checked_sub(2)?; + if text.as_bytes().get(conversion_start) != Some(&b'!') { + return None; + } + text.truncate(conversion_start); + } + Some(text) +} + +fn interpolation_suffix_end(text: &str) -> usize { + let mut end = text.len(); + // Debug markers and conversions can be followed by whitespace and line + // continuations. Whitespace preceding either marker remains significant. + loop { + let trimmed = text[..end].trim_end(); + let has_newline = text[trimmed.len()..end].contains(['\n', '\r']); + end = trimmed.len(); + if has_newline && text[..end].ends_with('\\') { + end -= 1; + } else { + return end; + } + } +} + fn strip_python_comments(text: &str) -> String { - let chars = text.chars().collect::>(); + // Tokenizer metadata uses universal newlines even when source ranges refer + // to the original CRLF or CR input. + let normalized; + let text = if text.contains('\r') { + normalized = text.replace("\r\n", "\n").replace('\r', "\n"); + normalized.as_str() + } else { + text + }; + let mut chars = text.chars(); let mut result = String::with_capacity(text.len()); let mut quote = None; - let mut triple_quoted = false; - let mut escaped = false; - let mut in_comment = false; - let mut index = 0; - while index < chars.len() { - let ch = chars[index]; - if in_comment { - if matches!(ch, '\n' | '\r') { - in_comment = false; - result.push(ch); - } - index += 1; - continue; - } - - if let Some(delimiter) = quote { + while let Some(ch) = chars.next() { + // Match tokenizer metadata reconstruction, including escaped quotes + // outside literals and its per-quote handling of triple strings. + if ch == '\\' { result.push(ch); - if escaped { - escaped = false; - } else if ch == '\\' { - escaped = true; - } else if triple_quoted - && ch == delimiter - && chars.get(index + 1) == Some(&delimiter) - && chars.get(index + 2) == Some(&delimiter) - { - result.push(delimiter); - result.push(delimiter); - quote = None; - index += 2; - } else if !triple_quoted && ch == delimiter { - quote = None; + if let Some(escaped) = chars.next() { + result.push(escaped); } - index += 1; - continue; - } - - match ch { - '#' => in_comment = true, - '\'' | '"' => { + } else if matches!(ch, '\'' | '"') { + if quote.is_none() { quote = Some(ch); - triple_quoted = - chars.get(index + 1) == Some(&ch) && chars.get(index + 2) == Some(&ch); - result.push(ch); - if triple_quoted { - result.push(ch); - result.push(ch); - index += 2; + } else if quote == Some(ch) { + quote = None; + } + result.push(ch); + } else if ch == '#' && quote.is_none() { + for comment_ch in chars.by_ref() { + if comment_ch == '\n' { + result.push('\n'); + break; } } - _ => result.push(ch), + } else { + result.push(ch); } - index += 1; } result } diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index 56512f4bb7d..4af1ee0bed4 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -1081,6 +1081,7 @@ struct SymbolTableBuilder { comprehension_yield_context: Option<&'static str>, // PEP 649: Track if we're inside a conditional block (if/for/while/etc.) in_conditional_block: bool, + lazy_prohibited_scopes: Vec, // Mirrors symtable ENTER_RECURSIVE guards during compilation. recursion_depth: usize, recursion_limit: usize, @@ -1122,6 +1123,7 @@ impl SymbolTableBuilder { in_iter_def_exp: false, comprehension_yield_context: None, in_conditional_block: false, + lazy_prohibited_scopes: Vec::new(), recursion_depth: 0, recursion_limit: DEFAULT_RECURSION_LIMIT, next_block_index: 0, @@ -1630,6 +1632,36 @@ impl SymbolTableBuilder { Ok(()) } + fn validate_lazy_import(&self, statement: &ast::Stmt) -> SymbolTableResult { + let (kind, star) = match statement { + ast::Stmt::Import(node) if node.is_lazy => ("import", false), + ast::Stmt::ImportFrom(node) if node.is_lazy => ( + "from ... import", + node.names + .first() + .is_some_and(|name| name.name.as_str() == "*"), + ), + _ => return Ok(()), + }; + let reason = if self.lazy_prohibited_scopes.contains(&self.tables.len()) { + Some(format!("lazy {kind} not allowed inside try/except blocks")) + } else { + match self.tables.last().unwrap().typ { + CompilerScope::Function | CompilerScope::AsyncFunction => { + Some(format!("lazy {kind} not allowed inside functions")) + } + CompilerScope::Class => Some(format!("lazy {kind} not allowed inside classes")), + _ if star => Some("lazy from ... import * is not allowed".to_owned()), + _ => None, + } + }; + if let Some(reason) = reason { + Err(self.error_ranged(reason, statement.range())) + } else { + Ok(()) + } + } + fn scan_statement(&mut self, statement: &ast::Stmt) -> SymbolTableResult { if self.recursion_depth >= self.recursion_limit { return Err(SymbolTableError { @@ -1641,6 +1673,7 @@ impl SymbolTableBuilder { self.recursion_depth += 1; let result = (|| { use ast::*; + self.validate_lazy_import(statement)?; self.track_future_statement(statement)?; match &statement { Stmt::Global(StmtGlobal { names, .. }) => { @@ -2005,6 +2038,7 @@ impl SymbolTableBuilder { // PEP 649: Track conditional block for annotations let saved_in_conditional_block = self.in_conditional_block; self.in_conditional_block = true; + self.lazy_prohibited_scopes.push(self.tables.len()); self.scan_statements(body)?; for handler in handlers { let ExceptHandler::ExceptHandler(ast::ExceptHandlerExceptHandler { @@ -2023,6 +2057,7 @@ impl SymbolTableBuilder { } self.scan_statements(orelse)?; self.scan_statements(finalbody)?; + self.lazy_prohibited_scopes.pop(); self.in_conditional_block = saved_in_conditional_block; } Stmt::Match(StmtMatch { subject, cases, .. }) => { diff --git a/crates/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs index 7c139e77a8e..06e8f27a604 100644 --- a/crates/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -1,8 +1,8 @@ -use crate::strip_python_comments; +use crate::{interpolation_debug_text, interpolation_expression_text}; use alloc::fmt; use core::fmt::Display as _; use ruff_python_ast as ast; -use ruff_text_size::{Ranged, TextSize, TextSlice}; +use ruff_text_size::Ranged; use rustpython_compiler_core::SourceFile; use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; @@ -615,11 +615,8 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { fmt::from_fn(|f| Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1)) .to_string(); if let Some(debug_text) = debug_text { - let leading = debug_text.leading(); - let trailing = debug_text.trailing(); - self.p(leading)?; - self.p(self.source.source_text().slice(val.range()))?; - self.p(trailing)?; + let (text, _) = interpolation_debug_text(self.source, debug_text, val.range()); + self.unparse_fstring_str(&text)?; if conversion == ast::ConversionFlag::None && spec.is_none() { conversion = ast::ConversionFlag::Repr; } @@ -725,23 +722,11 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { &mut self, interpolation: &ast::InterpolatedElement, ) -> fmt::Result { - let source_conversion = interpolation.conversion; - let mut conversion = source_conversion; - let debug_parts = interpolation.debug_text.as_ref().map(|debug_text| { - ( - strip_python_comments(debug_text.leading()), - strip_python_comments( - self.source - .source_text() - .slice(interpolation.expression.range()), - ), - strip_python_comments(debug_text.trailing()), - ) - }); - if let Some((leading, source, trailing)) = &debug_parts { - self.p(leading)?; - self.p(source)?; - self.p(trailing)?; + let mut conversion = interpolation.conversion; + if let Some(debug_text) = &interpolation.debug_text { + let (text, _) = + interpolation_debug_text(self.source, debug_text, interpolation.expression.range()); + self.unparse_fstring_str(&text)?; if conversion == ast::ConversionFlag::None && interpolation.format_spec.is_none() { conversion = ast::ConversionFlag::Repr; } @@ -749,42 +734,14 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { let expression = if let Some(ast::ConstantValue::Str(value)) = &interpolation.runtime_str { value.to_string() - } else if let Some((leading, source, trailing)) = &debug_parts { - let mut expression = leading.clone(); - expression.push_str(source); - let equal = trailing - .rfind('=') - .expect("debug interpolation must contain '='"); - expression.push_str(&trailing[..equal]); - expression.trim_end().to_owned() + } else if let Some(expression) = interpolation_expression_text(self.source, interpolation) { + expression } else { - let expression_range = interpolation.expression.range(); - let after_brace = interpolation.range.start() + TextSize::new(1); - let mut expression_end = interpolation.format_spec.as_ref().map_or_else( - || interpolation.range.end() - TextSize::new(1), - |format_spec| format_spec.range.start() - TextSize::new(1), - ); - if source_conversion != ast::ConversionFlag::None { - expression_end -= TextSize::new(2); - } - if interpolation.range.start() < expression_range.start() - && interpolation.range.end() >= expression_range.end() - && after_brace <= expression_end - { - strip_python_comments( - self.source - .source_text() - .slice(ruff_text_size::TextRange::new(after_brace, expression_end)), - ) - .trim_end() - .to_owned() - } else { - fmt::from_fn(|f| { - Unparser::new(f, self.source) - .unparse_expr(&interpolation.expression, precedence::TEST + 1) - }) - .to_string() - } + fmt::from_fn(|f| { + Unparser::new(f, self.source) + .unparse_expr(&interpolation.expression, precedence::TEST + 1) + }) + .to_string() }; self.p(if expression.starts_with('{') { diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 71c2dfad9a1..82f881d9598 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -955,7 +955,8 @@ pub fn too_deeply_nested_error( /// A bare generator expression is one: `f(x for x in y)` is the `primary /// genexp` alternative of a call, and a class header only takes `arguments`, /// which has no such alternative. A format spec nested more than two deep is -/// the other; the tokenizer runs out of nesting levels for it. +/// the other; the tokenizer runs out of nesting levels for it. A lazy +/// `__future__` import is rejected by the `checked_from_import` action. #[doc(hidden)] #[must_use] pub fn unsupported_grammar_error(ast: &ast::Mod, source_file: &SourceFile) -> Option { @@ -1011,6 +1012,18 @@ pub fn unsupported_grammar_error(ast: &ast::Mod, source_file: &SourceFile) -> Op if self.error.is_some() { return; } + if let ast::Stmt::ImportFrom(import) = stmt + && import.is_lazy + && import.level == 0 + && import + .module + .as_ref() + .is_some_and(|module| module.as_str() == "__future__") + { + let lazy = ruff_text_size::TextRange::at(import.range.start(), 4.into()); + self.fail("lazy from __future__ import is not allowed", lazy); + return; + } if let ast::Stmt::ClassDef(class_def) = stmt && let Some(arguments) = &class_def.arguments && let [ast::Expr::Generator(generator)] = &arguments.args[..] diff --git a/crates/derive-impl/src/compile_bytecode.rs b/crates/derive-impl/src/compile_bytecode.rs index 8c3198ff1ff..e08c230e27d 100644 --- a/crates/derive-impl/src/compile_bytecode.rs +++ b/crates/derive-impl/src/compile_bytecode.rs @@ -177,6 +177,10 @@ impl CompilationSource { }; code_map.extend(self.compile_dir(base, &path, &nparent, mode, compiler)?); } else if file_name.ends_with(".py") { + // These fixtures must be parsed at import time to exercise syntax errors. + if parent == "test.test_lazy_import.data.badsyntax" { + continue; + } let stem = path.file_stem().unwrap().to_str().unwrap(); let is_init = stem == "__init__"; let module_name = if is_init { diff --git a/crates/stdlib/src/_tokenize.rs b/crates/stdlib/src/_tokenize.rs index 65d6755632a..8d09da12dc8 100644 --- a/crates/stdlib/src/_tokenize.rs +++ b/crates/stdlib/src/_tokenize.rs @@ -664,6 +664,7 @@ mod _tokenize { match kind { TokenKind::EndOfFile => 0, TokenKind::Identifier + | TokenKind::Lazy | TokenKind::For | TokenKind::In | TokenKind::Pass @@ -764,7 +765,6 @@ mod _tokenize { TokenKind::TStringMiddle => 63, TokenKind::TStringEnd => 64, TokenKind::IpyEscapeCommand | TokenKind::Question | TokenKind::Unknown => 67, // ERRORTOKEN - TokenKind::Lazy => u8::MAX, // Placeholder: RustPython Doesn't support `lazy imports` yet } } } diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 60ed4e65887..dfc81cb167f 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -1220,6 +1220,18 @@ impl Py { } } + pub(crate) fn replace_if_identity( + &self, + name: &Py, + expected: &PyObject, + replacement: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { + let hash = PyDict::hash_or_unhashable(name, vm)?; + self.entries + .replace_if(vm, name, hash, replacement, |current| current.is(expected)) + } + /// Store using a cached entry index hint for the value-replace fast path. /// /// On a hint miss, returns a refreshed hint for the key (`None` when the diff --git a/crates/vm/src/builtins/interpolation.rs b/crates/vm/src/builtins/interpolation.rs index 73126102ea9..78505631563 100644 --- a/crates/vm/src/builtins/interpolation.rs +++ b/crates/vm/src/builtins/interpolation.rs @@ -19,7 +19,7 @@ pub struct PyInterpolation { #[pymember(type = "object_ex")] pub value: PyObjectRef, #[pymember(type = "object_ex")] - pub expression: PyStrRef, + pub expression: PyObjectRef, #[pymember(type = "object_ex")] pub conversion: PyObjectRef, // None or 's', 'r', 'a' #[pymember(type = "object_ex")] @@ -36,7 +36,7 @@ impl PyPayload for PyInterpolation { impl PyInterpolation { pub fn new( value: PyObjectRef, - expression: PyStrRef, + expression: PyObjectRef, conversion: PyObjectRef, format_spec: PyStrRef, vm: &VirtualMachine, @@ -80,7 +80,7 @@ impl Constructor for PyInterpolation { vm.ctx.none() }; - let expression = args.expression; + let expression = args.expression.into(); let format_spec = args.format_spec; Ok(Self { @@ -188,7 +188,7 @@ impl Representable for PyInterpolation { let mut result = Wtf8Buf::from("Interpolation("); result.push_wtf8(value_repr.as_wtf8()); result.push_str(", "); - result.push_str(&expr_repr); + result.push_wtf8(expr_repr.as_wtf8()); result.push_str(", "); if vm.is_none(&zelf.conversion) { result.push_str("None"); diff --git a/crates/vm/src/builtins/module.rs b/crates/vm/src/builtins/module.rs index e479cd54305..178ef2659c2 100644 --- a/crates/vm/src/builtins/module.rs +++ b/crates/vm/src/builtins/module.rs @@ -178,11 +178,56 @@ impl Py { Ok(()) } - fn getattr_inner(&self, name: &Py, vm: &VirtualMachine) -> PyResult { - if let Some(attr) = self.as_object().generic_getattr_opt(name, None, vm)? { + pub(crate) fn getattr_inner( + &self, + name: &Py, + suppress_cycle: bool, + vm: &VirtualMachine, + ) -> PyResult { + let mut attr = self.as_object().generic_getattr_opt(name, None, vm)?; + if attr.is_none() { + if let Some(value) = + crate::lazy_import::try_load_submodule(self, name, suppress_cycle, vm)? + { + return Ok(value); + } + // A concurrent import may have bound a child and removed its pending entry. + if self.class().is(vm.ctx.types.module_type) + || self + .class() + .get_attr(vm.ctx.intern_str(name.as_wtf8())) + .is_none_or(|descriptor| descriptor.class().slots().descr_get.load().is_none()) + { + attr = self.dict().get_item_opt(name, vm)?; + } + } + if let Some(attr) = attr { + if let Some(deferred) = + attr.downcast_ref_if_exact::(vm) + { + if name.to_str() != Some("__getattr__") + && let Some(getattr) = crate::lazy_import::resolved_dict_item( + &self.dict(), + identifier!(vm, __getattr__), + vm, + )? + { + match getattr.call((name.to_owned(),), vm) { + Ok(value) => return Ok(value), + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.attribute_error) => {} + Err(exc) => return Err(exc), + } + } + if suppress_cycle && crate::lazy_import::is_resolving(deferred, vm) { + return Err(vm.new_attribute_error(format!("module has no attribute '{name}'"))); + } + return crate::lazy_import::reify(deferred, name, self.dict().as_object(), vm); + } return Ok(attr); } - if let Ok(getattr) = self.dict().get_item(identifier!(vm, __getattr__), vm) { + if let Some(getattr) = + crate::lazy_import::resolved_dict_item(&self.dict(), identifier!(vm, __getattr__), vm)? + { return getattr.call((name.to_owned(),), vm); } let dict = self.dict(); @@ -299,7 +344,7 @@ impl Py { pub fn get_attr<'a>(&self, attr_name: impl AsPyStr<'a>, vm: &VirtualMachine) -> PyResult { let attr_name = attr_name.as_pystr(&vm.ctx); - self.getattr_inner(attr_name, vm) + self.getattr_inner(attr_name, false, vm) } pub fn set_attr<'a>( @@ -330,7 +375,9 @@ impl PyModule { .downcast::() .map_err(|_| vm.new_type_error(".__dict__ is not a dictionary"))?; // PEP 562: honor a module-level __dir__ if one is defined - if let Some(dir_func) = dict.get_item_opt(identifier!(vm, __dir__), vm)? { + if let Some(dir_func) = + crate::lazy_import::resolved_dict_item(&dict, identifier!(vm, __dir__), vm)? + { return dir_func.call((), vm)?.try_to_value(vm); } let attrs = dict.into_iter().map(|(k, _v)| k).collect(); @@ -467,7 +514,7 @@ impl Initializer for PyModule { impl GetAttr for PyModule { fn getattro(zelf: &Py, name: &Py, vm: &VirtualMachine) -> PyResult { - zelf.getattr_inner(name, vm) + zelf.getattr_inner(name, false, vm) } } diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index d75a937bb27..d7dca044bbe 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -3214,11 +3214,17 @@ impl SetAttr for PyType { } else { let prev_value = zelf.attributes().remove(attr_name); if prev_value.is_none() { - return Err(vm.new_attribute_error(format!( + let exc = vm.new_attribute_error(format!( "type object '{}' has no attribute '{}'", zelf.name(), attr_name, - ))); + )); + vm.set_attribute_error_context( + &exc, + zelf.to_owned().into(), + attr_name.to_owned(), + ); + return Err(exc); } prev_value }; diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 9bbc2550a75..8b4a8eb6def 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -1381,14 +1381,15 @@ impl Dict { Ok(removed) } - /// Move an existing entry to the end and return its current value without - /// making the key temporarily absent. Equality runs unlocked. - pub(crate) fn move_to_end( + // Locate an entry and mutate it under one guard. The operation must not + // run Python or drop owned Python references while the guard is held. + fn with_entry_mut( &self, vm: &VirtualMachine, key: &K, hash_value: HashValue, - ) -> PyResult> { + update: impl FnOnce(&mut DictInner, EntryIndex) -> R, + ) -> PyResult> { struct ProbeWitness { index_index: IndexIndex, entry_index: IndexEntry, @@ -1480,6 +1481,21 @@ impl Dict { } }, }; + let result = update(&mut inner, entry_index); + drop(inner); + return Ok(Some(result)); + } + } + + /// Move an existing entry to the end and return its current value without + /// making the key temporarily absent. Equality runs unlocked. + pub(crate) fn move_to_end( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult> { + self.with_entry_mut(vm, key, hash, |inner, entry_index| { let value = inner.entries[entry_index].as_ref().unwrap().value.clone(); if inner.entries[entry_index + 1..].iter().any(Option::is_some) { // Reserve before taking the entry. No reader can observe an @@ -1503,9 +1519,25 @@ impl Dict { inner.resize(indices_size); } } - drop(inner); - return Ok(Some(value)); - } + value + }) + } + + // Replace only a still-matching value. The predicate must not run Python. + // Keep both the replacement and removed value alive outside the guard. + pub(crate) fn replace_if( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + replacement: T, + matches: impl FnOnce(&T) -> bool, + ) -> PyResult { + let removed = self.with_entry_mut(vm, key, hash, |inner, index| { + let entry = inner.entries[index].as_mut().unwrap(); + matches(&entry.value).then(|| core::mem::replace(&mut entry.value, replacement.clone())) + })?; + Ok(removed.flatten().is_some()) } pub(crate) fn pop_back(&self) -> Option<(PyObjectRef, T)> { diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 40d714b4d38..8adee7270e9 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -549,6 +549,7 @@ pub struct ExceptionZoo { pub attribute_error: &'static Py, pub buffer_error: &'static Py, pub eof_error: &'static Py, + pub import_cycle_error: &'static Py, pub import_error: &'static Py, pub module_not_found_error: &'static Py, pub lookup_error: &'static Py, @@ -926,6 +927,7 @@ impl ExceptionZoo { let eof_error = PyEOFError::init_builtin_type(); let import_error = PyImportError::init_builtin_type(); + let import_cycle_error = PyImportCycleError::init_builtin_type(); let module_not_found_error = PyModuleNotFoundError::init_builtin_type(); let lookup_error = PyLookupError::init_builtin_type(); @@ -1010,6 +1012,7 @@ impl ExceptionZoo { attribute_error, buffer_error, eof_error, + import_cycle_error, import_error, module_not_found_error, lookup_error, @@ -1104,6 +1107,7 @@ impl ExceptionZoo { extend_exception!(PyEOFError, ctx, excs.eof_error); extend_exception!(PyImportError, ctx, excs.import_error); + extend_exception!(PyImportCycleError, ctx, excs.import_cycle_error); extend_exception!(PyModuleNotFoundError, ctx, excs.module_not_found_error); extend_exception!(PyLookupError, ctx, excs.lookup_error); @@ -2203,6 +2207,11 @@ pub(super) mod types { } } + #[pyexception(name, base = PyImportError, ctx = "import_cycle_error", impl)] + #[derive(Debug)] + #[repr(transparent)] + pub struct PyImportCycleError(PyImportError); + #[pyexception(name, base = PyImportError, ctx = "module_not_found_error", impl)] #[derive(Debug)] #[repr(transparent)] diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 1dd729367ac..07d5a020983 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4672,9 +4672,12 @@ impl ExecutingFrame<'_> { vm.ctx.empty_str.to_owned() }; - let expression = self.pop_value().downcast::().map_err(|_| { - vm.new_type_error("BUILD_INTERPOLATION expected str for expression") - })?; + let expression = self.pop_value(); + if !vm.is_none(&expression) && expression.downcast_ref::().is_none() { + return Err(vm.new_type_error( + "BUILD_INTERPOLATION expected str or None for expression", + )); + } let value = self.pop_value(); // conversion: 0=None, 1=Str, 2=Repr, 3=Ascii @@ -5117,7 +5120,8 @@ impl ExecutingFrame<'_> { Ok(None) } Instruction::ImportName { namei: idx } => { - self.import(vm, Some(self.code.names[(idx.get(arg) >> 2) as usize]))?; + let packed = idx.get(arg); + self.import(vm, self.code.names[(packed >> 2) as usize], packed & 3)?; Ok(None) } Instruction::IsOp { invert } => { @@ -5238,7 +5242,7 @@ impl ExecutingFrame<'_> { let name = self.localsplus_name(idx); let value = self.mapping_get_optional(&class_dict, name, vm)?; self.push_value(match value { - Some(v) => v, + Some(v) => crate::lazy_import::reify_value(v, name, &class_dict, vm)?, None => self .cell_ref(idx) .get() @@ -5253,7 +5257,7 @@ impl ExecutingFrame<'_> { let value = self.mapping_get_optional(&dict, name, vm)?; self.push_value(match value { - Some(v) => v, + Some(v) => crate::lazy_import::reify_value(v, name, &dict, vm)?, None => self.load_global_or_builtin(name, vm)?, }); Ok(None) @@ -5396,13 +5400,16 @@ impl ExecutingFrame<'_> { Instruction::LoadName { namei: idx } => { let name = self.code.names[idx.get(arg) as usize]; let result = self.locals.mapping(vm).subscript(name, vm); - match result { - Ok(x) => self.push_value(x), + let value = match result { + Ok(x) => { + crate::lazy_import::reify_value(x, name, self.locals.as_object(vm), vm)? + } Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { - self.push_value(self.load_name_global_or_builtin(name, vm)?); + self.load_name_global_or_builtin(name, vm)? } Err(e) => return Err(e), - } + }; + self.push_value(value); Ok(None) } Instruction::LoadSpecial { method } => { @@ -6395,6 +6402,9 @@ impl ExecutingFrame<'_> { module .dict() .get_cached_module_attr(attr_name, keys_version, index, vm) + && value + .downcast_ref_if_exact::(vm) + .is_none() { self.pop_stackref(); if oparg.is_method() { @@ -8178,6 +8188,8 @@ impl ExecutingFrame<'_> { && let Some(x) = globals.get_item_by_index_and_keys_version(cached_version, cached_index) { + let name = self.code.names[(oparg >> 1) as usize]; + let x = crate::lazy_import::reify_value(x, name, self.globals.as_object(), vm)?; self.push_value(x); if (oparg & 1) != 0 { self.push_value_opt(None); @@ -8207,6 +8219,8 @@ impl ExecutingFrame<'_> { && let Some(x) = builtins_dict .get_item_by_index_and_keys_version(cached_builtins_ver, cached_index) { + let name = self.code.names[(oparg >> 1) as usize]; + let x = crate::lazy_import::reify_value(x, name, self.builtins, vm)?; self.push_value(x); if (oparg & 1) != 0 { self.push_value_opt(None); @@ -8728,16 +8742,17 @@ impl ExecutingFrame<'_> { .as_object() .downcast_ref::() .expect("exact dict globals"); - let globals_exact = unsafe { PyExact::ref_unchecked(globals) }; - globals_exact - .get_chain_exact(builtins_dict, name, vm)? - .ok_or_else(|| { - vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) - }) + if let Some(value) = globals.get_item_opt(name, vm)? { + return crate::lazy_import::reify_value(value, name, globals.as_object(), vm); + } + let value = builtins_dict.get_item_opt(name, vm)?.ok_or_else(|| { + vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) + })?; + crate::lazy_import::reify_value(value, name, builtins_dict.as_object(), vm) } else { // Slow path: builtins is not a dict, use generic __getitem__ if let Some(value) = self.globals.get_item_opt(name, vm)? { - return Ok(value); + return crate::lazy_import::reify_value(value, name, self.globals.as_object(), vm); } self.load_builtin(name, vm) } @@ -8747,28 +8762,33 @@ impl ExecutingFrame<'_> { // LOAD_NAME reads the stored global entries even for dict subclasses; // LOAD_GLOBAL instead honors their __getitem__ implementation. if let Some(value) = self.globals.inner_getitem_opt(name, vm)? { - return Ok(value); + return crate::lazy_import::reify_value(value, name, self.globals.as_object(), vm); } self.load_builtin(name, vm) } fn load_builtin(&self, name: &Py, vm: &VirtualMachine) -> PyResult { - self.builtins.get_item(name, vm).map_err(|e| { + let value = self.builtins.get_item(name, vm).map_err(|e| { if e.fast_isinstance(vm.ctx.exceptions.key_error) { vm.new_name_error(format!("name '{name}' is not defined"), name.to_owned()) } else { e } - }) + })?; + crate::lazy_import::reify_value(value, name, self.builtins, vm) } #[cfg_attr(feature = "flame-it", flame("FrameObject"))] - fn import(&mut self, vm: &VirtualMachine, module_name: Option<&Py>) -> PyResult<()> { - let module_name = module_name.unwrap_or(vm.ctx.empty_str); + fn import( + &mut self, + vm: &VirtualMachine, + module_name: &Py, + policy: u32, + ) -> PyResult<()> { let from_list = self.pop_value(); let level = usize::try_from_object(vm, self.pop_value())?; - let module = vm.import_from(module_name, from_list, level)?; + let module = crate::lazy_import::import_name(module_name, from_list, level, policy, vm)?; self.push_value(module); Ok(()) @@ -8779,107 +8799,11 @@ impl ExecutingFrame<'_> { let module = self.top_value(); let name = self.code.names[idx as usize]; - // Load attribute, and transform any error into import error. - if let Some(obj) = vm.get_attribute_opt(module, name)? { - return Ok(obj); - } - // fallback to importing '{module.__name__}.{name}' from sys.modules - let fallback_module = (|| { - let mod_name = module.get_attr(identifier!(vm, __name__), vm).ok()?; - let mod_name = mod_name.downcast_ref::()?; - let full_mod_name = vm.ctx.new_utf8_str(format!("{}.{name}", mod_name.as_str())); - let sys_modules = vm.sys_module.get_attr("modules", vm).ok()?; - sys_modules.get_item(&*full_mod_name, vm).ok() - })(); - - if let Some(sub_module) = fallback_module { - return Ok(sub_module); - } - - use crate::import::{ - get_spec_file_origin, is_possibly_shadowing_path, is_stdlib_module_name, - }; - - // Get module name for the error message - let mod_name_obj = module.get_attr(identifier!(vm, __name__), vm).ok(); - let mod_name = mod_name_obj - .as_ref() - .and_then(|n| n.downcast_ref::()); - let module_name = mod_name.map_or("", |s| s.as_str()); - - let spec = module - .get_attr("__spec__", vm) - .ok() - .filter(|s| !vm.is_none(s)); - - let origin = get_spec_file_origin(spec.as_deref(), vm); - - let is_possibly_shadowing = origin - .as_ref() - .is_some_and(|o| is_possibly_shadowing_path(o, vm)); - let is_possibly_shadowing_stdlib = if is_possibly_shadowing { - if let Some(ref mod_name) = mod_name_obj { - is_stdlib_module_name(mod_name, vm)? - } else { - false - } - } else { - false - }; - - let msg = if is_possibly_shadowing_stdlib { - let origin = origin.as_ref().unwrap(); - format!( - "cannot import name '{name}' from '{module_name}' \ - (consider renaming '{origin}' since it has the same \ - name as the standard library module named '{module_name}' \ - and prevents importing that standard library module)" - ) - } else { - let is_init = is_module_initializing(module, vm); - if is_init { - if is_possibly_shadowing { - let origin = origin.as_ref().unwrap(); - format!( - "cannot import name '{name}' from '{module_name}' \ - (consider renaming '{origin}' if it has the same name \ - as a library you intended to import)" - ) - } else if let Some(ref path) = origin { - format!( - "cannot import name '{name}' from partially initialized module \ - '{module_name}' (most likely due to a circular import) ({path})" - ) - } else { - format!( - "cannot import name '{name}' from partially initialized module \ - '{module_name}' (most likely due to a circular import)" - ) - } - } else if let Some(ref path) = origin { - format!("cannot import name '{name}' from '{module_name}' ({path})") - } else { - format!("cannot import name '{name}' from '{module_name}' (unknown location)") - } - }; - let err = vm.new_import_error( - msg, - match mod_name { - Some(s) => s.to_owned().into_wtf8(), - None => vm.ctx.new_utf8_str("").into_wtf8(), - }, - ); - - if let Some(ref path) = origin { - let _ignore = err - .as_object() - .set_attr("path", vm.ctx.new_str(path.as_str()), vm); + if let Some(deferred) = module.downcast_ref_if_exact::(vm) + { + return crate::lazy_import::import_from(deferred, name, vm); } - - // name_from = the attribute name that failed to import (best-effort metadata) - let _ignore = err.as_object().set_attr("name_from", name.to_owned(), vm); - - Err(err) + crate::import::import_from_attribute(module, name, vm) } #[cfg_attr(feature = "flame-it", flame("FrameObject"))] @@ -12972,19 +12896,6 @@ fn get_special_method_error_msg( } } -fn is_module_initializing(module: &PyObject, vm: &VirtualMachine) -> bool { - let Ok(spec) = module.get_attr(&vm.ctx.new_str("__spec__"), vm) else { - return false; - }; - if vm.is_none(&spec) { - return false; - } - let Ok(initializing_attr) = spec.get_attr(&vm.ctx.new_str("_initializing"), vm) else { - return false; - }; - initializing_attr.try_to_bool(vm).unwrap_or(false) -} - fn expect_unchecked(optional: Option, err_msg: &'static str) -> T { if cfg!(debug_assertions) { optional.expect(err_msg) diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index 34911348b12..a6d137d7d2c 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -2,12 +2,22 @@ use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - builtins::{PyCode, PyStr, PyUtf8Str, PyUtf8StrRef, traceback::PyTraceback}, + builtins::{PyCode, PyStr, PyStrRef, PyUtf8Str, traceback::PyTraceback}, exceptions::types::PyBaseException, scope::Scope, vm::{VirtualMachine, resolve_frozen_alias, thread}, }; +pub(crate) fn find_and_load( + name: &Py, + function: &'static str, + vm: &VirtualMachine, +) -> PyResult { + vm.importlib + .get_attr(function, vm)? + .call((name.to_owned(), vm.import_func.clone()), vm) +} + pub(crate) fn check_pyc_magic_number_bytes(buf: &[u8]) -> bool { buf.starts_with(&crate::version::PYC_MAGIC_NUMBER_BYTES) } @@ -177,7 +187,7 @@ pub(crate) fn is_module_initializing(module: &PyObject, vm: &VirtualMachine) -> /// initializing by calling `_lock_unlock_module`. fn import_ensure_initialized( module: &PyObject, - name: &Py, + name: &Py, vm: &VirtualMachine, ) -> PyResult<()> { if is_module_initializing(module, vm)? { @@ -380,6 +390,41 @@ pub(crate) fn is_stdlib_module_name(name: &PyObject, vm: &VirtualMachine) -> PyR result.try_to_bool(vm) } +/// Resolve an import target without finding or executing its module. +pub(crate) fn absolute_import_name( + name: &Py, + globals: Option<&PyObject>, + level: i32, + vm: &VirtualMachine, +) -> PyResult { + if level < 0 { + return Err(vm.new_value_error("level must be >= 0")); + } + if level == 0 { + if name.is_empty() { + return Err(vm.new_value_error("Empty module name")); + } + return Ok(name.to_owned()); + } + let globals = globals + .ok_or_else(|| vm.new_key_error(vm.ctx.new_str("'__name__' not in globals").into()))?; + let empty: PyObjectRef; + let globals = if vm.is_none(globals) { + empty = vm.ctx.new_dict().into(); + &empty + } else { + globals + }; + let package = calc_package(Some(globals), vm)?; + if package.is_empty() { + return Err(vm.new_import_error( + "attempted relative import with no known parent package", + vm.ctx.new_utf8_str(""), + )); + } + resolve_name(name, &package, level as usize, vm) +} + /// PyImport_ImportModuleLevelObject pub(crate) fn import_module_level( name: &Py, @@ -392,21 +437,6 @@ pub(crate) fn import_module_level( return Err(vm.new_value_error("level must be >= 0")); } - let name = match name.as_utf8() { - Some(name) => name, - None => { - // Name contains surrogates. Try sys.modules lookup with the - // Python string key directly. - if level == 0 { - let sys_modules = vm.sys_module.get_attr("modules", vm)?; - return sys_modules.get_item(name, vm).map_err(|_| { - vm.new_import_error(format!("No module named '{name}'"), name.to_owned()) - }); - } - return Err(vm.new_import_error(format!("No module named '{name}'"), name.to_owned())); - } - }; - // Resolve absolute name let abs_name = if level > 0 { // When globals is not provided (Rust None), raise KeyError @@ -442,12 +472,10 @@ pub(crate) fn import_module_level( let module = match sys_modules.get_item(&*abs_name, vm) { Ok(m) if !vm.is_none(&m) => { import_ensure_initialized(&m, &abs_name, vm)?; + crate::lazy_import::clear_submodule(&abs_name, true, vm)?; m } - _ => { - let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; - find_and_load.call((abs_name.clone(), vm.import_func.clone()), vm)? - } + _ => find_and_load(&abs_name, "_find_and_load", vm)?, }; // Handle fromlist @@ -472,24 +500,23 @@ pub(crate) fn import_module_level( Ok(module) } } else if level == 0 || !name.is_empty() { - let name_str = name.as_str(); - match name_str.find('.') { + let name_bytes = name.as_bytes(); + match name_bytes.iter().position(|&byte| byte == b'.') { None => Ok(module), Some(dot) => { let to_return = if level == 0 { - vm.ctx.new_utf8_str(&name_str[..dot]) + vm.ctx.new_str(&name.as_wtf8()[..dot]) } else { - let cut_off = name_str.len() - dot; - let abs = abs_name.as_str(); - vm.ctx.new_utf8_str(&abs[..abs.len() - cut_off]) + let cut_off = name_bytes.len() - dot; + let abs = abs_name.as_wtf8(); + vm.ctx.new_str(&abs[..abs.len() - cut_off]) }; match sys_modules.get_item(&*to_return, vm) { Ok(m) => Ok(m), Err(_) if level == 0 => { // For absolute imports (level 0), try importing the // parent. Matches _bootstrap.__import__ behavior. - let find_and_load = vm.importlib.get_attr("_find_and_load", vm)?; - find_and_load.call((to_return, vm.import_func.clone()), vm) + find_and_load(&to_return, "_find_and_load", vm) } Err(_) => { // For relative imports (level > 0), raise KeyError @@ -509,37 +536,44 @@ pub(crate) fn import_module_level( /// resolve_name in import.c - resolve relative import name fn resolve_name( - name: &Py, - package: &Py, + name: &Py, + package: &Py, level: usize, vm: &VirtualMachine, -) -> PyResult { - // Python: bits = package.rsplit('.', level - 1) - // Rust: rsplitn(level, '.') gives maxsplit=level-1 - let package_str = package.as_str(); - let parts: Vec<&str> = package_str.rsplitn(level, '.').collect(); - if parts.len() < level { - return Err(vm.new_import_error( - "attempted relative import beyond top-level package", - name.to_owned(), - )); +) -> PyResult { + // ASCII dots are character boundaries in WTF-8 too. Keep the original + // Python string, including surrogates, while walking up package components. + let package_bytes = package.as_bytes(); + let mut end = package_bytes.len(); + for _ in 1..level { + end = package_bytes[..end] + .iter() + .rposition(|&byte| byte == b'.') + .ok_or_else(|| { + vm.new_import_error( + "attempted relative import beyond top-level package", + name.to_owned(), + ) + })?; } - // rsplitn returns parts right-to-left, so last() is the leftmost (base) - let base = *parts.last().unwrap(); + let base = &package.as_wtf8()[..end]; let abs_name = if name.is_empty() { - if base.len() == package_str.len() { + if end == package_bytes.len() { package.to_owned() } else { - vm.ctx.new_utf8_str(base) + vm.ctx.new_str(base) } } else { - vm.ctx.new_utf8_str(format!("{base}.{}", name.as_str())) + let mut absolute = base.to_owned(); + absolute.push_str("."); + absolute.push_wtf8(name.as_wtf8()); + vm.ctx.new_str(absolute) }; Ok(abs_name) } /// _calc___package__ - calculate package from globals for relative imports -fn calc_package(globals: Option<&PyObject>, vm: &VirtualMachine) -> PyResult { +fn calc_package(globals: Option<&PyObject>, vm: &VirtualMachine) -> PyResult { let globals = globals.ok_or_else(|| { vm.new_import_error( "attempted relative import with no known parent package", @@ -553,7 +587,7 @@ fn calc_package(globals: Option<&PyObject>, vm: &VirtualMachine) -> PyResult, vm: &VirtualMachine) -> PyResult, vm: &VirtualMachine) -> PyResult, vm: &VirtualMachine) -> PyResult vm.ctx.new_utf8_str(&s[..dot]), - None => vm.ctx.new_utf8_str(""), - }) + let name = mod_name_str.as_wtf8(); + Ok( + match name.as_bytes().iter().rposition(|&byte| byte == b'.') { + Some(dot) => vm.ctx.new_str(&name[..dot]), + None => vm.ctx.new_str(""), + }, + ) } else { Ok(mod_name_str) } } + +pub(crate) fn import_from_attribute( + module: &PyObject, + name: &Py, + vm: &VirtualMachine, +) -> PyResult { + // Load attribute, and transform any error into import error. + if let Some(obj) = vm.get_attribute_opt(module, name)? { + return Ok(obj); + } + // fallback to importing '{module.__name__}.{name}' from sys.modules + let fallback_module = (|| { + let mod_name = module.get_attr(identifier!(vm, __name__), vm).ok()?; + let mod_name = mod_name.downcast_ref::()?; + let full_mod_name = vm.ctx.new_utf8_str(format!("{}.{name}", mod_name.as_str())); + let sys_modules = vm.sys_module.get_attr("modules", vm).ok()?; + sys_modules.get_item(&*full_mod_name, vm).ok() + })(); + + if let Some(sub_module) = fallback_module { + return Ok(sub_module); + } + + use crate::import::{get_spec_file_origin, is_possibly_shadowing_path, is_stdlib_module_name}; + + // Get module name for the error message + let mod_name_obj = module.get_attr(identifier!(vm, __name__), vm).ok(); + let mod_name = mod_name_obj + .as_ref() + .and_then(|n| n.downcast_ref::()); + let module_name = mod_name.map_or("", |s| s.as_str()); + + let spec = module + .get_attr("__spec__", vm) + .ok() + .filter(|s| !vm.is_none(s)); + + let origin = get_spec_file_origin(spec.as_deref(), vm); + + let is_possibly_shadowing = origin + .as_ref() + .is_some_and(|o| is_possibly_shadowing_path(o, vm)); + let is_possibly_shadowing_stdlib = if is_possibly_shadowing { + if let Some(ref mod_name) = mod_name_obj { + is_stdlib_module_name(mod_name, vm)? + } else { + false + } + } else { + false + }; + + let msg = if is_possibly_shadowing_stdlib { + let origin = origin.as_ref().unwrap(); + format!( + "cannot import name '{name}' from '{module_name}' \ + (consider renaming '{origin}' since it has the same \ + name as the standard library module named '{module_name}' \ + and prevents importing that standard library module)" + ) + } else { + let is_init = is_module_initializing(module, vm).unwrap_or(false); + if is_init { + if is_possibly_shadowing { + let origin = origin.as_ref().unwrap(); + format!( + "cannot import name '{name}' from '{module_name}' \ + (consider renaming '{origin}' if it has the same name \ + as a library you intended to import)" + ) + } else if let Some(ref path) = origin { + format!( + "cannot import name '{name}' from partially initialized module \ + '{module_name}' (most likely due to a circular import) ({path})" + ) + } else { + format!( + "cannot import name '{name}' from partially initialized module \ + '{module_name}' (most likely due to a circular import)" + ) + } + } else if let Some(ref path) = origin { + format!("cannot import name '{name}' from '{module_name}' ({path})") + } else { + format!("cannot import name '{name}' from '{module_name}' (unknown location)") + } + }; + let err = vm.new_import_error( + msg, + match mod_name { + Some(s) => s.to_owned().into_wtf8(), + None => vm.ctx.new_utf8_str("").into_wtf8(), + }, + ); + + if let Some(ref path) = origin { + let _ignore = err + .as_object() + .set_attr("path", vm.ctx.new_str(path.as_str()), vm); + } + + // name_from = the attribute name that failed to import (best-effort metadata) + let _ignore = err.as_object().set_attr("name_from", name.to_owned(), vm); + + Err(err) +} diff --git a/crates/vm/src/lazy_import.rs b/crates/vm/src/lazy_import.rs new file mode 100644 index 00000000000..a6b37398ab3 --- /dev/null +++ b/crates/vm/src/lazy_import.rs @@ -0,0 +1,916 @@ +//! Native PEP 810 deferred bindings. Dictionaries themselves always expose raw values. + +use crate::{ + AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{ + PyBaseException, PyCode, PyDict, PyModule, PySet, PyStr, PyStrRef, PyTuple, PyType, + }, + class::PyClassImpl, + common::{ + lock::{OnceCell, PyMutex}, + wtf8::{Wtf8, Wtf8Buf, wtf8_concat}, + }, + frame, + types::{Constructor, GetAttr, Representable}, +}; +use core::sync::atomic::{AtomicBool, Ordering}; +use std::collections::HashMap; + +type PendingChildren = HashMap>>; + +fn split_parent(name: &Wtf8) -> Option<(&Wtf8, &Wtf8)> { + let dot = name.as_bytes().iter().rposition(|&byte| byte == b'.')?; + // ASCII dots always delimit code points, including next to surrogates. + Some((&name[..dot], &name[dot + 1..])) +} + +pub(crate) struct LazyImportsState { + pub(crate) modules: PyRef, + pub(crate) default_callback: OnceCell, + pub(crate) all: AtomicBool, + filter: PyMutex>, + // Removed declarations must be dropped after releasing this lock. + pending: PyMutex>, +} + +impl LazyImportsState { + pub(crate) fn new(ctx: &Context) -> Self { + Self { + modules: PySet::default().into_ref(ctx), + default_callback: OnceCell::new(), + all: AtomicBool::new(false), + filter: PyMutex::default(), + pending: PyMutex::default(), + } + } + + pub(crate) fn filter(&self) -> Option { + self.filter.lock().clone() + } + + pub(crate) fn set_filter(&self, filter: Option) { + // A removed callable's finalizer can re-enter this API. + let old = core::mem::replace(&mut *self.filter.lock(), filter); + drop(old); + } + + fn register( + &self, + name: &Py, + source: Option<&Py>, + vm: &VirtualMachine, + ) -> PyResult<()> { + let mut full = name.as_wtf8(); + while let Some((parent, child)) = split_parent(full) { + let cached = raw_imported_module(&vm.ctx.new_str(full), vm)?; + let value = if cached.as_ref().is_some_and(|value| !vm.is_none(value)) { + None + } else { + source.map(ToOwned::to_owned) + }; + let replaced = { + let mut pending = self.pending.lock(); + let children = pending.entry(parent.to_owned()).or_default(); + if source.is_some() { + children.insert(child.to_owned(), value) + } else { + children.entry(child.to_owned()).or_insert(value); + None + } + }; + drop(replaced); + full = parent; + } + Ok(()) + } +} + +#[pyclass(module = false, name = "lazy_import", traverse)] +#[derive(Debug)] +pub(crate) struct PyLazyImport { + builtins: PyObjectRef, + name: PyStrRef, + // None is an import, a tuple is an import with fromlist, and a string is one member. + fromlist: Option, + // Attribute projections retain the placeholder returned by IMPORT_NAME. + source: Option>, + declaration: Option>, + #[pytraverse(skip)] + instruction: u32, + #[pytraverse(skip)] + active: AtomicBool, +} + +impl PyPayload for PyLazyImport { + fn class(ctx: &Context) -> &'static Py { + ctx.types.lazy_import_type + } +} + +impl PyLazyImport { + fn new(builtins: PyObjectRef, name: PyStrRef, fromlist: Option) -> Self { + let frame = frame::current_thread_iframe(); + let (declaration, instruction) = if frame.is_null() { + (None, 0) + } else { + // The thread's active frame is live for this call; retain its code, not the frame. + let frame = unsafe { &*frame }; + (Some(frame.code().to_owned()), frame.get_lasti()) + }; + Self { + builtins, + name, + fromlist, + source: None, + declaration, + instruction, + active: AtomicBool::new(false), + } + } + + fn projection(source: &Py, name: &Py) -> Self { + let mut projection = Self::new( + source.builtins.clone(), + source.name.clone(), + Some(name.to_owned().into()), + ); + projection.source = Some(source.to_owned()); + projection + } + + fn root_and_attrs(&self) -> (&Self, Vec) { + let mut root = self; + let mut attrs = Vec::new(); + while let Some(source) = &root.source { + let attr = root + .fromlist + .as_ref() + .unwrap() + .downcast_ref::() + .unwrap(); + attrs.push(attr.to_owned()); + root = source; + } + attrs.reverse(); + (root, attrs) + } + + fn has_fromlist(&self) -> bool { + self.fromlist.as_ref().is_some_and(|from| { + from.downcast_ref::() + .is_none_or(|t| !t.as_slice().is_empty()) + }) + } + + fn path(&self) -> Wtf8Buf { + let (root, attrs) = self.root_and_attrs(); + let mut name = if root.has_fromlist() { + root.name.as_wtf8().to_owned() + } else { + root.name + .as_wtf8() + .split(Wtf8::new(".")) + .next() + .unwrap() + .to_owned() + }; + for attr in attrs { + name.push_char('.'); + name.push_wtf8(attr.as_wtf8()); + } + name + } + + fn display_name(&self) -> Wtf8Buf { + if self.source.is_some() { + self.path() + } else if self.has_fromlist() { + wtf8_concat!(self.name.as_wtf8(), "...") + } else { + self.name.as_wtf8().to_owned() + } + } +} + +#[pyclass(with(Constructor, GetAttr, Representable))] +impl Py { + /// Resolve this import and return its value without changing a containing dictionary. + #[pymethod] + fn resolve(&self, vm: &VirtualMachine) -> PyResult { + resolve(self, vm) + } +} + +impl Constructor for PyLazyImport { + type Args = crate::function::FuncArgs; + fn py_new(_cls: &Py, _args: Self::Args, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create 'lazy_import' instances")) + } +} + +impl GetAttr for PyLazyImport { + fn getattro(zelf: &Py, name: &Py, vm: &VirtualMachine) -> PyResult { + if let Some(value) = zelf.as_object().generic_getattr_opt(name, None, vm)? { + return Ok(value); + } + Err(vm.new_attribute_error(wtf8_concat!( + "cannot access attribute ", + name.as_object().repr(vm)?.as_wtf8(), + " on unresolved lazy import ", + vm.ctx.new_str(zelf.display_name()).repr(vm)?, + ))) + } +} + +impl Representable for PyLazyImport { + fn repr_wtf8(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok(wtf8_concat!("")) + } +} + +pub(crate) fn init_type(ctx: &'static Context) { + PyLazyImport::extend_class(ctx, ctx.types.lazy_import_type); +} + +pub(crate) fn is_module_level(vm: &VirtualMachine) -> bool { + let frame = frame::current_thread_iframe(); + if frame.is_null() { + return false; + } + let frame = unsafe { &*frame }; + frame.globals().as_object().is(frame.locals.as_object(vm)) +} + +fn optional_item( + mapping: &PyObject, + name: &Py, + vm: &VirtualMachine, +) -> PyResult> { + match mapping.get_item(name, vm) { + Ok(value) => Ok(Some(value)), + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.key_error) => Ok(None), + Err(exc) => Err(exc), + } +} + +pub(crate) fn import_name( + name: &Py, + fromlist: PyObjectRef, + level: usize, + policy: u32, + vm: &VirtualMachine, +) -> PyResult { + if policy & 2 != 0 { + return vm.import_from(name, fromlist, level); + } + let explicit = policy & 1 != 0; + let globals = vm.current_globals(); + let mut lazy = + explicit || (vm.state.lazy_imports.all.load(Ordering::Acquire) && is_module_level(vm)); + if !lazy + && is_module_level(vm) + && let Some(names) = optional_item( + globals.as_object(), + vm.ctx.intern_str("__lazy_modules__"), + vm, + )? + { + let absolute = + crate::import::absolute_import_name(name, Some(globals.as_object()), level as i32, vm)?; + lazy = names + .sequence_unchecked() + .contains(absolute.as_object(), vm)?; + } + if !lazy { + return vm.import_from(name, fromlist, level); + } + let builtins = frame::current_builtins().unwrap_or_else(|| vm.builtins.dict().into()); + let callback = optional_item(&builtins, vm.ctx.intern_str("__lazy_import__"), vm)? + .ok_or_else(|| vm.new_import_error("__lazy_import__ not found", name.to_owned()))?; + if vm + .state + .lazy_imports + .default_callback + .get() + .is_some_and(|default| callback.is(default)) + { + return create( + name, + globals.as_object(), + Some(fromlist), + level as i32, + builtins, + vm, + ); + } + let locals: PyObjectRef = vm.current_locals()?.into(); + callback.call((name.to_owned(), globals, locals, fromlist, level), vm) +} + +pub(crate) fn create( + name: &Py, + globals: &PyObject, + fromlist: Option, + level: i32, + builtins: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + let absolute = crate::import::absolute_import_name(name, Some(globals), level, vm)?; + if !is_module_level(vm) { + return Err(vm.new_exception_msg( + vm.ctx.exceptions.syntax_error.to_owned(), + "'lazy import' is only allowed at module level".into(), + )); + } + let fromlist = match fromlist.filter(|value| !vm.is_none(value)) { + Some(value) if value.downcast_ref::().is_some() => { + Some(PyTuple::new_ref(vec![value], &vm.ctx).into()) + } + value => value, + }; + if let Some(filter) = vm.state.lazy_imports.filter() { + let globals_dict = crate::builtins::PyAnyDict::from_object(globals) + .ok_or_else(|| vm.new_type_error("globals must be a dict or a frozendict"))?; + let importer = globals_dict + .inner_getitem_opt(identifier!(vm, __name__), vm)? + .unwrap_or_else(|| vm.ctx.none()); + let from = fromlist.clone().unwrap_or_else(|| vm.ctx.none()); + if !filter + .call((importer, absolute.clone(), from), vm)? + .try_to_bool(vm)? + { + return crate::import::import_module_level(name, Some(globals), fromlist, level, vm); + } + } + if let Some(value) = &fromlist { + let Some(names) = value.downcast_ref::() else { + return Err( + vm.new_type_error("lazy_import: fromlist must be None, a string, or a tuple") + ); + }; + for name in names.as_slice() { + if name.downcast_ref::().is_none() { + return Err(vm.new_type_error(format!( + "Item in ``from list'' must be str, not {:.200}", + name.class().name() + ))); + } + } + } + let deferred = + PyLazyImport::new(builtins, absolute.clone(), fromlist.clone()).into_ref(&vm.ctx); + let state = &vm.state.lazy_imports; + let existing = track_module(&absolute, vm)?; + if let Some(names) = fromlist + .as_ref() + .and_then(|value| value.downcast_ref::()) + && !names.as_slice().is_empty() + { + for name in names.as_slice() { + let name = name.downcast_ref::().unwrap(); + // Cached attributes are concrete or tracked by their own placeholder. + if let Some(module) = existing.as_ref().and_then(|m| m.downcast_ref::()) + && module.dict().get_item_opt(name, vm)?.is_some() + { + continue; + } + let fullname = vm + .ctx + .new_str(wtf8_concat!(absolute.as_wtf8(), '.', name.as_wtf8())); + track_module(&fullname, vm)?; + state.register(&fullname, None, vm)?; + } + } else { + state.register(&absolute, Some(&deferred), vm)?; + } + Ok(deferred.into()) +} + +pub(crate) fn import_from( + deferred: &Py, + name: &Py, + vm: &VirtualMachine, +) -> PyResult { + if deferred.source.is_none() + && deferred.has_fromlist() + && let Some(value) = loaded_attr(&deferred.name, name, vm)? + { + return Ok(value); + } + Ok(PyLazyImport::projection(deferred, name) + .into_ref(&vm.ctx) + .into()) +} + +struct ResolvingGuard<'a> { + vm: &'a VirtualMachine, + identity: usize, +} + +impl Drop for ResolvingGuard<'_> { + fn drop(&mut self) { + self.vm + .lazy_imports_resolving + .borrow_mut() + .remove(&self.identity); + } +} + +pub(crate) fn is_resolving(deferred: &Py, vm: &VirtualMachine) -> bool { + vm.lazy_imports_resolving + .borrow() + .contains(&deferred.as_object().get_id()) +} + +fn raw_imported_module(name: &Py, vm: &VirtualMachine) -> PyResult> { + let modules = vm.sys_module.dict().get_item("modules", vm)?; + optional_item(&modules, name, vm) +} + +// Tracking must not invoke module/spec descriptors or convert a flag to bool. +fn track_module(name: &Py, vm: &VirtualMachine) -> PyResult> { + let module = raw_imported_module(name, vm)?; + let loaded = if let Some(module) = module.as_ref().and_then(|m| m.downcast_ref::()) { + let spec = module + .dict() + .get_item_opt(vm.ctx.intern_str("__spec__"), vm)?; + match spec { + None => true, + Some(spec) if vm.is_none(&spec) => true, + Some(spec) if spec.has_inline_values() => { + let initializing = match spec.dict() { + Some(dict) => dict.get_item_opt(vm.ctx.intern_str("_initializing"), vm)?, + None => None, + }; + initializing.is_none_or(|flag| flag.is(&vm.ctx.false_value)) + } + Some(_) => false, + } + } else { + module.as_ref().is_some_and(|module| !vm.is_none(module)) + }; + if !loaded { + vm.state + .lazy_imports + .modules + .add(name.to_owned().into(), vm)?; + } + Ok(module) +} + +// Attribute caching may inspect __spec__, but never waits for module imports. +fn loaded_attr( + name: &Py, + attr: &Py, + vm: &VirtualMachine, +) -> PyResult> { + let lookup = || -> PyResult> { + let modules = vm.sys_module.dict().get_item("modules", vm)?; + let Some(module) = optional_item(&modules, name, vm)? else { + return Ok(None); + }; + let Some(module) = module.downcast_ref::() else { + return Ok(None); + }; + if crate::import::is_module_initializing(module.as_object(), vm)? { + return Ok(None); + } + if !vm.sys_module.dict().get_item("modules", vm)?.is(&modules) + || !optional_item(&modules, name, vm)?.is_some_and(|m| m.is(module)) + { + return Ok(None); + } + Ok(module + .dict() + .get_item_opt(attr, vm)? + .filter(|value| value.downcast_ref_if_exact::(vm).is_none())) + }; + match lookup() { + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.exception_type) => Ok(None), + result => result, + } +} + +fn add_declaration_context( + deferred: &PyLazyImport, + exc: &Py, + vm: &VirtualMachine, +) { + let Some(code) = &deferred.declaration else { + return; + }; + let offset = deferred.instruction.saturating_sub(1) as usize; + let location = code.locations.get(offset).map(|(location, _)| location); + if exc.__cause__().is_some() || exc.__context__().is_some() || exc.__suppress_context__() { + let note = vm.ctx.new_str(wtf8_concat!( + "lazy import of '", + deferred.display_name(), + format!( + "' declared in {} at {}:{}", + code.obj_name, + code.source_path(), + location.map_or(0, |location| location.line.get()), + ), + )); + // Adding diagnostic context must not replace the original resolution error. + let _ = (|| -> PyResult<()> { + if let Some(notes) = + vm.get_attribute_opt(exc.as_object(), vm.ctx.intern_str("__notes__"))? + && notes.sequence_unchecked().contains(note.as_object(), vm)? + { + return Ok(()); + } + exc.add_note(note, vm) + })(); + return; + } + let cause = vm.new_import_error( + wtf8_concat!( + "lazy import of '", + deferred.display_name(), + "' raised an exception during resolution", + ), + deferred.name.clone(), + ); + if let Some(location) = location { + let globals = vm.ctx.new_dict(); + let frame = frame::FrameObject::new_ref( + code.clone(), + crate::scope::Scope::new(None, globals), + deferred.builtins.clone(), + &[], + None, + false, + vm, + ); + frame.set_lasti(deferred.instruction); + let tb = crate::builtins::PyTraceback::new(None, frame, (offset * 2) as i32, location.line) + .into_ref(&vm.ctx); + cause.set_traceback(Some(tb)); + } + exc.set_cause(Some(cause)); +} + +fn resolve_impl( + deferred: &Py, + vm: &VirtualMachine, +) -> PyResult<(PyObjectRef, Option)> { + let (root, attrs) = deferred.root_and_attrs(); + if is_resolving(deferred, vm) { + let exc = vm.new_exception_msg( + vm.ctx.exceptions.import_cycle_error.to_owned(), + wtf8_concat!( + "cannot import name ", + vm.ctx.new_str(deferred.display_name()).repr(vm)?, + " (most likely due to a circular import)", + ), + ); + exc.as_object().set_attr("name", root.name.clone(), vm)?; + return Err(exc); + } + vm.with_recursion(" while resolving a lazy import", || { + let identity = deferred.as_object().get_id(); + vm.lazy_imports_resolving.borrow_mut().insert(identity); + let _resolving = ResolvingGuard { vm, identity }; + let result = (|| -> PyResult<(PyObjectRef, Option)> { + let callback = optional_item(&root.builtins, vm.ctx.intern_str("__import__"), vm)? + .ok_or_else(|| vm.new_import_error("__import__ not found", root.name.clone()))?; + let default_import = callback.is(&vm.import_func); + let fromlist = match attrs.first().filter(|_| root.has_fromlist()) { + Some(first) => PyTuple::new_ref(vec![first.clone().into()], &vm.ctx).into(), + None => root.fromlist.clone().unwrap_or_else(|| vm.ctx.none()), + }; + let mut name = root.name.clone(); + let fullname = root.name.as_wtf8(); + if attrs.is_empty() + && !root.has_fromlist() + && default_import + && root.builtins.is(vm.builtins.dict().as_object()) + && let Some(dot) = fullname.as_bytes().iter().position(|&byte| byte == b'.') + { + let mut regular = true; + let mut complete = true; + for (dot, _) in fullname + .as_bytes() + .iter() + .enumerate() + .filter(|(_, byte)| **byte == b'.') + { + let prefix = vm.ctx.new_str(&fullname[..dot]); + let Some(cached) = loaded_attr(&prefix, identifier!(vm, __name__), vm)? else { + complete = false; + break; + }; + regular = loaded_attr(&prefix, vm.ctx.intern_str("__path__"), vm)?.is_some() + && cached + .downcast_ref::() + .is_some_and(|s| s.as_wtf8() == prefix.as_wtf8()); + if !regular { + break; + } + } + let loaded = loaded_attr(&name, identifier!(vm, __name__), vm)?; + if regular && (!complete || loaded.is_none()) { + name = vm.ctx.new_str(&fullname[..dot]); + } + } + let globals = frame::current_globals() + .map_or_else(|| vm.ctx.none(), |g| g.as_object().to_owned()); + let import = |name: &PyStrRef| { + callback + .call( + ( + name.clone(), + globals.clone(), + globals.clone(), + fromlist.clone(), + 0i32, + ), + vm, + ) + .inspect_err(|exc| crate::import::remove_importlib_frames(vm, exc)) + }; + let mut module = import(&name)?; + if !name.is(&root.name) { + root.active.store(true, Ordering::Relaxed); + let package = match module.downcast_ref_if_exact::(vm) { + Some(module) => module + .dict() + .get_item_opt(vm.ctx.intern_str("__path__"), vm)? + .is_some(), + None => false, + }; + if !package { + module = import(&root.name)?; + } + } else if default_import { + clear_submodule(&name, false, vm)?; + } + let root_value = resolve_value(module, vm)?; + let imported_module = + if default_import && root_value.downcast_ref::().is_some() { + Some(root_value.clone()) + } else { + None + }; + // A custom importer's root must survive all projected attribute lookups. + let mut value = root_value.clone(); + for attr in attrs { + value = + resolve_value(crate::import::import_from_attribute(&value, &attr, vm)?, vm)?; + } + drop(root_value); + vm.state + .lazy_imports + .modules + .discard(vm.ctx.new_str(deferred.path()).into(), vm)?; + Ok((value, imported_module)) + })(); + result.inspect_err(|exc| add_declaration_context(deferred, exc, vm)) + }) +} + +pub(crate) fn resolve(deferred: &Py, vm: &VirtualMachine) -> PyResult { + resolve_impl(deferred, vm).map(|(value, _)| value) +} + +pub(crate) fn resolve_value(value: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match value.downcast_ref_if_exact::(vm) { + Some(deferred) => resolve(deferred, vm), + None => Ok(value), + } +} + +pub(crate) fn reify( + deferred: &Py, + name: &Py, + namespace: &PyObject, + vm: &VirtualMachine, +) -> PyResult { + let (value, imported_module) = resolve_impl(deferred, vm)?; + if let Some(dict) = namespace.downcast_ref_if_exact::(vm) { + let replaced = dict.replace_if_identity(name, deferred.as_object(), value.clone(), vm)?; + if !replaced + && deferred.source.is_some() + && let Some(child) = imported_module + { + let (root, _) = deferred.root_and_attrs(); + if let Some((parent_name, child_name)) = split_parent(root.name.as_wtf8()) + && name.as_wtf8() == child_name + && let Some(parent) = raw_imported_module(&vm.ctx.new_str(parent_name), vm)? + && let Some(parent) = parent.downcast_ref::() + && parent.dict().as_object().is(namespace) + { + dict.replace_if_identity(name, &child, value.clone(), vm)?; + } + } + } else if namespace + .mapping_unchecked() + .slots() + .ass_subscript + .load() + .is_some() + && optional_item(namespace, name, vm)?.is_some_and(|current| current.is(deferred)) + { + namespace.set_item(name, value.clone(), vm)?; + } + Ok(value) +} + +pub(crate) fn reify_value( + value: PyObjectRef, + name: &Py, + namespace: &PyObject, + vm: &VirtualMachine, +) -> PyResult { + match value.downcast_ref_if_exact::(vm) { + Some(deferred) => reify(deferred, name, namespace, vm), + None => Ok(value), + } +} + +pub(crate) fn resolved_dict_item( + dict: &Py, + name: &Py, + vm: &VirtualMachine, +) -> PyResult> { + let Some(value) = dict.get_item_opt(name, vm)? else { + return Ok(None); + }; + if let Some(deferred) = value.downcast_ref_if_exact::(vm) { + if is_resolving(deferred, vm) { + return Ok(None); + } + return reify(deferred, name, dict.as_object(), vm).map(Some); + } + Ok(Some(value)) +} + +fn load_child(source: &Py, name: &Py, vm: &VirtualMachine) -> PyResult { + vm.with_recursion(" while resolving a lazy import", || { + let mut child = PyLazyImport::new(source.builtins.clone(), name.to_owned(), None); + child.declaration = None; + for attr in name.as_wtf8().split(Wtf8::new(".")).skip(1) { + child = PyLazyImport::projection(&child.into_ref(&vm.ctx), &vm.ctx.new_str(attr)); + child.declaration = None; + } + child.declaration.clone_from(&source.declaration); + child.instruction = source.instruction; + let result = resolve(&child.into_ref(&vm.ctx), vm)?; + if let Some(module) = result.downcast_ref::() + && source.name.as_wtf8() != name.as_wtf8() + && module + .dict() + .get_item_opt(vm.ctx.intern_str("__path__"), vm)? + .is_none() + { + return load_child(source, &source.name, vm); + } + Ok(result) + }) +} + +// Keep pending names while their parent is initializing, without retaining +// completed declarations and their namespace mappings indefinitely. +pub(crate) fn clear_submodule(name: &Py, bind: bool, vm: &VirtualMachine) -> PyResult<()> { + let Some((parent_name, child)) = split_parent(name.as_wtf8()) else { + return Ok(()); + }; + let state = &vm.state.lazy_imports; + if !state + .pending + .lock() + .get(parent_name) + .is_some_and(|children| children.contains_key(child)) + { + return Ok(()); + } + if let Some(loaded) = raw_imported_module(name, vm)? { + match crate::import::is_module_initializing(&loaded, vm) { + Ok(true) => return Ok(()), + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.exception_type) => return Ok(()), + Err(exc) => return Err(exc), + Ok(false) => {} + } + } + let parent = raw_imported_module(&vm.ctx.new_str(parent_name), vm)?; + let initializing = match &parent { + Some(parent) => crate::import::is_module_initializing(parent, vm)?, + None => false, + }; + if bind + && !initializing + && let Some(parent) = parent + .as_ref() + .and_then(|p| p.downcast_ref_if_exact::(vm)) + && let Some(value) = raw_imported_module(name, vm)? + && !vm.is_none(&value) + { + parent + .dict() + .setdefault(vm.ctx.new_str(child).into(), value, vm)?; + } + let removed = { + let mut pending = state.pending.lock(); + if let Some(children) = pending.get_mut(parent_name) { + if initializing { + children.get_mut(child).and_then(Option::take) + } else { + children.remove(child).flatten() + } + } else { + None + } + }; + drop(removed); + Ok(()) +} + +pub(crate) fn try_load_submodule( + module: &Py, + name: &Py, + suppress: bool, + vm: &VirtualMachine, +) -> PyResult> { + let Some(module_name) = module.dict().get_item_opt(identifier!(vm, __name__), vm)? else { + return Ok(None); + }; + let Some(module_name) = module_name.downcast_ref::() else { + return Ok(None); + }; + let module_name = module_name.as_wtf8(); + let child = name.as_wtf8(); + let source = vm + .state + .lazy_imports + .pending + .lock() + .get(module_name) + .and_then(|children| children.get(child).cloned()); + let Some(source) = source else { + return Ok(None); + }; + let fullname = vm.ctx.new_str(wtf8_concat!(module_name, '.', child)); + let imported = (|| { + if let Some(source) = &source + && !suppress + && (source.active.load(Ordering::Relaxed) + || module + .dict() + .get_item_opt(identifier!(vm, __getattr__), vm)? + .is_none()) + { + load_child(source, &fullname, vm) + } else { + if let Some(loaded) = raw_imported_module(&fullname, vm)? + && !vm.is_none(&loaded) + { + return Ok(loaded); + } + crate::import::find_and_load(&fullname, "_find_and_load_lazy_submodule", vm) + } + })(); + let imported = match imported { + Ok(imported) => imported, + Err(exc) => { + if let Ok(Some(loaded)) = raw_imported_module(&fullname, vm) + && !vm.is_none(&loaded) + { + let _ = PyDict::hash_or_unhashable(name, vm).and_then(|hash| { + module + .dict() + .entries + .delete_if(vm, name, hash, |value| Ok(value.is(&loaded))) + }); + let mut pending = vm.state.lazy_imports.pending.lock(); + pending + .entry(module_name.to_owned()) + .or_default() + .entry(child.to_owned()) + .or_insert_with(|| source.clone()); + } + crate::import::remove_importlib_frames(vm, &exc); + return Err(exc); + } + }; + if vm.is_none(&imported) { + return Ok(None); + } + let initializing = match crate::import::is_module_initializing(&imported, vm) { + Ok(initializing) => initializing, + Err(exc) if exc.fast_isinstance(vm.ctx.exceptions.exception_type) => false, + Err(exc) => return Err(exc), + }; + if !initializing { + module.dict().set_item(name, imported.clone(), vm)?; + let removed = vm + .state + .lazy_imports + .pending + .lock() + .get_mut(module_name) + .and_then(|children| children.remove(child)); + drop(removed); + } + Ok(Some(imported)) +} diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index c6813f63e3a..f770193aa72 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -73,6 +73,7 @@ pub mod getpath; pub mod import; mod intern; pub mod iter; +pub(crate) mod lazy_import; pub mod object; #[cfg(feature = "host_env")] diff --git a/crates/vm/src/stdlib/_ast.rs b/crates/vm/src/stdlib/_ast.rs index 57b86c9ef1b..a899e248115 100644 --- a/crates/vm/src/stdlib/_ast.rs +++ b/crates/vm/src/stdlib/_ast.rs @@ -465,12 +465,26 @@ fn copy_ast_passthrough_fields( &["kind"] } else if is_node_instance(vm, target, pyast::NodeExprInterpolation::static_type())? { &["str"] + } else if is_node_instance(vm, target, pyast::NodeStmtImport::static_type())? + || is_node_instance(vm, target, pyast::NodeStmtImportFrom::static_type())? + { + &["is_lazy"] } else { &[] }; for field in fields { if let Some(value) = vm.get_attribute_opt(source, *field)? { + let value = if *field == "is_lazy" { + let value = if vm.is_none(&value) { + 0 + } else { + node_object_to_i32(vm, &value)? + }; + vm.ctx.new_int(value).into() + } else { + value + }; target.set_attr(*field, value, vm)?; } } diff --git a/crates/vm/src/stdlib/_ast/pyast.rs b/crates/vm/src/stdlib/_ast/pyast.rs index 46bf0918d15..20e0940fb0e 100644 --- a/crates/vm/src/stdlib/_ast/pyast.rs +++ b/crates/vm/src/stdlib/_ast/pyast.rs @@ -351,14 +351,14 @@ impl_node!( impl_node!( #[pyclass(module = "ast", name = "Import", base = NodeStmt)] pub(crate) struct NodeStmtImport, - fields: ["names"], + fields: ["names", "is_lazy"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); impl_node!( #[pyclass(module = "ast", name = "ImportFrom", base = NodeStmt)] pub(crate) struct NodeStmtImportFrom, - fields: ["module", "names", "level"], + fields: ["module", "names", "level", "is_lazy"], attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"], ); @@ -1209,13 +1209,20 @@ const FIELD_TYPES: &[(&str, &[(&str, FieldType)])] = &[ ("msg", FieldType::Optional("expr")), ], ), - ("Import", &[("names", FieldType::ListOf("alias"))]), + ( + "Import", + &[ + ("names", FieldType::ListOf("alias")), + ("is_lazy", FieldType::OptionalBuiltin("int")), + ], + ), ( "ImportFrom", &[ ("module", FieldType::OptionalBuiltin("str")), ("names", FieldType::ListOf("alias")), ("level", FieldType::OptionalBuiltin("int")), + ("is_lazy", FieldType::OptionalBuiltin("int")), ], ), ("Global", &[("names", FieldType::ListOfBuiltin("str"))]), diff --git a/crates/vm/src/stdlib/_ast/python.rs b/crates/vm/src/stdlib/_ast/python.rs index 297ffbc6787..7e146a5d479 100644 --- a/crates/vm/src/stdlib/_ast/python.rs +++ b/crates/vm/src/stdlib/_ast/python.rs @@ -13,6 +13,7 @@ pub(crate) mod _ast { function::{ArgIterable, FuncArgs, KwArgs, PyMethodDef, PyMethodFlags}, stdlib::_ast::repr, types::{Constructor, Initializer}, + warn, }; #[pyattr] #[pyclass(module = "ast", name = "AST")] @@ -153,22 +154,6 @@ pub(crate) mod _ast { Ok(contained) } - fn ast_replace_set_difference_update( - expecting: &Py, - iterable: Option<&PyObject>, - vm: &VirtualMachine, - ) -> PyResult<()> { - let Some(iterable) = iterable else { - return Ok(()); - }; - let iterable = iterable.to_owned().try_into_value::(vm)?; - for item in iterable.iter(vm)? { - let item = item?; - ast_replace_set_discard(expecting, &item, vm)?; - } - Ok(()) - } - fn ast_set_attr( obj: &PyObject, name: &PyObject, @@ -192,57 +177,6 @@ pub(crate) mod _ast { let attributes = cls.get_attr(vm.ctx.intern_str("_attributes")); let dict = zelf.as_object().dict(); - let expecting = PySet::default().into_ref(&vm.ctx); - ast_replace_set_update(&expecting, fields.as_deref(), vm)?; - ast_replace_set_update(&expecting, attributes.as_deref(), vm)?; - - for (key, _value) in &args.kwargs { - let key_obj: PyObjectRef = vm.ctx.new_str(key.as_ref()).into(); - if !ast_replace_set_discard(&expecting, &key_obj, vm)? { - return Err(vm.new_type_error(format!( - "{}.__replace__ got an unexpected keyword argument '{}'.", - cls.name(), - key - ))); - } - } - - if let Some(dict) = dict.as_deref() { - for (key, _value) in dict.items_vec() { - ast_replace_set_discard(&expecting, &key, vm)?; - } - ast_replace_set_difference_update(&expecting, attributes.as_deref(), vm)?; - } - - // Discard optional fields (T | None). - if let Some(field_types) = cls.get_attr(vm.ctx.intern_str("_field_types")) - && let Ok(field_types) = field_types.downcast::() - { - for (key, value) in field_types.items_vec() { - if value.fast_isinstance(vm.ctx.types.union_type) { - ast_replace_set_discard(&expecting, &key, vm)?; - } - } - } - - let remaining = expecting.elements(); - if !remaining.is_empty() { - let mut names = Vec::with_capacity(remaining.len()); - for name in &remaining { - names.push(name.repr(vm)?.to_string()); - } - names.sort(); - let missing = names.join(", "); - let count = names.len(); - return Err(vm.new_type_error(format!( - "{}.__replace__ missing {} keyword argument{}: {}.", - cls.name(), - count, - if count == 1 { "" } else { "s" }, - missing - ))); - } - let payload = vm.ctx.new_dict(); if let Some(dict) = dict { ast_replace_update_payload(&payload, fields.as_deref(), &dict, vm)?; @@ -308,6 +242,54 @@ pub(crate) mod _ast { Ok(vm.ctx.new_str(repr)) } + fn abstract_types() -> [&'static Py; 13] { + [ + NodeAst::static_type(), + super::super::pyast::NodeMod::static_type(), + super::super::pyast::NodeStmt::static_type(), + super::super::pyast::NodeExpr::static_type(), + super::super::pyast::NodeExprContext::static_type(), + super::super::pyast::NodeBoolOp::static_type(), + super::super::pyast::NodeOperator::static_type(), + super::super::pyast::NodeUnaryOp::static_type(), + super::super::pyast::NodeCmpOp::static_type(), + super::super::pyast::NodeExceptHandler::static_type(), + super::super::pyast::NodePattern::static_type(), + super::super::pyast::NodeTypeIgnore::static_type(), + super::super::pyast::NodeTypeParam::static_type(), + ] + } + + #[pyfunction] + fn _is_abstract(cls: PyObjectRef, vm: &VirtualMachine) -> PyResult { + is_abstract(&cls, vm) + } + + fn is_abstract(cls: &PyObject, vm: &VirtualMachine) -> PyResult { + // PySet_Contains rejects mutable sets and preserves comparison errors; + // Python-level set containment has different behavior for both cases. + let hash = match cls.hash(vm) { + Ok(hash) => hash, + Err(err) if err.class().is(vm.ctx.exceptions.type_error) => { + return Err(vm.new_type_error(format!( + "cannot use '{}' as a set element ({})", + cls.class().fully_qualified_name(vm)?, + err.as_object().str(vm)? + ))); + } + Err(err) => return Err(err), + }; + for typ in abstract_types() { + let typ = typ.as_object(); + if typ.hash(vm)? == hash + && typ.rich_compare_bool(cls, crate::types::PyComparisonOp::Eq, vm)? + { + return Ok(true); + } + } + Ok(false) + } + impl Constructor for NodeAst { type Args = FuncArgs; @@ -345,6 +327,21 @@ pub(crate) mod _ast { type Args = FuncArgs; fn slot_init(zelf: &PyObject, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> { + if is_abstract(zelf.class().as_object(), vm)? { + let message = vm.ctx.new_str(format!( + "Instantiating abstract AST node class {} is deprecated. \ +This will become an error in Python 3.20", + zelf.class().fully_qualified_name(vm)? + )); + warn::warn( + message.into(), + Some(vm.ctx.exceptions.deprecation_warning.to_owned()), + 1, + None, + vm, + )?; + } + let fields = zelf .class() .get_attr(vm.ctx.intern_str("_fields")) diff --git a/crates/vm/src/stdlib/_ast/statement.rs b/crates/vm/src/stdlib/_ast/statement.rs index 59950d6af75..7f0b5d75f5f 100644 --- a/crates/vm/src/stdlib/_ast/statement.rs +++ b/crates/vm/src/stdlib/_ast/statement.rs @@ -1484,7 +1484,11 @@ fn stmt_import_from_object_with_range( node_index: range.node_index(), names: get_node_list_field(vm, source_file, object, "names", "Import")?, range: range.range, - is_lazy: false, + is_lazy: get_node_field_opt(vm, object, "is_lazy")? + .map(|value| node_object_to_i32(vm, &value)) + .transpose()? + .unwrap_or(0) + != 0, }) } @@ -1494,7 +1498,7 @@ impl Node for ast::StmtImport { node_index, names, range: _range, - is_lazy: _, + is_lazy, } = self; let node = NodeAst .into_ref_with_type(vm, pyast::NodeStmtImport::static_type().to_owned()) @@ -1502,6 +1506,8 @@ impl Node for ast::StmtImport { let dict = node.as_object().dict().unwrap(); dict.set_item("names", names.ast_to_object(vm, source_file), vm) .unwrap(); + dict.set_item("is_lazy", vm.ctx.new_int(i32::from(is_lazy)).into(), vm) + .unwrap(); node_add_location_for_node(&dict, _range, vm, source_file, node_index.load()); node.into() } @@ -1532,7 +1538,11 @@ fn stmt_import_from_from_object_with_range( names: get_node_list_field(vm, source_file, object, "names", "ImportFrom")?, level, range: range.range, - is_lazy: false, + is_lazy: get_node_field_opt(vm, object, "is_lazy")? + .map(|value| node_object_to_i32(vm, &value)) + .transpose()? + .unwrap_or(0) + != 0, runtime_level, }) } @@ -1561,7 +1571,7 @@ impl Node for ast::StmtImportFrom { names, level, range, - is_lazy: _, + is_lazy, runtime_level, } = self; let node = NodeAst @@ -1577,6 +1587,8 @@ impl Node for ast::StmtImportFrom { |level| vm.ctx.new_int(level).to_pyobject(vm), ); dict.set_item("level", level, vm).unwrap(); + dict.set_item("is_lazy", vm.ctx.new_int(i32::from(is_lazy)).into(), vm) + .unwrap(); node_add_location_for_node(&dict, range, vm, source_file, node_index.load()); node.into() } diff --git a/crates/vm/src/stdlib/_ast/string.rs b/crates/vm/src/stdlib/_ast/string.rs index 4cee870f92d..03daf9144e7 100644 --- a/crates/vm/src/stdlib/_ast/string.rs +++ b/crates/vm/src/stdlib/_ast/string.rs @@ -1000,38 +1000,34 @@ fn push_ruff_tstring_element( constant.node_index = literal.node_index; output.push(TemplateStrPart::Constant(constant)); } - ast::InterpolatedStringElement::Interpolation(ast::InterpolatedElement { - range, - expression, - debug_text, - mut conversion, - format_spec, - node_index, - runtime_str, - runtime_interpolation_format_spec, - runtime_formatted_value_format_spec: _, - }) => { - let expr_range = - extend_expr_range_with_wrapping_parens(source_file, range, expression.range()) - .unwrap_or_else(|| expression.range()); - let expr_str = if let Some(debug_text) = &debug_text { + ast::InterpolatedStringElement::Interpolation(interpolation) => { + let expr_str = + rustpython_codegen::interpolation_expression_text(source_file, &interpolation) + .unwrap_or_else(|| { + source_file + .source_text() + .slice(interpolation.expression.range()) + .to_owned() + }); + let ast::InterpolatedElement { + range, + expression, + debug_text, + mut conversion, + format_spec, + node_index, + runtime_str, + runtime_interpolation_format_spec, + .. + } = interpolation; + if let Some(debug_text) = &debug_text { output.push(TemplateStrPart::Constant(interpolation_debug_constant( source_file, debug_text, expression.range(), ))); conversion = debug_conversion(conversion, format_spec.is_some()); - let expr_source = source_file.source_text().slice(expr_range); - let mut expr_with_debug = String::with_capacity( - debug_text.leading().len() + expr_source.len() + debug_text.trailing().len(), - ); - expr_with_debug.push_str(debug_text.leading()); - expr_with_debug.push_str(expr_source); - expr_with_debug.push_str(debug_text.trailing()); - strip_interpolation_expr(&expr_with_debug) - } else { - tstring_interpolation_expr_str(source_file, range, expr_range) - }; + } let runtime_interpolation = super::constant::runtime_interpolation_object( vm, runtime_str, @@ -1056,85 +1052,6 @@ fn push_ruff_tstring_element( } } -fn tstring_interpolation_expr_str( - source_file: &AstSourceContext, - interpolation_range: TextRange, - expr_range: TextRange, -) -> String { - let expr_range = - extend_expr_range_with_wrapping_parens(source_file, interpolation_range, expr_range) - .unwrap_or(expr_range); - let start = interpolation_range.start() + TextSize::from(1); - let start = if start > expr_range.end() { - expr_range.start() - } else { - start - }; - let expr_source = source_file - .source_text() - .slice(TextRange::new(start, expr_range.end())); - strip_interpolation_expr(expr_source) -} - -fn extend_expr_range_with_wrapping_parens( - source_file: &AstSourceContext, - interpolation_range: TextRange, - expr_range: TextRange, -) -> Option { - let left_slice = source_file.source_text().slice(TextRange::new( - interpolation_range.start(), - expr_range.start(), - )); - let mut left_char: Option<(usize, char)> = None; - for (idx, ch) in left_slice - .char_indices() - .collect::>() - .into_iter() - .rev() - { - if !ch.is_whitespace() { - left_char = Some((idx, ch)); - break; - } - } - let (left_idx, left_ch) = left_char?; - if left_ch != '(' { - return None; - } - - let right_slice = source_file - .source_text() - .slice(TextRange::new(expr_range.end(), interpolation_range.end())); - let mut right_char: Option<(usize, char)> = None; - for (idx, ch) in right_slice.char_indices() { - if !ch.is_whitespace() { - right_char = Some((idx, ch)); - break; - } - } - let (right_idx, right_ch) = right_char?; - if right_ch != ')' { - return None; - } - - let left_pos = interpolation_range.start() + TextSize::from(left_idx as u32); - let right_pos = expr_range.end() + TextSize::from(right_idx as u32); - Some(TextRange::new(left_pos, right_pos + TextSize::from(1))) -} - -fn strip_interpolation_expr(expr_source: &str) -> String { - let mut end = expr_source.len(); - for (idx, ch) in expr_source.char_indices().rev() { - if ch.is_whitespace() || ch == '=' { - end = idx; - continue; - } - end = idx + ch.len_utf8(); - break; - } - expr_source[..end].to_owned() -} - #[derive(Debug)] pub(super) struct TemplateStr { node_index: ast::AtomicNodeIndex, diff --git a/crates/vm/src/stdlib/_ast/validate.rs b/crates/vm/src/stdlib/_ast/validate.rs index 2bd51611e12..b2d2b0a4b8b 100644 --- a/crates/vm/src/stdlib/_ast/validate.rs +++ b/crates/vm/src/stdlib/_ast/validate.rs @@ -118,6 +118,21 @@ fn validate_nonempty_seq( Ok(()) } +fn validate_import_names( + vm: &VirtualMachine, + names: &[ast::Alias], + owner: &'static str, +) -> PyResult<()> { + validate_nonempty_seq(vm, names.len(), "names", owner)?; + for alias in names { + validate_name(vm, alias.name.id())?; + if let Some(asname) = &alias.asname { + validate_name(vm, asname.id())?; + } + } + Ok(()) +} + fn validate_assignlist( vm: &VirtualMachine, targets: &[ast::Expr], @@ -739,6 +754,7 @@ fn validate_stmt(vm: &VirtualMachine, stmt: &ast::Stmt) -> PyResult<()> { "FunctionDef" }; validate_body(vm, &func.body, func.runtime_body.as_ref(), owner)?; + validate_name(vm, func.name.id())?; validate_type_params(vm, func.type_params.as_deref())?; validate_parameters(vm, &func.parameters)?; validate_runtime_expr_list_slots( @@ -759,6 +775,7 @@ fn validate_stmt(vm: &VirtualMachine, stmt: &ast::Stmt) -> PyResult<()> { class_def.runtime_body.as_ref(), "ClassDef", )?; + validate_name(vm, class_def.name.id())?; validate_type_params(vm, class_def.type_params.as_deref())?; if let Some(arguments) = &class_def.arguments { validate_runtime_expr_list_slots( @@ -909,6 +926,11 @@ fn validate_stmt(vm: &VirtualMachine, stmt: &ast::Stmt) -> PyResult<()> { if let Some(type_expr) = &handler.type_ { validate_expr(vm, type_expr, ast::ExprContext::Load)?; } + if !try_stmt.is_star + && let Some(name) = &handler.name + { + validate_name(vm, name.id())?; + } validate_body( vm, &handler.body, @@ -928,18 +950,14 @@ fn validate_stmt(vm: &VirtualMachine, stmt: &ast::Stmt) -> PyResult<()> { } Ok(()) } - ast::Stmt::Import(import) => { - validate_nonempty_seq(vm, import.names.len(), "names", "Import")?; - Ok(()) - } + ast::Stmt::Import(import) => validate_import_names(vm, &import.names, "Import"), ast::Stmt::ImportFrom(import) => { if let Some(level) = import.runtime_level && level < 0 { return Err(vm.new_value_error("Negative ImportFrom level")); } - validate_nonempty_seq(vm, import.names.len(), "names", "ImportFrom")?; - Ok(()) + validate_import_names(vm, &import.names, "ImportFrom") } ast::Stmt::Global(global) => { validate_nonempty_seq(vm, global.names.len(), "names", "Global")?; diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index d9ee8d1bba7..6d3a6b0b868 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -227,17 +227,15 @@ mod _imp { Vec::new() } - // CPython removes the name from its pending lazy-module registry here. - // RustPython currently performs only eager imports, so that registry is empty. + // This hook is also used by eager imports that satisfy pending lazy targets. #[pyfunction] fn _set_lazy_attributes( _modobj: PyObjectRef, name: PyStrRef, vm: &VirtualMachine, ) -> PyResult<()> { - // Even an empty set checks the hash of a str subclass. - name.as_object().hash(vm)?; - Ok(()) + crate::lazy_import::clear_submodule(&name, false, vm)?; + vm.state.lazy_imports.modules.discard(name.into(), vm) } #[pyfunction] diff --git a/crates/vm/src/stdlib/_sysconfig.rs b/crates/vm/src/stdlib/_sysconfig.rs index 2de27d71ac9..5581be0227f 100644 --- a/crates/vm/src/stdlib/_sysconfig.rs +++ b/crates/vm/src/stdlib/_sysconfig.rs @@ -1,5 +1,44 @@ pub(crate) use _sysconfig::module_def; +#[pymodule] +pub(crate) mod _missing_stdlib_info { + use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; + + #[pyattr(name = "_MISSING_STDLIB_MODULE_MESSAGES")] + fn missing_stdlib_module_messages(vm: &VirtualMachine) -> PyDictRef { + let messages = vm.ctx.new_dict(); + // CPython's Tools/build/check_extension_modules.py emits these entries + // unconditionally, including on Windows. Do not infer disabled build + // features from CPython's module inventory: ours is different. + for name in [ + "_overlapped", + "_testconsole", + "_winapi", + "_wmi", + "msvcrt", + "nt", + "winreg", + "winsound", + ] { + let message = + format!("Unsupported platform for Windows-only standard library module '{name}'"); + messages + .set_item(name, message.to_pyobject(vm), vm) + .unwrap(); + } + // This module is only implemented for macOS in our stdlib as well. + #[cfg(not(target_os = "macos"))] + messages + .set_item( + "_scproxy", + "Unsupported platform for standard library module '_scproxy'".to_pyobject(vm), + vm, + ) + .unwrap(); + messages + } +} + #[pymodule] pub(crate) mod _sysconfig { use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject}; diff --git a/crates/vm/src/stdlib/_types.rs b/crates/vm/src/stdlib/_types.rs index 081097e01de..f45d513fc5d 100644 --- a/crates/vm/src/stdlib/_types.rs +++ b/crates/vm/src/stdlib/_types.rs @@ -63,6 +63,11 @@ mod _types { vm.ctx.types.frame_locals_proxy_type.to_owned().into() } + #[pyattr] + fn LazyImportType(vm: &VirtualMachine) -> PyObjectRef { + vm.ctx.types.lazy_import_type.to_owned().into() + } + #[pyattr] fn FrameType(vm: &VirtualMachine) -> PyObjectRef { vm.ctx.types.frame_type.to_owned().into() diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index e4c21007a0b..96b6db38ab3 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -1320,6 +1320,53 @@ mod builtins { ) } + #[derive(FromArgs)] + struct LazyImportArgs { + #[pyarg(any)] + name: PyObjectRef, + #[pyarg(any, optional)] + globals: OptionalArg, + #[pyarg(any, optional)] + locals: Option, + #[pyarg(any, default, py_default = "()")] + fromlist: Option, + #[pyarg(any, default)] + level: i32, + } + + #[pyfunction] + fn __lazy_import__(args: LazyImportArgs, vm: &VirtualMachine) -> PyResult { + let globals = args + .globals + .into_option() + .or_else(|| crate::frame::current_globals().map(Into::into)) + .ok_or_else(|| { + vm.new_type_error("__lazy_import__() missing globals when called without a frame") + })?; + let globals_dict = globals + .downcast_ref::() + .ok_or_else(|| { + vm.new_type_error(format!( + "expect dict for globals, got {}", + globals.class().name() + )) + })?; + let builtins = globals_dict + .inner_getitem_opt(identifier!(vm, __builtins__), vm)? + .ok_or_else(|| vm.new_value_error("unable to get builtins for lazy import"))?; + let builtins = if let Some(module) = builtins.downcast_ref::() { + module.dict().into() + } else { + builtins + }; + let name = args + .name + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("module name must be a string"))?; + let _ = args.locals; + crate::lazy_import::create(name, &globals, args.fromlist, args.level, builtins, vm) + } + #[pyfunction] fn vars(obj: OptionalArg, vm: &VirtualMachine) -> PyResult { if let OptionalArg::Present(obj) = obj { @@ -1570,6 +1617,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { "BufferError" => ctx.exceptions.buffer_error.to_owned(), "EOFError" => ctx.exceptions.eof_error.to_owned(), "ImportError" => ctx.exceptions.import_error.to_owned(), + "ImportCycleError" => ctx.exceptions.import_cycle_error.to_owned(), "ModuleNotFoundError" => ctx.exceptions.module_not_found_error.to_owned(), "LookupError" => ctx.exceptions.lookup_error.to_owned(), "IndexError" => ctx.exceptions.index_error.to_owned(), diff --git a/crates/vm/src/stdlib/mod.rs b/crates/vm/src/stdlib/mod.rs index d5dde116ec1..b1442645b7f 100644 --- a/crates/vm/src/stdlib/mod.rs +++ b/crates/vm/src/stdlib/mod.rs @@ -140,6 +140,7 @@ pub fn builtin_module_defs(ctx: &Context) -> Vec<&'static PyModuleDef> { _symtable::module_def(ctx), _sysconfigdata::module_def(ctx), _sysconfig::module_def(ctx), + _sysconfig::_missing_stdlib_info::module_def(ctx), #[cfg(feature = "threading")] _thread::module_def(ctx), _interpchannels::module_def(ctx), diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 44c3d57e576..775296624ee 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -960,6 +960,64 @@ pub mod sys { } } + #[pyfunction] + fn get_lazy_imports(vm: &VirtualMachine) -> &'static str { + if vm + .state + .lazy_imports + .all + .load(core::sync::atomic::Ordering::Acquire) + { + "all" + } else { + "normal" + } + } + + #[pyfunction] + fn set_lazy_imports(mode: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let mode = mode + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("mode must be a string: 'normal' or 'all'"))?; + let all = match mode.to_str() { + Some("normal") => false, + Some("all") => true, + _ => return Err(vm.new_value_error("mode must be 'normal' or 'all'")), + }; + vm.state + .lazy_imports + .all + .store(all, core::sync::atomic::Ordering::Release); + Ok(()) + } + + #[pyfunction] + fn get_lazy_imports_filter(vm: &VirtualMachine) -> PyObjectRef { + vm.state + .lazy_imports + .filter() + .unwrap_or_else(|| vm.ctx.none()) + } + + #[pyfunction] + fn set_lazy_imports_filter(filter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let filter = if vm.is_none(&filter) { + None + } else { + if !filter.is_callable() { + return Err(vm.new_value_error("filter provided but is not callable")); + } + Some(filter) + }; + vm.state.lazy_imports.set_filter(filter); + Ok(()) + } + + #[pyattr] + fn lazy_modules(vm: &VirtualMachine) -> PyObjectRef { + vm.state.lazy_imports.modules.clone().into() + } + #[pyfunction] fn exc_info(vm: &VirtualMachine) -> (PyObjectRef, PyObjectRef, PyObjectRef) { match vm.topmost_exception() { @@ -1606,6 +1664,11 @@ pub mod sys { fn thread_inherit_context(&self, vm: &VirtualMachine) -> bool { vm.state.config.settings.thread_inherit_context } + + #[pygetset] + fn lazy_imports(&self, vm: &VirtualMachine) -> i8 { + vm.state.config.settings.lazy_imports + } } #[cfg(feature = "threading")] diff --git a/crates/vm/src/types/zoo.rs b/crates/vm/src/types/zoo.rs index 5291e677add..3076a7b79d7 100644 --- a/crates/vm/src/types/zoo.rs +++ b/crates/vm/src/types/zoo.rs @@ -47,6 +47,7 @@ pub struct TypeZoo { pub iter_type: &'static Py, pub reverse_iter_type: &'static Py, pub complex_type: &'static Py, + pub lazy_import_type: &'static Py, pub list_type: &'static Py, pub list_iterator_type: &'static Py, pub list_reverseiterator_type: &'static Py, @@ -143,6 +144,7 @@ impl TypeZoo { frozendict_type: dict::PyFrozenDict::init_builtin_type(), frozenset_type: set::PyFrozenSet::init_builtin_type(), filter_type: filter::PyFilter::init_builtin_type(), + lazy_import_type: crate::lazy_import::PyLazyImport::init_builtin_type(), list_type: list::PyList::init_builtin_type(), map_type: map::PyMap::init_builtin_type(), memoryview_type: memory::PyMemoryView::init_builtin_type(), @@ -237,6 +239,7 @@ impl TypeZoo { set::init(context); tuple::init(context); dict::init(context); + crate::lazy_import::init_type(context); builtin_func::init(context); function::init(context); staticmethod::init(context); diff --git a/crates/vm/src/vm/compile.rs b/crates/vm/src/vm/compile.rs index 2f13fb63a4c..575e954d034 100644 --- a/crates/vm/src/vm/compile.rs +++ b/crates/vm/src/vm/compile.rs @@ -433,7 +433,24 @@ impl VirtualMachine { None }; + let emitted_parser_escapes = if is_ast_only { + self.emit_ast_parser_warnings( + source, + filename, + module, + start, + future_features.contains(crate::bytecode::CodeFlags::FUTURE_BARRY_AS_BDFL), + ) + .map_err(|warning| { + VmCompileError::Warning(warning).into_pyexception(self, Some(source)) + })? + } else { + Vec::new() + }; let mut emit_ast_warning = |offset, ch| { + if emitted_parser_escapes.binary_search(&offset).is_ok() { + return Ok(()); + } escape_warnings::warn_invalid_escape_sequence( source, escape_warnings::InvalidEscape::Char { ch, offset }, @@ -700,6 +717,26 @@ mod escape_warnings { Octal { digits: [u8; 3], offset: usize }, } + #[derive(Clone, Copy, PartialEq, Eq)] + enum EscapeOrigin { + Parser, + Tokenizer, + } + + enum ParserWarning { + Escape(InvalidEscape, EscapeOrigin), + Numeric { offset: usize, message: String }, + } + + impl ParserWarning { + fn offset(&self) -> usize { + match self { + Self::Escape(escape, _) => escape.offset(), + Self::Numeric { offset, .. } => *offset, + } + } + } + impl InvalidEscape { fn offset(&self) -> usize { match self { @@ -1158,38 +1195,45 @@ mod escape_warnings { ) } - fn warn_quoted_literal( + fn collect_quoted_literal_warning( source: &str, - quote_index: usize, + literal_start: usize, end: usize, is_bytes: bool, - filename: &str, - module: Option<&Py>, - vm: &VirtualMachine, - ) -> Result<(), CompileWarningError> { + warnings: &mut Vec, + ) { if let Some((start, content_end)) = - content_bounds(source, text_range_from_bounds(quote_index, end)) + content_bounds(source, text_range_from_bounds(literal_start, end)) && let Some(escape) = first_invalid_escape(source, start, content_end, is_bytes) { - warn_invalid_escape_sequence(source, escape, filename, module, vm)?; + warnings.push(ParserWarning::Escape(escape, EscapeOrigin::Parser)); } - Ok(()) } - fn warn_fstring_literal_part( + fn collect_fstring_literal_warning( source: &str, start: usize, end: usize, - filename: &str, - module: Option<&Py>, - vm: &VirtualMachine, - ) -> Result<(), CompileWarningError> { + warnings: &mut Vec, + ) { if start >= end || end > source.len() { - return Ok(()); + return; } if let Some(escape) = first_invalid_escape(source, start, end, false) { - return warn_invalid_escape_sequence(source, escape, filename, module, vm); - } + let origin = if matches!(&escape, InvalidEscape::Char { ch: '{' | '}', .. }) { + EscapeOrigin::Tokenizer + } else { + EscapeOrigin::Parser + }; + warnings.push(ParserWarning::Escape(escape, origin)); + return; + } + // In CPython, _PyTokenizer_warn_invalid_escape_sequence handles + // `\{` and `\}` for FSTRING_MIDDLE/FSTRING_END tokens. Ruff + // splits the literal element before the interpolation delimiter, + // so the `\` sits at the end of the literal range and the `{`/`}` + // sits just after it. Only warn when the number of trailing + // backslashes is odd (an even count means they are all escaped). let trailing_bs = source.as_bytes()[start..end] .iter() .rev() @@ -1199,18 +1243,14 @@ mod escape_warnings { && let Some(&after) = source.as_bytes().get(end) && (after == b'{' || after == b'}') { - warn_invalid_escape_sequence( - source, + warnings.push(ParserWarning::Escape( InvalidEscape::Char { ch: after as char, offset: end - 1, }, - filename, - module, - vm, - )?; + EscapeOrigin::Tokenizer, + )); } - Ok(()) } fn find_interpolation_end(bytes: &[u8], open: usize, limit: usize) -> usize { @@ -1267,10 +1307,8 @@ mod escape_warnings { start: usize, end: usize, is_raw: bool, - filename: &str, - module: Option<&Py>, - vm: &VirtualMachine, - ) -> Result<(), CompileWarningError> { + warnings: &mut Vec, + ) { let bytes = source.as_bytes(); let mut index = start; let mut literal_start = start; @@ -1281,26 +1319,17 @@ mod escape_warnings { } b'{' => { if !is_raw { - warn_fstring_literal_part( - source, - literal_start, - index, - filename, - module, - vm, - )?; + collect_fstring_literal_warning(source, literal_start, index, warnings); } let close = find_interpolation_end(bytes, index, end); let body_end = if close > index + 1 { close - 1 } else { close }; - emit_string_escape_warnings_in_range( + collect_string_escape_warnings_in_range( source, index + 1, body_end, Some(is_raw), - filename, - module, - vm, - )?; + warnings, + ); index = close.max(index + 1); literal_start = index; } @@ -1312,31 +1341,19 @@ mod escape_warnings { } } if !is_raw { - warn_fstring_literal_part(source, literal_start, end, filename, module, vm)?; + collect_fstring_literal_warning(source, literal_start, end, warnings); } - Ok(()) } /// Scan quoted literals without a successful parse, matching the tokenizer /// path that warns before the parser rejects the rest of the source. - fn emit_string_escape_warnings_unparsed( - source: &str, - filename: &str, - module: Option<&Py>, - vm: &VirtualMachine, - ) -> Result<(), CompileWarningError> { - emit_string_escape_warnings_in_range(source, 0, source.len(), None, filename, module, vm) - } - - fn emit_string_escape_warnings_in_range( + fn collect_string_escape_warnings_in_range( source: &str, start: usize, end: usize, format_spec_raw: Option, - filename: &str, - module: Option<&Py>, - vm: &VirtualMachine, - ) -> Result<(), CompileWarningError> { + warnings: &mut Vec, + ) { let bytes = source.as_bytes(); let end = end.min(bytes.len()); let mut index = start; @@ -1368,21 +1385,24 @@ mod escape_warnings { content_start, content_end.min(end), is_raw, - filename, - module, - vm, - )?; + warnings, + ); } } else if !is_raw { - warn_quoted_literal( - source, index, quote_end, is_bytes, filename, module, vm, - )?; + collect_quoted_literal_warning( + // Non-raw ordinary prefixes are one-byte b/u. + source, + index - 1, + quote_end, + is_bytes, + warnings, + ); } } StringPrefix::None => { - warn_quoted_literal( - source, index, quote_end, false, filename, module, vm, - )?; + collect_quoted_literal_warning( + source, index, quote_end, false, warnings, + ); } StringPrefix::Invalid => {} } @@ -1393,16 +1413,8 @@ mod escape_warnings { && bracket == 0 && brace == 0 => { - scan_interpolated_content( - source, - index + 1, - end, - is_raw, - filename, - module, - vm, - )?; - return Ok(()); + scan_interpolated_content(source, index + 1, end, is_raw, warnings); + return; } b'(' if format_spec_raw.is_some() => { paren += 1; @@ -1431,15 +1443,9 @@ mod escape_warnings { _ => index += 1, } } - Ok(()) } - fn emit_numeric_literal_warnings( - source: &str, - filename: &str, - module: Option<&Py>, - vm: &VirtualMachine, - ) -> Result<(), CompileWarningError> { + fn collect_numeric_literal_warnings(source: &str, warnings: &mut Vec) { let bytes = source.as_bytes(); let mut index = 0; while index < bytes.len() { @@ -1466,64 +1472,39 @@ mod escape_warnings { continue; }; if end > index && numeric_keyword_suffix(&bytes[end..]) { - warn_syntax_at_offset( - source, - filename, - module, - index, - format!("invalid {kind} literal"), - vm, - )?; + warnings.push(ParserWarning::Numeric { + offset: index, + message: format!("invalid {kind} literal"), + }); } index = end.max(index + 1); } _ => index += 1, } } - Ok(()) } - struct EscapeWarningVisitor<'a> { + struct EscapeWarningVisitor<'a, 'w> { source: &'a str, - filename: &'a str, - module: Option<&'a Py>, - vm: &'a VirtualMachine, - error: Option, - emitted_escapes: &'a [usize], + warnings: &'w mut Vec, + ast_only: bool, /// This pass runs before the compile that rejects an over-nested /// tree, so it has to stop itself. depth: usize, depth_limit: usize, } - impl<'a> EscapeWarningVisitor<'a> { - fn record_warning(&mut self, result: Result<(), CompileWarningError>) { - if self.error.is_none() - && let Err(err) = result - { - self.error = Some(err); - } - } - + impl<'a> EscapeWarningVisitor<'a, '_> { /// Check a quoted string/bytes literal for invalid escapes. /// The range must include the prefix and quote delimiters. fn check_quoted_literal(&mut self, range: TextRange, is_bytes: bool) { - if let Some((start, end)) = content_bounds(self.source, range) - && let Some(escape) = first_invalid_escape(self.source, start, end, is_bytes) - && self - .emitted_escapes - .binary_search(&escape.offset()) - .is_err() - { - let result = warn_invalid_escape_sequence( - self.source, - escape, - self.filename, - self.module, - self.vm, - ); - self.record_warning(result); - } + collect_quoted_literal_warning( + self.source, + range.start().to_usize(), + range.end().to_usize(), + is_bytes, + self.warnings, + ); } /// Check an f-string literal element for invalid escapes. @@ -1533,70 +1514,32 @@ mod escape_warnings { /// equivalent to `_PyTokenizer_warn_invalid_escape_sequence` handling /// `FSTRING_MIDDLE` / `FSTRING_END` tokens. fn check_fstring_literal(&mut self, range: TextRange) { - let start = range.start().to_usize(); - let end = range.end().to_usize(); - if start >= end || end > self.source.len() { - return; - } - if let Some(escape) = first_invalid_escape(self.source, start, end, false) { - if self.emitted_escapes.binary_search(&escape.offset()).is_ok() { - return; - } - let result = warn_invalid_escape_sequence( - self.source, - escape, - self.filename, - self.module, - self.vm, - ); - self.record_warning(result); - return; - } - // In CPython, _PyTokenizer_warn_invalid_escape_sequence handles - // `\{` and `\}` for FSTRING_MIDDLE/FSTRING_END tokens. Ruff - // splits the literal element before the interpolation delimiter, - // so the `\` sits at the end of the literal range and the `{`/`}` - // sits just after it. Only warn when the number of trailing - // backslashes is odd (an even count means they are all escaped). - let trailing_bs = self.source.as_bytes()[start..end] - .iter() - .rev() - .take_while(|&&b| b == b'\\') - .count(); - if trailing_bs % 2 == 1 - && let Some(&after) = self.source.as_bytes().get(end) - && (after == b'{' || after == b'}') - && self.emitted_escapes.binary_search(&(end - 1)).is_err() - { - let result = warn_invalid_escape_sequence( - self.source, - InvalidEscape::Char { - ch: after as char, - offset: end - 1, - }, - self.filename, - self.module, - self.vm, - ); - self.record_warning(result); - } + collect_fstring_literal_warning( + self.source, + range.start().to_usize(), + range.end().to_usize(), + self.warnings, + ); } /// Visit f-string elements, checking literals and recursing into /// interpolation expressions and format specs. - fn visit_fstring_elements(&mut self, elements: &'a ast::InterpolatedStringElements) { + fn visit_fstring_elements( + &mut self, + elements: &'a ast::InterpolatedStringElements, + raw: bool, + ) { for element in elements { - if self.error.is_some() { - return; - } match element { ast::InterpolatedStringElement::Literal(lit) => { - self.check_fstring_literal(lit.range); + if !raw { + self.check_fstring_literal(lit.range); + } } ast::InterpolatedStringElement::Interpolation(interp) => { self.visit_expr(&interp.expression); if let Some(spec) = &interp.format_spec { - self.visit_fstring_elements(&spec.elements); + self.visit_fstring_elements(&spec.elements, raw); } } } @@ -1604,11 +1547,8 @@ mod escape_warnings { } } - impl<'a> Visitor<'a> for EscapeWarningVisitor<'a> { + impl<'a> Visitor<'a> for EscapeWarningVisitor<'a, '_> { fn visit_expr(&mut self, expr: &'a ast::Expr) { - if self.error.is_some() { - return; - } match expr { // Regular string literals — decode_unicode_with_escapes path ast::Expr::StringLiteral(string) => { @@ -1646,17 +1586,30 @@ mod escape_warnings { } } ast::FStringPartRef::FString(fstring) => { - if matches!( - fstring.flags.prefix(), - ast::str_prefix::FStringPrefix::Raw { .. } - ) { + if !self.ast_only + && matches!( + fstring.flags.prefix(), + ast::str_prefix::FStringPrefix::Raw { .. } + ) + { continue; } - self.visit_fstring_elements(&fstring.elements); + self.visit_fstring_elements( + &fstring.elements, + matches!( + fstring.flags.prefix(), + ast::str_prefix::FStringPrefix::Raw { .. } + ), + ); } } } } + ast::Expr::TString(tstring) if self.ast_only => { + for part in tstring.value.as_slice() { + self.visit_fstring_elements(&part.elements, part.flags.prefix().is_raw()); + } + } _ => { if self.depth < self.depth_limit { self.depth += 1; @@ -1668,6 +1621,103 @@ mod escape_warnings { } } + fn collect_parser_warnings( + source: &str, + filename: &str, + mode: Option, + inherited_barry_as_flufl: bool, + ) -> (Vec, bool) { + let mut ast_tokenizer_recovery = true; + let mut warnings = Vec::new(); + collect_numeric_literal_warnings(source, &mut warnings); + // The compile that follows rejects this source; parsing it here + // would build a tree that exhausts the stack when dropped. + let source_file = compiler::core::SourceFileBuilder::new(filename, source).finish(); + if compiler::pre_parse_source_error(&source_file).is_ok() + && let Some(mode) = mode + { + let options = ruff_python_parser::ParseOptions::from(mode); + let barry_source = compiler::prepare_barry_as_flufl_source( + source, + options.clone(), + inherited_barry_as_flufl, + ); + let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), options); + if parsed.has_valid_syntax() { + let mut visitor = EscapeWarningVisitor { + source, + warnings: &mut warnings, + ast_only: true, + depth: 0, + depth_limit: compiler::CompileOpts::default().recursion_limit, + }; + match parsed.syntax() { + ast::Mod::Module(module) => { + for stmt in &module.body { + visitor.visit_stmt(stmt); + } + } + ast::Mod::Expression(expr) => visitor.visit_expr(&expr.body), + } + } else { + ast_tokenizer_recovery = false; + // The tokenizer can read the token at which parsing fails, + // but later recovered statements were never parsed by + // CPython and must not produce string/parser warnings. + let end = parsed + .errors() + .iter() + // An unclosed-bracket diagnostic points back to its + // opening token, not where parsing stopped. Other + // errors retain that boundary; otherwise it was EOF. + .filter(|error| { + !matches!( + error.error, + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::UnclosedBracket { .. } + ) + ) + }) + .min_by_key(|error| error.location.start()) + .map_or(source.len(), |error| { + // Tokens after a complete statement or eval + // expression were not decoded by CPython. + if matches!( + error.error, + ruff_python_parser::ParseErrorType::SimpleStatementsOnSameLine + | ruff_python_parser::ParseErrorType::UnexpectedExpressionToken + ) { + error.location.start().to_usize() + } else { + error.location.end().to_usize() + } + }); + collect_string_escape_warnings_in_range(source, 0, end, None, &mut warnings); + } + } else if mode.is_none() && compiler::pre_parse_source_error(&source_file).is_ok() { + ast_tokenizer_recovery = false; + // Function-type input has a separate AST parser; its return + // expression follows an arrow that expression mode rejects. + // There are no import statements in this grammar. + collect_string_escape_warnings_in_range(source, 0, source.len(), None, &mut warnings); + } + warnings.sort_by_key(ParserWarning::offset); + // Interpolated tokenizer warnings have their own stop/error boundary. + // Numeric-only recovery must not skip across one of those events. + if warnings + .iter() + .any(|warning| matches!(warning, ParserWarning::Escape(_, EscapeOrigin::Tokenizer))) + { + ast_tokenizer_recovery = false; + } + (warnings, ast_tokenizer_recovery) + } + + enum WarningEmission<'a> { + Standard { emitted_escapes: &'a [usize] }, + Ast { tokenizer_recovery: bool }, + } + impl VirtualMachine { /// Emit tokenizer-level SyntaxWarnings raised before /// code generation. @@ -1677,7 +1727,18 @@ mod escape_warnings { filename: &str, module: Option<&Py>, ) -> Result<(), CompileWarningError> { - emit_numeric_literal_warnings(source, filename, module, self) + let mut warnings = Vec::new(); + collect_numeric_literal_warnings(source, &mut warnings); + self.emit_parser_warning_batch( + source, + filename, + module, + warnings, + WarningEmission::Standard { + emitted_escapes: &[], + }, + ) + .map(|_| ()) } /// Walk all string literals in `source` and emit `SyntaxWarning` for @@ -1695,33 +1756,125 @@ mod escape_warnings { if compiler::pre_parse_source_error(&source_file).is_err() { return Ok(()); } - let Ok(parsed) = + let mut warnings = Vec::new(); + if let Ok(parsed) = ruff_python_parser::parse(source, ruff_python_parser::Mode::Module.into()) - else { - return emit_string_escape_warnings_unparsed(source, filename, module, self); + { + let mut visitor = EscapeWarningVisitor { + source, + warnings: &mut warnings, + ast_only: false, + depth: 0, + depth_limit: compiler::CompileOpts::default().recursion_limit, + }; + match parsed.syntax() { + ast::Mod::Module(module) => { + for stmt in &module.body { + visitor.visit_stmt(stmt); + } + } + ast::Mod::Expression(expr) => visitor.visit_expr(&expr.body), + } + } else { + collect_string_escape_warnings_in_range( + source, + 0, + source.len(), + None, + &mut warnings, + ); + } + self.emit_parser_warning_batch( + source, + filename, + module, + warnings, + WarningEmission::Standard { emitted_escapes }, + ) + .map(|_| ()) + } + + pub(super) fn emit_ast_parser_warnings( + &self, + source: &str, + filename: &str, + module: Option<&Py>, + start: CompileStart, + inherited_barry_as_flufl: bool, + ) -> Result, CompileWarningError> { + let mode = match start { + CompileStart::Single | CompileStart::File => Some(ruff_python_parser::Mode::Module), + CompileStart::Eval => Some(ruff_python_parser::Mode::Expression), + CompileStart::FuncType => None, }; - let ast = parsed.into_syntax(); - let mut visitor = EscapeWarningVisitor { + let (warnings, recovery) = + collect_parser_warnings(source, filename, mode, inherited_barry_as_flufl); + self.emit_parser_warning_batch( source, filename, module, - vm: self, - error: None, - emitted_escapes, - depth: 0, - depth_limit: compiler::CompileOpts::default().recursion_limit, + warnings, + WarningEmission::Ast { + tokenizer_recovery: recovery + && matches!(start, CompileStart::File | CompileStart::Eval), + }, + ) + } + + fn emit_parser_warning_batch( + &self, + source: &str, + filename: &str, + module: Option<&Py>, + warnings: Vec, + emission: WarningEmission<'_>, + ) -> Result, CompileWarningError> { + let (emitted_escapes, ast_tokenizer_recovery) = match emission { + WarningEmission::Standard { emitted_escapes } => (emitted_escapes, false), + WarningEmission::Ast { tokenizer_recovery } => (&[][..], tokenizer_recovery), }; - match &ast { - ast::Mod::Module(module) => { - for stmt in &module.body { - visitor.visit_stmt(stmt); - } + let mut considered_escapes = Vec::new(); + let mut parser_error = None; + for warning in warnings { + // CPython resumes tokenization after a parser-origin SyntaxError. + // Preserve ordinary warning order and never replay prior warnings. + if parser_error.is_some() && !matches!(warning, ParserWarning::Numeric { .. }) { + continue; } - ast::Mod::Expression(expr) => { - visitor.visit_expr(&expr.body); + match warning { + ParserWarning::Escape(escape, origin) => { + let offset = escape.offset(); + if considered_escapes.last() == Some(&offset) { + continue; + } + considered_escapes.push(offset); + if emitted_escapes.binary_search(&offset).is_err() { + let result = warn_invalid_escape_sequence( + source, escape, filename, module, self, + ); + if let Err(error) = result { + if ast_tokenizer_recovery + && origin == EscapeOrigin::Parser + && error + .exception + .fast_isinstance(self.ctx.exceptions.syntax_warning) + { + parser_error = Some(error); + } else { + return Err(error); + } + } + } + } + ParserWarning::Numeric { offset, message } => { + warn_syntax_at_offset(source, filename, module, offset, message, self)?; + } } } - visitor.error.map_or(Ok(()), Err) + if let Some(error) = parser_error { + return Err(error); + } + Ok(considered_escapes) } } diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index afebf052a1f..d6141223704 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -133,6 +133,7 @@ where // Per-interpreter ephemeral state (must not be shared across interpreters). let codec_registry = CodecsRegistry::new(&ctx); let warnings = WarningsState::init_state(&ctx); + let lazy_imports = crate::lazy_import::LazyImportsState::new(&ctx); let interpreter_id = runtime::alloc_interpreter_id(); let runtime_root_id = parent_state.map_or(interpreter_id, |parent| parent.runtime_root_id); @@ -161,6 +162,7 @@ where atexit_funcs: PyMutex::default(), audit_hooks: PyMutex::default(), codec_registry, + lazy_imports, struct_format_cache: crate::buffer::FormatSpecCache::default(), finalizing: AtomicBool::new(false), #[cfg(feature = "threading")] @@ -269,6 +271,7 @@ fn create_subinterpreter_from_parent( // Every interpreter has a `__main__` module once it is initialized. interp.enter(|vm| { let _ = vm.ensure_main_module(); + vm.apply_startup_lazy_imports(); }); Ok(interp) } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index e6d530abf4a..d34ded1bca9 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -44,7 +44,7 @@ use crate::{ scope::Scope, signal::{self, SignalHandlers}, stdlib, - types::{GetattroFunc, fn_addr}, + types::{GetAttr, GetattroFunc, fn_addr}, warn::WarningsState, }; use alloc::{borrow::Cow, collections::BTreeMap}; @@ -104,6 +104,7 @@ pub struct VirtualMachine { pub(crate) signal_handlers: OnceCell, pub(crate) signal_rx: Option, pub repr_guards: RefCell>, + pub(crate) lazy_imports_resolving: RefCell>, pub state: PyRc, pub initialized: bool, recursion_depth: Cell, @@ -868,6 +869,7 @@ pub struct PyGlobalState { /// `sys.addaudithook` hooks, shared by all threads of this interpreter. pub(crate) audit_hooks: PyMutex>, pub codec_registry: CodecsRegistry, + pub(crate) lazy_imports: crate::lazy_import::LazyImportsState, pub struct_format_cache: crate::buffer::FormatSpecCache, pub finalizing: AtomicBool, /// The thread performing finalization, which need not be the process main thread. @@ -1315,6 +1317,7 @@ impl VirtualMachine { signal_handlers, signal_rx: None, repr_guards: RefCell::default(), + lazy_imports_resolving: RefCell::default(), state, initialized: false, recursion_depth: Cell::new(0), @@ -1461,6 +1464,11 @@ impl VirtualMachine { drop(prewarmed_memory_errors); stdlib::builtins::init_module(self, &self.builtins); + // Explicit lazy imports can occur in frozen startup code. + let lazy_callback = self.builtins.get_attr("__lazy_import__", self); + let lazy_callback = + self.expect_pyresult(lazy_callback, "failed to initialize lazy imports"); + let _ = self.state.lazy_imports.default_callback.set(lazy_callback); let callable_cache_init = self.init_callable_cache(); self.expect_pyresult(callable_cache_init, "failed to initialize callable cache"); stdlib::sys::init_module(self, &self.sys_module, &self.builtins); @@ -1625,6 +1633,14 @@ impl VirtualMachine { self.initialized = true; } + /// Apply configured global lazy imports after startup imports, including site. + /// Hosts that perform their own main-module setup can call this at that boundary. + pub fn apply_startup_lazy_imports(&self) { + if self.state.config.settings.lazy_imports == 1 { + self.state.lazy_imports.all.store(true, Ordering::Release); + } + } + /// Set the custom signal channel for the interpreter pub fn set_user_signal_channel(&mut self, signal_rx: signal::UserSignalReceiver) { self.signal_rx = Some(signal_rx); @@ -3351,6 +3367,7 @@ impl VirtualMachine { if self.is_none(&found) || import::is_module_initializing(&found, self)? { return Ok(None); } + crate::lazy_import::clear_submodule(module, true, self)?; let Some(dot) = name_str.find('.') else { return Ok(Some(found)); @@ -3590,6 +3607,10 @@ impl VirtualMachine { let getattro = obj.class().slots().getattro.load().unwrap(); let result = if fn_addr(getattro) == fn_addr(PyBaseObject::getattro as GetattroFunc) { obj.generic_getattr_opt(attr_name, None, self) + } else if fn_addr(getattro) == fn_addr(PyModule::slot_getattro as GetattroFunc) + && let Some(module) = obj.downcast_ref::() + { + module.getattr_inner(attr_name, true, self).map(Some) } else { obj.get_attr_inner(attr_name, self).map(Some) }; diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index f582e51f1e1..f6354d3310b 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -100,6 +100,10 @@ pub struct Settings { /// -X context_aware_warnings, whether warnings are context aware pub context_aware_warnings: bool, + /// -X lazy_imports, PYTHON_LAZY_IMPORTS: -1 for normal, 1 for all. + /// This startup value is independent of later sys.set_lazy_imports() calls. + pub lazy_imports: i8, + /// -i pub inspect: bool, @@ -208,6 +212,7 @@ impl Default for Settings { warn_default_encoding: false, thread_inherit_context: false, context_aware_warnings: false, + lazy_imports: -1, warnoptions: vec![], path_list: vec![], argv: vec![], diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 3c21fe4f487..8d0aa70e6eb 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -1476,6 +1476,7 @@ impl VirtualMachine { signal_handlers: core::cell::OnceCell::new(), signal_rx: None, repr_guards: RefCell::default(), + lazy_imports_resolving: RefCell::default(), state: self.state.clone(), initialized: self.initialized, recursion_depth: Cell::new(0), diff --git a/scripts/update_lib/cmd_copy_lib.py b/scripts/update_lib/cmd_copy_lib.py index 26b899d21de..848ee9213c2 100644 --- a/scripts/update_lib/cmd_copy_lib.py +++ b/scripts/update_lib/cmd_copy_lib.py @@ -74,6 +74,14 @@ def copy_lib( path_str = str(src_path).replace("\\", "/") if "/Lib/" not in path_str: raise ValueError(f"Path must contain '/Lib/' (got: {src_path})") + if not src_path.exists(): + raise FileNotFoundError(f"Source path does not exist: {src_path}") + + # Data files name an exact asset, not the containing top-level module. + # Expanding Lib/test/data.txt as "test" would replace the entire test tree. + if src_path.is_file() and src_path.suffix != ".py": + _copy_single(src_path, parse_lib_path(src_path), verbose) + return cpython_prefix, after_lib = path_str.split("/Lib/", 1) # Get module name (first component, without .py) diff --git a/scripts/update_lib/tests/test_copy_lib.py b/scripts/update_lib/tests/test_copy_lib.py index 2268a830cc9..69b430af985 100644 --- a/scripts/update_lib/tests/test_copy_lib.py +++ b/scripts/update_lib/tests/test_copy_lib.py @@ -1,5 +1,6 @@ """Tests for copy_lib.py - library copying with dependencies.""" +import contextlib import pathlib import tempfile import unittest @@ -100,6 +101,71 @@ def test_directory_copy_excludes_bytecode_cache_directories(self): class TestCopyLib(unittest.TestCase): """Tests for copy_lib function.""" + def test_nested_data_file_preserves_siblings(self): + from update_lib.cmd_copy_lib import copy_lib + + for name in ("hierarchy.txt", "fixture.bin", "README"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/test/data" / name + source.parent.mkdir(parents=True) + source.write_bytes(b"canonical\x00\xff\n") + (source.parent / "other.txt").write_bytes(b"unselected source") + target = root / "Lib/test/data" / name + target.parent.mkdir(parents=True) + target.write_bytes(b"old") + sibling = root / "Lib/test/test_local.py" + sibling.write_bytes(b"local guards\n") + with contextlib.chdir(root): + copy_lib(source, verbose=False) + self.assertEqual(target.read_bytes(), source.read_bytes()) + self.assertEqual(sibling.read_bytes(), b"local guards\n") + self.assertFalse((target.parent / "other.txt").exists()) + + def test_missing_data_file_does_not_copy_containing_package(self): + from update_lib.cmd_copy_lib import copy_lib + + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/test/missing.txt" + source.parent.mkdir(parents=True) + (source.parent / "other.py").write_text("unselected source") + sibling = root / "Lib/test/test_local.py" + sibling.parent.mkdir(parents=True) + sibling.write_bytes(b"local guards\n") + with contextlib.chdir(root), self.assertRaises(FileNotFoundError): + copy_lib(source, verbose=False) + self.assertEqual(sibling.read_bytes(), b"local guards\n") + self.assertFalse((sibling.parent / "other.py").exists()) + + def test_python_module_still_copies_hard_dependencies(self): + from update_lib.cmd_copy_lib import copy_lib + + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib" + source.mkdir(parents=True) + for name in ("pickle.py", "_compat_pickle.py"): + (source / name).write_text(name) + with contextlib.chdir(root): + copy_lib(source / "pickle.py", verbose=False) + for name in ("pickle.py", "_compat_pickle.py"): + self.assertEqual((root / "Lib" / name).read_text(), name) + + def test_package_still_copies_directory(self): + from update_lib.cmd_copy_lib import copy_lib + + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/json" + source.mkdir(parents=True) + for name in ("__init__.py", "decoder.py"): + (source / name).write_text(name) + with contextlib.chdir(root): + copy_lib(source, verbose=False) + for name in ("__init__.py", "decoder.py"): + self.assertEqual((root / "Lib/json" / name).read_text(), name) + def test_raises_on_path_without_lib(self): """Test that copy_lib raises ValueError when path doesn't contain /Lib/.""" from update_lib.cmd_copy_lib import copy_lib diff --git a/src/lib.rs b/src/lib.rs index 1dd1d7b44b7..e1047a08f3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -319,6 +319,9 @@ fn run_rustpython(vm: &VirtualMachine, run_mode: RunMode) -> PyResult<()> { ); } + // CPython activates configured all-mode after site to avoid startup cycles. + vm.apply_startup_lazy_imports(); + // _PyPathConfig_ComputeSysPath0 - set sys.path[0] after site import if !vm.state.config.settings.safe_path { let path0: Option = match &run_mode { diff --git a/src/settings.rs b/src/settings.rs index 975204c00e0..fa8536573af 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -95,6 +95,8 @@ Options (and corresponding environment variables): also PYTHONWARNINGS=arg -x : skip first line of source, allowing use of non-Unix forms of #!cmd -X opt : set implementation-specific option +-X lazy_imports=all|normal: control global lazy imports (default: normal); + also PYTHON_LAZY_IMPORTS --check-hash-based-pycs always|default|never: control how Python invalidates hash-based .pyc files --help-env: print help about Python environment variables and exit @@ -299,6 +301,12 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { }; } + if let Some(val) = get_env("PYTHON_LAZY_IMPORTS").filter(|val| !val.is_empty()) { + settings.lazy_imports = parse_lazy_imports(val.to_str(), "PYTHON_LAZY_IMPORTS"); + } + + // CPython uses the first occurrence of this -X option. + let mut lazy_imports_option_seen = false; let xopts = args.implementation_option.into_iter().map(|s| { let (name, value) = match s.split_once('=') { Some((name, value)) => (name.to_owned(), Some(value)), @@ -325,6 +333,10 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> { } "no_sig_int" => settings.install_signal_handlers = false, "no_debug_ranges" => settings.code_debug_ranges = false, + "lazy_imports" if !lazy_imports_option_seen => { + settings.lazy_imports = parse_lazy_imports(value, "-X lazy_imports"); + lazy_imports_option_seen = true; + } "cpu_count" => { settings.cpu_count = match parse_cpu_count(value) { Ok(cpu_count) => cpu_count, @@ -488,6 +500,22 @@ fn parse_cpu_count(value: Option<&str>) -> Result, ()> { } } +/// = config_init_lazy_imports +fn parse_lazy_imports(value: Option<&str>, source: &str) -> i8 { + match value { + Some("normal") => -1, + Some("all") => 1, + _ => { + eprintln!( + "Fatal Python error: config_init_lazy_imports: \ + {source}: invalid value; expected 'all' or 'normal'\n\ + Python runtime state: preinitialized\n" + ); + std::process::exit(1); + } + } +} + fn get_paths(env_variable_name: &str) -> impl Iterator + '_ { env::var_os(env_variable_name) .filter(|v| !v.is_empty()) diff --git a/src/shell.rs b/src/shell.rs index 73496981521..0d5ab590faf 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -56,19 +56,27 @@ fn shell_exec( Err(err) => { if matches!( &err, - VmCompileError::Compile(CompileError::Parse(ParseError { - error: ParseErrorType::Lexical( - LexicalErrorType::Eof - | LexicalErrorType::FStringError( - InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, - ) - | LexicalErrorType::UnclosedStringError { - triple_quoted: true, - .. - } - ), - .. - })) + VmCompileError::Compile(CompileError::Parse( + ParseError { + error: ParseErrorType::Lexical( + LexicalErrorType::Eof + | LexicalErrorType::FStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, + ) + | LexicalErrorType::TStringError( + InterpolatedStringErrorType::UnterminatedTripleQuotedString { .. }, + ) + | LexicalErrorType::UnclosedStringError { + triple_quoted: true, + .. + } + ), + .. + } | ParseError { + is_unclosed_bracket: true, + .. + } + )) ) { return ShellExecResult::ContinueLine; }